From bf3f4bcc9b4b754a85abe9f7f4050de188e61a0f Mon Sep 17 00:00:00 2001 From: Romain Bardou Date: Mon, 21 Nov 2022 14:50:42 +0100 Subject: [PATCH 1/3] Scripts: simplify static executable build --- Makefile | 4 ++ manifest/manifest.ml | 13 ------ script-inputs/static-packages | 24 ----------- scripts/build_and_install_static_binaries.sh | 43 -------------------- scripts/ci/build_static_binaries.sh | 35 +++++++++++++++- 5 files changed, 37 insertions(+), 82 deletions(-) delete mode 100644 script-inputs/static-packages delete mode 100755 scripts/build_and_install_static_binaries.sh diff --git a/Makefile b/Makefile index 0a5ecd0c54f7..4f46a14764c3 100644 --- a/Makefile +++ b/Makefile @@ -70,6 +70,10 @@ all: release: @$(MAKE) build PROFILE=release +.PHONY: static +static: + @$(MAKE) build build-unreleased PROFILE=static + .PHONY: build-parameters build-parameters: @dune build --profile=$(PROFILE) $(COVERAGE_OPTIONS) @copy-parameters diff --git a/manifest/manifest.ml b/manifest/manifest.ml index d59410075811..5961469c7523 100644 --- a/manifest/manifest.ml +++ b/manifest/manifest.ml @@ -2843,18 +2843,6 @@ let generate_binaries_for_release () = |> List.iter (fun (full_name : Target.full_name) -> Format.fprintf fmt "%s@." full_name.public_name))) -let generate_static_packages () = - write "script-inputs/static-packages" @@ fun fmt -> - Target.iter_internal_by_opam (fun package internals -> - if - List.exists - (fun (internal : Target.internal) -> - match internal.kind with - | Public_executable _ | Private_executable _ -> internal.static - | Public_library _ | Private_library _ | Test_executable _ -> false) - internals - then Format.fprintf fmt "%s\n" package) - let generate_workspace env dune = let pp_dune fmt dune = if not (Dune.is_empty dune) then Format.fprintf fmt "@.%a@." Dune.pp dune @@ -3336,7 +3324,6 @@ let generate ~make_tezt_exe ~default_profile ~add_to_meta_package = generate_opam_files () ; generate_dune_project_files () ; generate_package_json_file () ; - generate_static_packages () ; let opam_release_graph = compute_opam_release_graph () in generate_opam_ci opam_release_graph ; generate_binaries_for_release () ; diff --git a/script-inputs/static-packages b/script-inputs/static-packages deleted file mode 100644 index 1ffe64ec0990..000000000000 --- a/script-inputs/static-packages +++ /dev/null @@ -1,24 +0,0 @@ -octez-accuser-PtKathma -octez-accuser-PtLimaPt -octez-accuser-alpha -octez-baker-PtKathma -octez-baker-PtLimaPt -octez-baker-alpha -octez-client -octez-codec -octez-dal-node -octez-node -octez-protocol-compiler -octez-proxy-server -octez-sc-rollup-client-alpha -octez-sc-rollup-node-alpha -octez-signer -octez-snoop -octez-tx-rollup-client-PtKathma -octez-tx-rollup-client-PtLimaPt -octez-tx-rollup-client-alpha -octez-tx-rollup-node-PtKathma -octez-tx-rollup-node-PtLimaPt -octez-tx-rollup-node-alpha -octez-wasm-repl -tezos-version diff --git a/scripts/build_and_install_static_binaries.sh b/scripts/build_and_install_static_binaries.sh deleted file mode 100755 index 82fa483d460b..000000000000 --- a/scripts/build_and_install_static_binaries.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env sh - -# Used in the CI to build and install the static binaries. - -set -eu - -if [ $# -ne 1 ]; then - echo "usage: $0 DESTDIR" - exit 1 -fi - -tmp_dir=$(mktemp -dt tezos_static_install.XXXXXXXX) -cleanup () { - set +e - echo Cleaning up... - rm -rf "$tmp_dir" -} -trap cleanup EXIT INT - -# shellcheck disable=SC2046 -dune build --profile static $(xargs -I {} echo {}.install < script-inputs/static-packages) -# shellcheck disable=SC2046 -dune install --profile static --prefix "$tmp_dir" $(cat script-inputs/static-packages) - -mv "$tmp_dir/bin" "$1" - -# Make sure binaries are statically linked -find -L "$1" -type f -not -name "*.sh" | -while read -r b; do - file "$(realpath "$b")" | grep "statically linked"; -done - -# Make sure octez-client knows about build-infos -SHA=$(git rev-parse --short=8 HEAD) -client_version=$("$1/octez-client" --version | cut -f 1 -d ' ') -if [ "$SHA" != "$client_version" ]; then - echo "Unexpected version for octez-client (expected $SHA, found $client_version)" - exit 1 -fi -echo "octez-client --version returned the expected commit hash: $SHA" - -# Trim the dune cache, if used. -dune cache trim --size=250GB diff --git a/scripts/ci/build_static_binaries.sh b/scripts/ci/build_static_binaries.sh index 34b5593480b8..e5d457f8de8f 100755 --- a/scripts/ci/build_static_binaries.sh +++ b/scripts/ci/build_static_binaries.sh @@ -2,10 +2,41 @@ set -eu echo "Create destination directory" -mkdir -pv octez-binaries +mkdir -pv "octez-binaries/$ARCH" echo "Build and install static binaries" -./scripts/build_and_install_static_binaries.sh "octez-binaries/${ARCH}" +make static + +# Trim the dune cache, if used. +dune cache trim --size=250GB + +echo "Check executables and move them to the destination directory" +for executable in $(cat script-inputs/binaries-for-release); do + if [ ! -f "$executable" ]; then + echo "Error: $executable does not exist" + exit 1 + fi + + if (file "$executable" | grep -q "statically linked"); then + echo "Statically linked: $executable" + else + echo "Error: $executable is not statically linked" + exit 1 + fi + + mv "$executable" "octez-binaries/$ARCH/$executable" + # Write access is needed by strip below. + chmod +w "octez-binaries/$ARCH/$executable" +done + +echo 'Check octez-client --version' +SHA=$(git rev-parse --short=8 HEAD) +client_version=$("octez-binaries/$ARCH/octez-client" --version | cut -f 1 -d ' ') +if [ "$SHA" != "$client_version" ]; then + echo "Unexpected version for octez-client (expected $SHA, found $client_version)" + exit 1 +fi +echo "octez-client --version returned the expected commit hash: $SHA" echo "Strip debug symbols and compress binaries (parallelized)" # shellcheck disable=SC2046,SC2038 -- GitLab From dff5683ed69e171092b692c52e39a065661242fa Mon Sep 17 00:00:00 2001 From: Romain Bardou Date: Tue, 22 Nov 2022 15:01:30 +0100 Subject: [PATCH 2/3] CI: {released,experimental}-executables --- .../jobs/build/static_arm64_experimental.yml | 11 ++++++ ..._binaries.yml => static_arm64_release.yml} | 5 +-- .../jobs/build/static_x86_64_experimental.yml | 24 +++++++++++++ .../build/static_x86_64_linux_binaries.yml | 17 --------- .../ci/jobs/build/static_x86_64_release.yml | 12 +++++++ .gitlab/ci/pipelines/before_merging.yml | 2 +- .gitlab/ci/pipelines/master_branch.yml | 4 +-- .gitlab/ci/pipelines/release_tag.yml | 4 +-- .gitlab/ci/pipelines/release_tag_test.yml | 4 +-- .../ci/pipelines/schedule_extended_test.yml | 2 +- manifest/manifest.ml | 35 +++++++++---------- script-inputs/experimental-executables | 8 +++++ ...aries-for-release => released-executables} | 0 scripts/ci/build_static_binaries.sh | 10 +++++- scripts/ci/release.sh | 2 +- 15 files changed, 92 insertions(+), 48 deletions(-) create mode 100644 .gitlab/ci/jobs/build/static_arm64_experimental.yml rename .gitlab/ci/jobs/build/{static_arm64_linux_binaries.yml => static_arm64_release.yml} (57%) create mode 100644 .gitlab/ci/jobs/build/static_x86_64_experimental.yml delete mode 100644 .gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml create mode 100644 .gitlab/ci/jobs/build/static_x86_64_release.yml create mode 100644 script-inputs/experimental-executables rename script-inputs/{binaries-for-release => released-executables} (100%) diff --git a/.gitlab/ci/jobs/build/static_arm64_experimental.yml b/.gitlab/ci/jobs/build/static_arm64_experimental.yml new file mode 100644 index 000000000000..52c239152c48 --- /dev/null +++ b/.gitlab/ci/jobs/build/static_arm64_experimental.yml @@ -0,0 +1,11 @@ +include: .gitlab/ci/jobs/build/common.yml + +# See comments in static_x86_64_experimental.yml. +build:static-arm64-linux-binaries: + extends: + - .build_static_binaries_template + variables: + ARCH: "arm64" + EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" + tags: + - arm64 diff --git a/.gitlab/ci/jobs/build/static_arm64_linux_binaries.yml b/.gitlab/ci/jobs/build/static_arm64_release.yml similarity index 57% rename from .gitlab/ci/jobs/build/static_arm64_linux_binaries.yml rename to .gitlab/ci/jobs/build/static_arm64_release.yml index 86b191e70192..81becf4d396d 100644 --- a/.gitlab/ci/jobs/build/static_arm64_linux_binaries.yml +++ b/.gitlab/ci/jobs/build/static_arm64_release.yml @@ -1,10 +1,11 @@ include: .gitlab/ci/jobs/build/common.yml -# arm64 static binaries are only built for master and releases. +# See comments in static_x86_64_experimental.yml. build:static-arm64-linux-binaries: extends: - .build_static_binaries_template variables: ARCH: "arm64" + EXECUTABLE_FILES: "script-inputs/released-executables" tags: - - arm64 \ No newline at end of file + - arm64 diff --git a/.gitlab/ci/jobs/build/static_x86_64_experimental.yml b/.gitlab/ci/jobs/build/static_x86_64_experimental.yml new file mode 100644 index 000000000000..b88d3c049274 --- /dev/null +++ b/.gitlab/ci/jobs/build/static_x86_64_experimental.yml @@ -0,0 +1,24 @@ +include: .gitlab/ci/jobs/build/common.yml + +# This version of the job builds both released and experimental executables. +# It is used in the following pipelines: +# - Before merging: check whether static executables still compile, +# i.e. that we do pass the -static flag and that when we do it does compile +# - Master branch: executables (including experimental ones) are used in some test networks +# Variants: +# - an arm64 variant exist, but is only used in the master branch pipeline +# (no need to test that we pass the -static flag twice) +# - released variants exist, that are used in release tag pipelines +# (they do not build experimental executables) +build:static-x86_64-linux-binaries: + extends: + - .build_static_binaries_template + # Even though not many tests depend on static executables, some of those that do + # are limiting factors in the total duration of pipelines. + # So we start this job as early as possible, without waiting for sanity_ci. + needs: [trigger] + variables: + ARCH: "x86_64" + EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" + tags: + - build_dedicated diff --git a/.gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml b/.gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml deleted file mode 100644 index 1d0ba5f2f3c9..000000000000 --- a/.gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml +++ /dev/null @@ -1,17 +0,0 @@ -include: .gitlab/ci/jobs/build/common.yml - -# x86_64 static binaries are necessary for development branches, as we want to make -# sure the build does not break. We also need it for release tags, to as its artifacts -# are published in the stage `publish_release`. For this reason, we build it in all pipelines, -# except on latest release branches. -build:static-x86_64-linux-binaries: - extends: - - .build_static_binaries_template - # Even though not many tests depend on static executables, some of those that do - # are limiting factors in the total duration of pipelines. - # So we start this job as early as possible, without waiting for sanity_ci. - needs: [trigger] - variables: - ARCH: "x86_64" - tags: - - build_dedicated diff --git a/.gitlab/ci/jobs/build/static_x86_64_release.yml b/.gitlab/ci/jobs/build/static_x86_64_release.yml new file mode 100644 index 000000000000..2a37903cd72d --- /dev/null +++ b/.gitlab/ci/jobs/build/static_x86_64_release.yml @@ -0,0 +1,12 @@ +include: .gitlab/ci/jobs/build/common.yml + +# See comments in static_x86_64_experimental.yml. +build:static-x86_64-linux-binaries: + extends: + - .build_static_binaries_template + needs: [trigger] + variables: + ARCH: "x86_64" + EXECUTABLE_FILES: "script-inputs/released-executables" + tags: + - build_dedicated diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 41c8d0aea3f9..443b768b4467 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -9,7 +9,7 @@ include: # Stage: build - .gitlab/ci/jobs/build/check.yml - - .gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml + - .gitlab/ci/jobs/build/static_x86_64_experimental.yml - .gitlab/ci/jobs/build/arm64.yml - .gitlab/ci/jobs/build/x86_64.yml - .gitlab/ci/jobs/build/get_contracts.yml diff --git a/.gitlab/ci/pipelines/master_branch.yml b/.gitlab/ci/pipelines/master_branch.yml index 36a26c7bfc0d..83cd91076419 100644 --- a/.gitlab/ci/pipelines/master_branch.yml +++ b/.gitlab/ci/pipelines/master_branch.yml @@ -1,7 +1,7 @@ include: # Stage: build - - .gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml - - .gitlab/ci/jobs/build/static_arm64_linux_binaries.yml + - .gitlab/ci/jobs/build/static_x86_64_experimental.yml + - .gitlab/ci/jobs/build/static_arm64_experimental.yml - .gitlab/ci/jobs/build/arm64.yml - .gitlab/ci/jobs/build/docker_amd64_release.yml - .gitlab/ci/jobs/build/docker_arm64_release.yml diff --git a/.gitlab/ci/pipelines/release_tag.yml b/.gitlab/ci/pipelines/release_tag.yml index afcefdbfe8fb..d734123b6122 100644 --- a/.gitlab/ci/pipelines/release_tag.yml +++ b/.gitlab/ci/pipelines/release_tag.yml @@ -1,7 +1,7 @@ include: # Stage: build - - .gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml - - .gitlab/ci/jobs/build/static_arm64_linux_binaries.yml + - .gitlab/ci/jobs/build/static_x86_64_release.yml + - .gitlab/ci/jobs/build/static_arm64_release.yml - .gitlab/ci/jobs/build/docker_amd64_release.yml - .gitlab/ci/jobs/build/docker_arm64_release.yml diff --git a/.gitlab/ci/pipelines/release_tag_test.yml b/.gitlab/ci/pipelines/release_tag_test.yml index 246770a0658f..a50d5e6ec4f1 100644 --- a/.gitlab/ci/pipelines/release_tag_test.yml +++ b/.gitlab/ci/pipelines/release_tag_test.yml @@ -1,7 +1,7 @@ include: # Stage: build - - .gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml - - .gitlab/ci/jobs/build/static_arm64_linux_binaries.yml + - .gitlab/ci/jobs/build/static_x86_64_release.yml + - .gitlab/ci/jobs/build/static_arm64_release.yml - .gitlab/ci/jobs/build/docker_amd64_test.yml - .gitlab/ci/jobs/build/docker_arm64_test.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index a153c85ba6e4..6813921cd125 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -2,7 +2,7 @@ include: # Stage: build - .gitlab/ci/jobs/build/x86_64.yml - .gitlab/ci/jobs/build/arm64.yml - - .gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml + - .gitlab/ci/jobs/build/static_x86_64_experimental.yml # Stage: test - .gitlab/ci/jobs/test/integration_pytest_old_protocols.yml diff --git a/manifest/manifest.ml b/manifest/manifest.ml index 5961469c7523..96781ce50c43 100644 --- a/manifest/manifest.ml +++ b/manifest/manifest.ml @@ -2824,24 +2824,18 @@ let generate_package_json_file () = pp_dep) (List.sort compare !l) -let generate_binaries_for_release () = - write "script-inputs/binaries-for-release" @@ fun fmt -> - !Target.registered - |> List.iter (fun (internal : Target.internal) -> - match internal.release_status with - | Unreleased | Auto_opam -> () - | Experimental -> - (* [binaries-for-release] is only used in tag releases. *) - () - | Released -> ( - match internal.kind with - | Public_library _ | Private_library _ | Private_executable _ - | Test_executable _ -> - () - | Public_executable ne_list -> - Ne_list.to_list ne_list - |> List.iter (fun (full_name : Target.full_name) -> - Format.fprintf fmt "%s@." full_name.public_name))) +let generate_executable_list filename release_status_to_list = + write filename @@ fun fmt -> + Fun.flip List.iter !Target.registered @@ fun (internal : Target.internal) -> + if internal.release_status = release_status_to_list then + match internal.kind with + | Public_library _ | Private_library _ | Private_executable _ + | Test_executable _ -> + () + | Public_executable ne_list -> + Fun.flip List.iter (Ne_list.to_list ne_list) + @@ fun (full_name : Target.full_name) -> + Format.fprintf fmt "%s@." full_name.public_name let generate_workspace env dune = let pp_dune fmt dune = @@ -3326,7 +3320,10 @@ let generate ~make_tezt_exe ~default_profile ~add_to_meta_package = generate_package_json_file () ; let opam_release_graph = compute_opam_release_graph () in generate_opam_ci opam_release_graph ; - generate_binaries_for_release () ; + generate_executable_list "script-inputs/released-executables" Released ; + generate_executable_list + "script-inputs/experimental-executables" + Experimental ; generate_profiles ~default_profile ; Option.iter (generate_opam_files_for_release diff --git a/script-inputs/experimental-executables b/script-inputs/experimental-executables new file mode 100644 index 000000000000..5fef88016180 --- /dev/null +++ b/script-inputs/experimental-executables @@ -0,0 +1,8 @@ +octez-dal-node +octez-tx-rollup-node-alpha +octez-tx-rollup-client-alpha +octez-wasm-repl-alpha +octez-sc-rollup-node-alpha +octez-sc-rollup-client-alpha +octez-accuser-alpha +octez-baker-alpha diff --git a/script-inputs/binaries-for-release b/script-inputs/released-executables similarity index 100% rename from script-inputs/binaries-for-release rename to script-inputs/released-executables diff --git a/scripts/ci/build_static_binaries.sh b/scripts/ci/build_static_binaries.sh index e5d457f8de8f..c1b6de6f4013 100755 --- a/scripts/ci/build_static_binaries.sh +++ b/scripts/ci/build_static_binaries.sh @@ -11,7 +11,15 @@ make static dune cache trim --size=250GB echo "Check executables and move them to the destination directory" -for executable in $(cat script-inputs/binaries-for-release); do +# Disable https://www.shellcheck.net/wiki/SC2086 because: +# - it's meant to prevent word splitting and we actually want to split words +# in $EXECUTABLE_FILES and we can't use an array because the variable +# comes from the CI +# - it's meant to prevent globbing but actually we don't mind globbing here +# even if we don't use it right now +# shellcheck disable=SC2086 +cat $EXECUTABLE_FILES | +while read -r executable; do if [ ! -f "$executable" ]; then echo "Error: $executable does not exist" exit 1 diff --git a/scripts/ci/release.sh b/scripts/ci/release.sh index 1b33f2d8cc7e..ed049a8101be 100755 --- a/scripts/ci/release.sh +++ b/scripts/ci/release.sh @@ -10,7 +10,7 @@ scripts_dir=$(dirname "$current_dir") src_dir=$(dirname "$scripts_dir") script_inputs_dir="$src_dir/script-inputs" -binaries="$(cat "$script_inputs_dir/binaries-for-release")" +binaries="$(cat "$script_inputs_dir/released-executables")" ### Compute GitLab release names -- GitLab From 0783d8f643f2cd21631c6cb944b1459c46bca6c8 Mon Sep 17 00:00:00 2001 From: Romain Bardou Date: Wed, 23 Nov 2022 17:04:33 +0100 Subject: [PATCH 3/3] Scripts: move set -u from build_static_binaries.sh set -u causes the script to fail if $EXECUTABLE_FILES or $ARCH is unset. But it's not enough: the script should also fail if $EXECUTABLE_FILES or $ARCH is empty. So we add an explicit check at the beginning of the script. This improves the error message in case a variable is unset. --- scripts/ci/build_static_binaries.sh | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/ci/build_static_binaries.sh b/scripts/ci/build_static_binaries.sh index c1b6de6f4013..6a235a18afb8 100755 --- a/scripts/ci/build_static_binaries.sh +++ b/scripts/ci/build_static_binaries.sh @@ -1,5 +1,20 @@ #!/bin/sh -set -eu +set -e + +if [ -z "$EXECUTABLE_FILES" ]; then + echo "Error: environment variable EXECUTABLE_FILES is empty." + echo "Set it to e.g. 'script-inputs/released-executables'" + echo "or to 'script-inputs/released-executables script-inputs/experimental-executables'." + exit 1 +fi + +if [ -z "$ARCH" ]; then + echo "Error: environment variable ARCH is empty." + echo "Set it to e.g. 'x86_64' or 'amd64'." + exit 1 +fi + +set -u echo "Create destination directory" mkdir -pv "octez-binaries/$ARCH" -- GitLab