diff --git a/.gitlab/ci/jobs/build/static_arm64_experimental.yml b/.gitlab/ci/jobs/build/static_arm64_experimental.yml new file mode 100644 index 0000000000000000000000000000000000000000..52c239152c48317ad51dd84bb077cf5ff79d9d90 --- /dev/null +++ b/.gitlab/ci/jobs/build/static_arm64_experimental.yml @@ -0,0 +1,11 @@ +include: .gitlab/ci/jobs/build/common.yml + +# See comments in static_x86_64_experimental.yml. +build:static-arm64-linux-binaries: + extends: + - .build_static_binaries_template + variables: + ARCH: "arm64" + EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" + tags: + - arm64 diff --git a/.gitlab/ci/jobs/build/static_arm64_linux_binaries.yml b/.gitlab/ci/jobs/build/static_arm64_release.yml similarity index 57% rename from .gitlab/ci/jobs/build/static_arm64_linux_binaries.yml rename to .gitlab/ci/jobs/build/static_arm64_release.yml index 86b191e70192b9339ab86d164faf43b61a8ac220..81becf4d396d219aed8a46a5413e380669e7110f 100644 --- a/.gitlab/ci/jobs/build/static_arm64_linux_binaries.yml +++ b/.gitlab/ci/jobs/build/static_arm64_release.yml @@ -1,10 +1,11 @@ include: .gitlab/ci/jobs/build/common.yml -# arm64 static binaries are only built for master and releases. +# See comments in static_x86_64_experimental.yml. build:static-arm64-linux-binaries: extends: - .build_static_binaries_template variables: ARCH: "arm64" + EXECUTABLE_FILES: "script-inputs/released-executables" tags: - - arm64 \ No newline at end of file + - arm64 diff --git a/.gitlab/ci/jobs/build/static_x86_64_experimental.yml b/.gitlab/ci/jobs/build/static_x86_64_experimental.yml new file mode 100644 index 0000000000000000000000000000000000000000..b88d3c0492741486f08d87457d840218446df2cd --- /dev/null +++ b/.gitlab/ci/jobs/build/static_x86_64_experimental.yml @@ -0,0 +1,24 @@ +include: .gitlab/ci/jobs/build/common.yml + +# This version of the job builds both released and experimental executables. +# It is used in the following pipelines: +# - Before merging: check whether static executables still compile, +# i.e. that we do pass the -static flag and that when we do it does compile +# - Master branch: executables (including experimental ones) are used in some test networks +# Variants: +# - an arm64 variant exist, but is only used in the master branch pipeline +# (no need to test that we pass the -static flag twice) +# - released variants exist, that are used in release tag pipelines +# (they do not build experimental executables) +build:static-x86_64-linux-binaries: + extends: + - .build_static_binaries_template + # Even though not many tests depend on static executables, some of those that do + # are limiting factors in the total duration of pipelines. + # So we start this job as early as possible, without waiting for sanity_ci. + needs: [trigger] + variables: + ARCH: "x86_64" + EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" + tags: + - build_dedicated diff --git a/.gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml b/.gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml deleted file mode 100644 index 1d0ba5f2f3c9adc15957b104205fb685d6244c6c..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml +++ /dev/null @@ -1,17 +0,0 @@ -include: .gitlab/ci/jobs/build/common.yml - -# x86_64 static binaries are necessary for development branches, as we want to make -# sure the build does not break. We also need it for release tags, to as its artifacts -# are published in the stage `publish_release`. For this reason, we build it in all pipelines, -# except on latest release branches. -build:static-x86_64-linux-binaries: - extends: - - .build_static_binaries_template - # Even though not many tests depend on static executables, some of those that do - # are limiting factors in the total duration of pipelines. - # So we start this job as early as possible, without waiting for sanity_ci. - needs: [trigger] - variables: - ARCH: "x86_64" - tags: - - build_dedicated diff --git a/.gitlab/ci/jobs/build/static_x86_64_release.yml b/.gitlab/ci/jobs/build/static_x86_64_release.yml new file mode 100644 index 0000000000000000000000000000000000000000..2a37903cd72dc4db4c8347cf5dbc5bae765f9b46 --- /dev/null +++ b/.gitlab/ci/jobs/build/static_x86_64_release.yml @@ -0,0 +1,12 @@ +include: .gitlab/ci/jobs/build/common.yml + +# See comments in static_x86_64_experimental.yml. +build:static-x86_64-linux-binaries: + extends: + - .build_static_binaries_template + needs: [trigger] + variables: + ARCH: "x86_64" + EXECUTABLE_FILES: "script-inputs/released-executables" + tags: + - build_dedicated diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 41c8d0aea3f92575045f8fe83ff4ab2dcea4f9c1..443b768b4467983391a2ac8da00b161ea4efb180 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -9,7 +9,7 @@ include: # Stage: build - .gitlab/ci/jobs/build/check.yml - - .gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml + - .gitlab/ci/jobs/build/static_x86_64_experimental.yml - .gitlab/ci/jobs/build/arm64.yml - .gitlab/ci/jobs/build/x86_64.yml - .gitlab/ci/jobs/build/get_contracts.yml diff --git a/.gitlab/ci/pipelines/master_branch.yml b/.gitlab/ci/pipelines/master_branch.yml index 36a26c7bfc0d76f601f67f7e10b3a95845284799..83cd9107641911ef05754822ab22b7558bba816d 100644 --- a/.gitlab/ci/pipelines/master_branch.yml +++ b/.gitlab/ci/pipelines/master_branch.yml @@ -1,7 +1,7 @@ include: # Stage: build - - .gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml - - .gitlab/ci/jobs/build/static_arm64_linux_binaries.yml + - .gitlab/ci/jobs/build/static_x86_64_experimental.yml + - .gitlab/ci/jobs/build/static_arm64_experimental.yml - .gitlab/ci/jobs/build/arm64.yml - .gitlab/ci/jobs/build/docker_amd64_release.yml - .gitlab/ci/jobs/build/docker_arm64_release.yml diff --git a/.gitlab/ci/pipelines/release_tag.yml b/.gitlab/ci/pipelines/release_tag.yml index afcefdbfe8fbc8c77e46df6c7c97a7ba011f0bc6..d734123b612226615bfa424470fc5c6f8507cc11 100644 --- a/.gitlab/ci/pipelines/release_tag.yml +++ b/.gitlab/ci/pipelines/release_tag.yml @@ -1,7 +1,7 @@ include: # Stage: build - - .gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml - - .gitlab/ci/jobs/build/static_arm64_linux_binaries.yml + - .gitlab/ci/jobs/build/static_x86_64_release.yml + - .gitlab/ci/jobs/build/static_arm64_release.yml - .gitlab/ci/jobs/build/docker_amd64_release.yml - .gitlab/ci/jobs/build/docker_arm64_release.yml diff --git a/.gitlab/ci/pipelines/release_tag_test.yml b/.gitlab/ci/pipelines/release_tag_test.yml index 246770a0658f77393653aaaf5054a6d3408d5db8..a50d5e6ec4f1f175b0091e686f0eeb0687dd72cd 100644 --- a/.gitlab/ci/pipelines/release_tag_test.yml +++ b/.gitlab/ci/pipelines/release_tag_test.yml @@ -1,7 +1,7 @@ include: # Stage: build - - .gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml - - .gitlab/ci/jobs/build/static_arm64_linux_binaries.yml + - .gitlab/ci/jobs/build/static_x86_64_release.yml + - .gitlab/ci/jobs/build/static_arm64_release.yml - .gitlab/ci/jobs/build/docker_amd64_test.yml - .gitlab/ci/jobs/build/docker_arm64_test.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index a153c85ba6e49d897f433a65fb14333952283d67..6813921cd12527c41ec2e6fae2bec2fe6eae936f 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -2,7 +2,7 @@ include: # Stage: build - .gitlab/ci/jobs/build/x86_64.yml - .gitlab/ci/jobs/build/arm64.yml - - .gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml + - .gitlab/ci/jobs/build/static_x86_64_experimental.yml # Stage: test - .gitlab/ci/jobs/test/integration_pytest_old_protocols.yml diff --git a/Makefile b/Makefile index 0a5ecd0c54f7d78934da37d500f936ca7e8d3370..4f46a14764c34d949a079843a7b4a4425eae952a 100644 --- a/Makefile +++ b/Makefile @@ -70,6 +70,10 @@ all: release: @$(MAKE) build PROFILE=release +.PHONY: static +static: + @$(MAKE) build build-unreleased PROFILE=static + .PHONY: build-parameters build-parameters: @dune build --profile=$(PROFILE) $(COVERAGE_OPTIONS) @copy-parameters diff --git a/manifest/manifest.ml b/manifest/manifest.ml index d59410075811dad42ba6a376d7b53cdfa11b2cde..96781ce50c4318e53dfd7b68df4248af6b6a886d 100644 --- a/manifest/manifest.ml +++ b/manifest/manifest.ml @@ -2824,36 +2824,18 @@ let generate_package_json_file () = pp_dep) (List.sort compare !l) -let generate_binaries_for_release () = - write "script-inputs/binaries-for-release" @@ fun fmt -> - !Target.registered - |> List.iter (fun (internal : Target.internal) -> - match internal.release_status with - | Unreleased | Auto_opam -> () - | Experimental -> - (* [binaries-for-release] is only used in tag releases. *) - () - | Released -> ( - match internal.kind with - | Public_library _ | Private_library _ | Private_executable _ - | Test_executable _ -> - () - | Public_executable ne_list -> - Ne_list.to_list ne_list - |> List.iter (fun (full_name : Target.full_name) -> - Format.fprintf fmt "%s@." full_name.public_name))) - -let generate_static_packages () = - write "script-inputs/static-packages" @@ fun fmt -> - Target.iter_internal_by_opam (fun package internals -> - if - List.exists - (fun (internal : Target.internal) -> - match internal.kind with - | Public_executable _ | Private_executable _ -> internal.static - | Public_library _ | Private_library _ | Test_executable _ -> false) - internals - then Format.fprintf fmt "%s\n" package) +let generate_executable_list filename release_status_to_list = + write filename @@ fun fmt -> + Fun.flip List.iter !Target.registered @@ fun (internal : Target.internal) -> + if internal.release_status = release_status_to_list then + match internal.kind with + | Public_library _ | Private_library _ | Private_executable _ + | Test_executable _ -> + () + | Public_executable ne_list -> + Fun.flip List.iter (Ne_list.to_list ne_list) + @@ fun (full_name : Target.full_name) -> + Format.fprintf fmt "%s@." full_name.public_name let generate_workspace env dune = let pp_dune fmt dune = @@ -3336,10 +3318,12 @@ let generate ~make_tezt_exe ~default_profile ~add_to_meta_package = generate_opam_files () ; generate_dune_project_files () ; generate_package_json_file () ; - generate_static_packages () ; let opam_release_graph = compute_opam_release_graph () in generate_opam_ci opam_release_graph ; - generate_binaries_for_release () ; + generate_executable_list "script-inputs/released-executables" Released ; + generate_executable_list + "script-inputs/experimental-executables" + Experimental ; generate_profiles ~default_profile ; Option.iter (generate_opam_files_for_release diff --git a/script-inputs/experimental-executables b/script-inputs/experimental-executables new file mode 100644 index 0000000000000000000000000000000000000000..5fef8801618048f49afc2ed3f1c473dcfc53ce8f --- /dev/null +++ b/script-inputs/experimental-executables @@ -0,0 +1,8 @@ +octez-dal-node +octez-tx-rollup-node-alpha +octez-tx-rollup-client-alpha +octez-wasm-repl-alpha +octez-sc-rollup-node-alpha +octez-sc-rollup-client-alpha +octez-accuser-alpha +octez-baker-alpha diff --git a/script-inputs/binaries-for-release b/script-inputs/released-executables similarity index 100% rename from script-inputs/binaries-for-release rename to script-inputs/released-executables diff --git a/script-inputs/static-packages b/script-inputs/static-packages deleted file mode 100644 index 1ffe64ec09903d2463407179736af81bd80b8c97..0000000000000000000000000000000000000000 --- a/script-inputs/static-packages +++ /dev/null @@ -1,24 +0,0 @@ -octez-accuser-PtKathma -octez-accuser-PtLimaPt -octez-accuser-alpha -octez-baker-PtKathma -octez-baker-PtLimaPt -octez-baker-alpha -octez-client -octez-codec -octez-dal-node -octez-node -octez-protocol-compiler -octez-proxy-server -octez-sc-rollup-client-alpha -octez-sc-rollup-node-alpha -octez-signer -octez-snoop -octez-tx-rollup-client-PtKathma -octez-tx-rollup-client-PtLimaPt -octez-tx-rollup-client-alpha -octez-tx-rollup-node-PtKathma -octez-tx-rollup-node-PtLimaPt -octez-tx-rollup-node-alpha -octez-wasm-repl -tezos-version diff --git a/scripts/build_and_install_static_binaries.sh b/scripts/build_and_install_static_binaries.sh deleted file mode 100755 index 82fa483d460b9c6cdd6288afda00e6e5c9dffdcb..0000000000000000000000000000000000000000 --- a/scripts/build_and_install_static_binaries.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env sh - -# Used in the CI to build and install the static binaries. - -set -eu - -if [ $# -ne 1 ]; then - echo "usage: $0 DESTDIR" - exit 1 -fi - -tmp_dir=$(mktemp -dt tezos_static_install.XXXXXXXX) -cleanup () { - set +e - echo Cleaning up... - rm -rf "$tmp_dir" -} -trap cleanup EXIT INT - -# shellcheck disable=SC2046 -dune build --profile static $(xargs -I {} echo {}.install < script-inputs/static-packages) -# shellcheck disable=SC2046 -dune install --profile static --prefix "$tmp_dir" $(cat script-inputs/static-packages) - -mv "$tmp_dir/bin" "$1" - -# Make sure binaries are statically linked -find -L "$1" -type f -not -name "*.sh" | -while read -r b; do - file "$(realpath "$b")" | grep "statically linked"; -done - -# Make sure octez-client knows about build-infos -SHA=$(git rev-parse --short=8 HEAD) -client_version=$("$1/octez-client" --version | cut -f 1 -d ' ') -if [ "$SHA" != "$client_version" ]; then - echo "Unexpected version for octez-client (expected $SHA, found $client_version)" - exit 1 -fi -echo "octez-client --version returned the expected commit hash: $SHA" - -# Trim the dune cache, if used. -dune cache trim --size=250GB diff --git a/scripts/ci/build_static_binaries.sh b/scripts/ci/build_static_binaries.sh index 34b5593480b8b4883f3e9c18fbc3209e2fa4ea38..6a235a18afb830af09a878da405094d95f14c437 100755 --- a/scripts/ci/build_static_binaries.sh +++ b/scripts/ci/build_static_binaries.sh @@ -1,11 +1,65 @@ #!/bin/sh -set -eu +set -e + +if [ -z "$EXECUTABLE_FILES" ]; then + echo "Error: environment variable EXECUTABLE_FILES is empty." + echo "Set it to e.g. 'script-inputs/released-executables'" + echo "or to 'script-inputs/released-executables script-inputs/experimental-executables'." + exit 1 +fi + +if [ -z "$ARCH" ]; then + echo "Error: environment variable ARCH is empty." + echo "Set it to e.g. 'x86_64' or 'amd64'." + exit 1 +fi + +set -u echo "Create destination directory" -mkdir -pv octez-binaries +mkdir -pv "octez-binaries/$ARCH" echo "Build and install static binaries" -./scripts/build_and_install_static_binaries.sh "octez-binaries/${ARCH}" +make static + +# Trim the dune cache, if used. +dune cache trim --size=250GB + +echo "Check executables and move them to the destination directory" +# Disable https://www.shellcheck.net/wiki/SC2086 because: +# - it's meant to prevent word splitting and we actually want to split words +# in $EXECUTABLE_FILES and we can't use an array because the variable +# comes from the CI +# - it's meant to prevent globbing but actually we don't mind globbing here +# even if we don't use it right now +# shellcheck disable=SC2086 +cat $EXECUTABLE_FILES | +while read -r executable; do + if [ ! -f "$executable" ]; then + echo "Error: $executable does not exist" + exit 1 + fi + + if (file "$executable" | grep -q "statically linked"); then + echo "Statically linked: $executable" + else + echo "Error: $executable is not statically linked" + exit 1 + fi + + mv "$executable" "octez-binaries/$ARCH/$executable" + # Write access is needed by strip below. + chmod +w "octez-binaries/$ARCH/$executable" +done + +echo 'Check octez-client --version' +SHA=$(git rev-parse --short=8 HEAD) +client_version=$("octez-binaries/$ARCH/octez-client" --version | cut -f 1 -d ' ') +if [ "$SHA" != "$client_version" ]; then + echo "Unexpected version for octez-client (expected $SHA, found $client_version)" + exit 1 +fi +echo "octez-client --version returned the expected commit hash: $SHA" echo "Strip debug symbols and compress binaries (parallelized)" # shellcheck disable=SC2046,SC2038 diff --git a/scripts/ci/release.sh b/scripts/ci/release.sh index 1b33f2d8cc7e747fb6ac3019395d372b2501d2e5..ed049a8101be7927b68f1d9ba047dad9b76e04ea 100755 --- a/scripts/ci/release.sh +++ b/scripts/ci/release.sh @@ -10,7 +10,7 @@ scripts_dir=$(dirname "$current_dir") src_dir=$(dirname "$scripts_dir") script_inputs_dir="$src_dir/script-inputs" -binaries="$(cat "$script_inputs_dir/binaries-for-release")" +binaries="$(cat "$script_inputs_dir/released-executables")" ### Compute GitLab release names