diff --git a/.gitlab/ci/jobs/build/x86_64.yml b/.gitlab/ci/jobs/build/x86_64.yml index 70918fd9280d117cd13dcce68f84ea8486d552ce..e11e19d5b7b2a1a948d96e57ef8dc3ca11db9da1 100644 --- a/.gitlab/ci/jobs/build/x86_64.yml +++ b/.gitlab/ci/jobs/build/x86_64.yml @@ -1,10 +1,27 @@ include: .gitlab/ci/jobs/build/common.yml -build_x86_64: +.build_x86_64: extends: - .build - # This job is the main bottleneck since a lot of tests depend on it. - # So we start it even before sanity_ci. + # These job are the main bottleneck since a lot of tests depend on + # them. So we start them even before sanity_ci. needs: [trigger] variables: ARCH: "x86_64" + +# The build_x86_64 jobs are split in two to keep the artifact size +# under the 1GB hard limit set by GitLab. +# 'build_x86_64-released' builds the released executables. +build_x86_64-released: + extends: .build_x86_64 + variables: + EXECUTABLE_FILES: "script-inputs/released-executables" + +# 'build_x86_64-exp-dev-extra' builds the developer and experimental +# executables, as well as the tezt test suite used by the subsequent +# 'tezt' jobs and TPS evaluation tool. +build_x86_64-exp-dev-extra: + extends: .build_x86_64 + variables: + EXECUTABLE_FILES: "script-inputs/experimental-executables script-inputs/dev-executables" + BUILD_EXTRA: "src/bin_tps_evaluation/main_tps_evaluation.exe tezt/tests/main.exe" diff --git a/.gitlab/ci/jobs/test/check_scripts.yml b/.gitlab/ci/jobs/test/check_scripts.yml index c65b4f050531889391bf8996a827f4f995021d34..90c769016cb76d425c590f323944bc21131ac423 100644 --- a/.gitlab/ci/jobs/test/check_scripts.yml +++ b/.gitlab/ci/jobs/test/check_scripts.yml @@ -11,7 +11,9 @@ script:snapshot_alpha_and_link: stage: test before_script: - . ./scripts/version.sh - needs: ["build_x86_64"] + needs: + - "build_x86_64-released" + - "build_x86_64-exp-dev-extra" script: - make -C ${CI_PROJECT_DIR}/.gitlab/ci integration-test-snapshot-alpha-and-link rules: diff --git a/.gitlab/ci/jobs/test/common.yml b/.gitlab/ci/jobs/test/common.yml index b79fdf13aa3f6002a720e58a4b994113fc8da1a9..8a6e9590f4e20240acd9e945e119e054df50f0fa 100644 --- a/.gitlab/ci/jobs/test/common.yml +++ b/.gitlab/ci/jobs/test/common.yml @@ -34,8 +34,15 @@ - .test_template - .image_template__runtime_build_e2etest_dependencies_template - .template__coverage_files - dependencies: ["build_x86_64", "tezt:fetch-records"] + dependencies: + - "build_x86_64-released" + - "build_x86_64-exp-dev-extra" + - "tezt:fetch-records" # Start immediately after 'build_x86_64' and don't wait for # intermediate stages to succeed - needs: ["build_x86_64", "tezt:fetch-records"] + needs: + - "build_x86_64-released" + - "build_x86_64-exp-dev-extra" + - "tezt:fetch-records" + diff --git a/.gitlab/ci/jobs/test/tezt.yml b/.gitlab/ci/jobs/test/tezt.yml index 3b6e8bcbef82186b8e6cc7f8144f5ead3d2701c6..7b54fa873ca74b6c8430f39476445bfb6dd8ba08 100644 --- a/.gitlab/ci/jobs/test/tezt.yml +++ b/.gitlab/ci/jobs/test/tezt.yml @@ -43,11 +43,13 @@ tezt: # the individual test for the disabling rationale. TESTS: "/ci_disable" dependencies: - - "build_x86_64" + - "build_x86_64-released" + - "build_x86_64-exp-dev-extra" - "build_kernels" - "tezt:fetch-records" needs: - - "build_x86_64" + - "build_x86_64-released" + - "build_x86_64-exp-dev-extra" - "build_kernels" - "tezt:fetch-records" # WARNING: if you increase the number of parallel jobs, you need to @@ -62,13 +64,13 @@ tezt:static-binaries: - .image_template__runtime_build_test_dependencies_template - .tezt_template dependencies: - # Fetch src/proto_*/parameters/*.json from build_x86_64 - - build_x86_64 + # Fetch src/proto_*/parameters/*.json from build_x86_64-released + - build_x86_64-released # And fetch the static executables from build:static - build:static-x86_64-linux-binaries - tezt:fetch-records needs: - - build_x86_64 + - build_x86_64-released - build:static-x86_64-linux-binaries - tezt:fetch-records # WARNING: if you increase the number of parallel jobs, you need to diff --git a/.gitlab/ci/jobs/test/unit.yml b/.gitlab/ci/jobs/test/unit.yml index 69b68667ae54a5a829e34697242bd4fd4726275f..b0d023aa8f086b1545085cedd7f3318c16e6bfc8 100644 --- a/.gitlab/ci/jobs/test/unit.yml +++ b/.gitlab/ci/jobs/test/unit.yml @@ -16,7 +16,9 @@ .unit_test_template_x86_64: extends: .unit_test_template - needs: ["build_x86_64"] + needs: + - "build_x86_64-released" + - "build_x86_64-exp-dev-extra" variables: ARCH: "x86_64" diff --git a/Makefile b/Makefile index 23cc7b3ba4e7961aa893349ed05ff532434d245b..3ebeff1fdfb530290caf3a825ba6e2126ce66d25 100644 --- a/Makefile +++ b/Makefile @@ -38,13 +38,15 @@ EXPERIMENTAL_EXECUTABLES := $(shell cat script-inputs/experimental-executables) # are not useful for users. # - scripts/snapshot_alpha.sh expects octez-protocol-compiler to be at the root. # - Some tests expect octez-snoop to be at the root. -DEV_EXECUTABLES := octez-protocol-compiler octez-snoop +DEV_EXECUTABLES := $(shell cat script-inputs/dev-executables) ALL_EXECUTABLES := $(RELEASED_EXECUTABLES) $(EXPERIMENTAL_EXECUTABLES) $(DEV_EXECUTABLES) # Set of Dune targets to build, in addition to OCTEZ_EXECUTABLES, in -# the `build` target's Dune invocation. -BUILD_EXTRA := +# the `build` target's Dune invocation. This is used in the CI to +# build the TPS evaluation tool and the Tezt test suite in the +# 'build_x86_64-dev-exp-misc' job. +BUILD_EXTRA ?= # See first mention of TEZOS_WITHOUT_OPAM. ifndef TEZOS_WITHOUT_OPAM @@ -91,8 +93,9 @@ endif # This is more efficient than 'make octez-node octez-client' # because it only calls 'dune' once. # -# Targets 'all', 'release', 'experimental-release', 'static' and 'all-extras' define different -# default lists of executables to build but they all can be overridden from the command-line. +# Targets 'all', 'release', 'experimental-release' and 'static' define +# different default lists of executables to build but they all can be +# overridden from the command-line. .PHONY: all all: @$(MAKE) build OCTEZ_EXECUTABLES?="$(ALL_EXECUTABLES)" @@ -105,13 +108,6 @@ release: experimental-release: @$(MAKE) build PROFILE=release OCTEZ_EXECUTABLES?="$(RELEASED_EXECUTABLES) $(EXPERIMENTAL_EXECUTABLES)" -# all-extras is targetted by the CI build jobs 'x86_64' and 'arm64', -# and builds test executables in addition to the default set of -# executables. -.PHONY: all-extras -all-extras: - @$(MAKE) all BUILD_EXTRA="src/bin_tps_evaluation tezt/tests/main.exe" - .PHONY: strip strip: all @chmod +w $(ALL_EXECUTABLES) diff --git a/script-inputs/dev-executables b/script-inputs/dev-executables new file mode 100644 index 0000000000000000000000000000000000000000..809b24b55955a758341589d474870425e00e5940 --- /dev/null +++ b/script-inputs/dev-executables @@ -0,0 +1,2 @@ +octez-protocol-compiler +octez-snoop diff --git a/scripts/ci/build_full_unreleased.sh b/scripts/ci/build_full_unreleased.sh index c59062c5dccc8f6c809f7fc8cfbc531864f15a59..7faba65bc33835a0f16b9be379a364990749ab86 100755 --- a/scripts/ci/build_full_unreleased.sh +++ b/scripts/ci/build_full_unreleased.sh @@ -1,4 +1,5 @@ #!/bin/sh + set -eu if [ -z "${build_deps_image_name}" ]; then echo "build_deps_image_name is unset" && exit 3; fi @@ -20,15 +21,27 @@ diff poetry.lock /home/tezos/poetry.lock diff pyproject.toml /home/tezos/pyproject.toml # 2. Actually build. -# Build the 'all-extras' target instead of the default, which also -# build the tps evaluation tool which is not part of the default -# build, and the tezt main entrypoint for integration tests. # NOTE: This makes one call to `dune build`, as calling `dune` # several time would otherwise need to reconstruct its rules. Do # not split this invocation. # NOTE: This ensure that $COVERAGE_OPTIONS is used consistently. -make all-extras -# 3. Strip the built binaries -chmod +w octez-* _build/default/tezt/tests/main.exe _build/default/src/lib_protocol_compiler/bin/main_native.exe -strip -s octez-* _build/default/tezt/tests/main.exe _build/default/src/lib_protocol_compiler/bin/main_native.exe +# EXECUTABLE_FILES may contain multiple paths and so must be split. +# shellcheck disable=SC2086 +OCTEZ_EXECUTABLES="$(cat $EXECUTABLE_FILES)" +make build OCTEZ_EXECUTABLES="$OCTEZ_EXECUTABLES" + +# 3. Strip the built binaries in OCTEZ_EXECUTABLES and in BUILD_EXTRA. +# shellcheck disable=SC2086 +chmod +w ${OCTEZ_EXECUTABLES} +# shellcheck disable=SC2086 +strip -s ${OCTEZ_EXECUTABLES} + +if [ -n "${BUILD_EXTRA:-}" ]; then + paths=$(for executable in ${BUILD_EXTRA}; do echo _build/default/"$executable"; done) + # Paths must be split so disable shellcheck's SC2086 here. + # shellcheck disable=SC2086 + chmod +w $paths + # shellcheck disable=SC2086 + strip -s $paths +fi