From 6559ddc0f43516578b303fad3ff61f1f3be62408 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Thu, 15 Jun 2023 12:06:20 +0200 Subject: [PATCH] CI: split [build_x86_64] into released / un-released --- .gitlab/ci/jobs/build/x86_64.yml | 23 +++++++++++++++++++--- .gitlab/ci/jobs/test/check_scripts.yml | 4 +++- .gitlab/ci/jobs/test/common.yml | 11 +++++++++-- .gitlab/ci/jobs/test/tezt.yml | 12 +++++++----- .gitlab/ci/jobs/test/unit.yml | 4 +++- Makefile | 20 ++++++++----------- script-inputs/dev-executables | 2 ++ scripts/ci/build_full_unreleased.sh | 27 +++++++++++++++++++------- 8 files changed, 72 insertions(+), 31 deletions(-) create mode 100644 script-inputs/dev-executables diff --git a/.gitlab/ci/jobs/build/x86_64.yml b/.gitlab/ci/jobs/build/x86_64.yml index 70918fd9280d..e11e19d5b7b2 100644 --- a/.gitlab/ci/jobs/build/x86_64.yml +++ b/.gitlab/ci/jobs/build/x86_64.yml @@ -1,10 +1,27 @@ include: .gitlab/ci/jobs/build/common.yml -build_x86_64: +.build_x86_64: extends: - .build - # This job is the main bottleneck since a lot of tests depend on it. - # So we start it even before sanity_ci. + # These job are the main bottleneck since a lot of tests depend on + # them. So we start them even before sanity_ci. needs: [trigger] variables: ARCH: "x86_64" + +# The build_x86_64 jobs are split in two to keep the artifact size +# under the 1GB hard limit set by GitLab. +# 'build_x86_64-released' builds the released executables. +build_x86_64-released: + extends: .build_x86_64 + variables: + EXECUTABLE_FILES: "script-inputs/released-executables" + +# 'build_x86_64-exp-dev-extra' builds the developer and experimental +# executables, as well as the tezt test suite used by the subsequent +# 'tezt' jobs and TPS evaluation tool. +build_x86_64-exp-dev-extra: + extends: .build_x86_64 + variables: + EXECUTABLE_FILES: "script-inputs/experimental-executables script-inputs/dev-executables" + BUILD_EXTRA: "src/bin_tps_evaluation/main_tps_evaluation.exe tezt/tests/main.exe" diff --git a/.gitlab/ci/jobs/test/check_scripts.yml b/.gitlab/ci/jobs/test/check_scripts.yml index c65b4f050531..90c769016cb7 100644 --- a/.gitlab/ci/jobs/test/check_scripts.yml +++ b/.gitlab/ci/jobs/test/check_scripts.yml @@ -11,7 +11,9 @@ script:snapshot_alpha_and_link: stage: test before_script: - . ./scripts/version.sh - needs: ["build_x86_64"] + needs: + - "build_x86_64-released" + - "build_x86_64-exp-dev-extra" script: - make -C ${CI_PROJECT_DIR}/.gitlab/ci integration-test-snapshot-alpha-and-link rules: diff --git a/.gitlab/ci/jobs/test/common.yml b/.gitlab/ci/jobs/test/common.yml index b79fdf13aa3f..8a6e9590f4e2 100644 --- a/.gitlab/ci/jobs/test/common.yml +++ b/.gitlab/ci/jobs/test/common.yml @@ -34,8 +34,15 @@ - .test_template - .image_template__runtime_build_e2etest_dependencies_template - .template__coverage_files - dependencies: ["build_x86_64", "tezt:fetch-records"] + dependencies: + - "build_x86_64-released" + - "build_x86_64-exp-dev-extra" + - "tezt:fetch-records" # Start immediately after 'build_x86_64' and don't wait for # intermediate stages to succeed - needs: ["build_x86_64", "tezt:fetch-records"] + needs: + - "build_x86_64-released" + - "build_x86_64-exp-dev-extra" + - "tezt:fetch-records" + diff --git a/.gitlab/ci/jobs/test/tezt.yml b/.gitlab/ci/jobs/test/tezt.yml index 3b6e8bcbef82..7b54fa873ca7 100644 --- a/.gitlab/ci/jobs/test/tezt.yml +++ b/.gitlab/ci/jobs/test/tezt.yml @@ -43,11 +43,13 @@ tezt: # the individual test for the disabling rationale. TESTS: "/ci_disable" dependencies: - - "build_x86_64" + - "build_x86_64-released" + - "build_x86_64-exp-dev-extra" - "build_kernels" - "tezt:fetch-records" needs: - - "build_x86_64" + - "build_x86_64-released" + - "build_x86_64-exp-dev-extra" - "build_kernels" - "tezt:fetch-records" # WARNING: if you increase the number of parallel jobs, you need to @@ -62,13 +64,13 @@ tezt:static-binaries: - .image_template__runtime_build_test_dependencies_template - .tezt_template dependencies: - # Fetch src/proto_*/parameters/*.json from build_x86_64 - - build_x86_64 + # Fetch src/proto_*/parameters/*.json from build_x86_64-released + - build_x86_64-released # And fetch the static executables from build:static - build:static-x86_64-linux-binaries - tezt:fetch-records needs: - - build_x86_64 + - build_x86_64-released - build:static-x86_64-linux-binaries - tezt:fetch-records # WARNING: if you increase the number of parallel jobs, you need to diff --git a/.gitlab/ci/jobs/test/unit.yml b/.gitlab/ci/jobs/test/unit.yml index 69b68667ae54..b0d023aa8f08 100644 --- a/.gitlab/ci/jobs/test/unit.yml +++ b/.gitlab/ci/jobs/test/unit.yml @@ -16,7 +16,9 @@ .unit_test_template_x86_64: extends: .unit_test_template - needs: ["build_x86_64"] + needs: + - "build_x86_64-released" + - "build_x86_64-exp-dev-extra" variables: ARCH: "x86_64" diff --git a/Makefile b/Makefile index 23cc7b3ba4e7..3ebeff1fdfb5 100644 --- a/Makefile +++ b/Makefile @@ -38,13 +38,15 @@ EXPERIMENTAL_EXECUTABLES := $(shell cat script-inputs/experimental-executables) # are not useful for users. # - scripts/snapshot_alpha.sh expects octez-protocol-compiler to be at the root. # - Some tests expect octez-snoop to be at the root. -DEV_EXECUTABLES := octez-protocol-compiler octez-snoop +DEV_EXECUTABLES := $(shell cat script-inputs/dev-executables) ALL_EXECUTABLES := $(RELEASED_EXECUTABLES) $(EXPERIMENTAL_EXECUTABLES) $(DEV_EXECUTABLES) # Set of Dune targets to build, in addition to OCTEZ_EXECUTABLES, in -# the `build` target's Dune invocation. -BUILD_EXTRA := +# the `build` target's Dune invocation. This is used in the CI to +# build the TPS evaluation tool and the Tezt test suite in the +# 'build_x86_64-dev-exp-misc' job. +BUILD_EXTRA ?= # See first mention of TEZOS_WITHOUT_OPAM. ifndef TEZOS_WITHOUT_OPAM @@ -91,8 +93,9 @@ endif # This is more efficient than 'make octez-node octez-client' # because it only calls 'dune' once. # -# Targets 'all', 'release', 'experimental-release', 'static' and 'all-extras' define different -# default lists of executables to build but they all can be overridden from the command-line. +# Targets 'all', 'release', 'experimental-release' and 'static' define +# different default lists of executables to build but they all can be +# overridden from the command-line. .PHONY: all all: @$(MAKE) build OCTEZ_EXECUTABLES?="$(ALL_EXECUTABLES)" @@ -105,13 +108,6 @@ release: experimental-release: @$(MAKE) build PROFILE=release OCTEZ_EXECUTABLES?="$(RELEASED_EXECUTABLES) $(EXPERIMENTAL_EXECUTABLES)" -# all-extras is targetted by the CI build jobs 'x86_64' and 'arm64', -# and builds test executables in addition to the default set of -# executables. -.PHONY: all-extras -all-extras: - @$(MAKE) all BUILD_EXTRA="src/bin_tps_evaluation tezt/tests/main.exe" - .PHONY: strip strip: all @chmod +w $(ALL_EXECUTABLES) diff --git a/script-inputs/dev-executables b/script-inputs/dev-executables new file mode 100644 index 000000000000..809b24b55955 --- /dev/null +++ b/script-inputs/dev-executables @@ -0,0 +1,2 @@ +octez-protocol-compiler +octez-snoop diff --git a/scripts/ci/build_full_unreleased.sh b/scripts/ci/build_full_unreleased.sh index c59062c5dccc..7faba65bc338 100755 --- a/scripts/ci/build_full_unreleased.sh +++ b/scripts/ci/build_full_unreleased.sh @@ -1,4 +1,5 @@ #!/bin/sh + set -eu if [ -z "${build_deps_image_name}" ]; then echo "build_deps_image_name is unset" && exit 3; fi @@ -20,15 +21,27 @@ diff poetry.lock /home/tezos/poetry.lock diff pyproject.toml /home/tezos/pyproject.toml # 2. Actually build. -# Build the 'all-extras' target instead of the default, which also -# build the tps evaluation tool which is not part of the default -# build, and the tezt main entrypoint for integration tests. # NOTE: This makes one call to `dune build`, as calling `dune` # several time would otherwise need to reconstruct its rules. Do # not split this invocation. # NOTE: This ensure that $COVERAGE_OPTIONS is used consistently. -make all-extras -# 3. Strip the built binaries -chmod +w octez-* _build/default/tezt/tests/main.exe _build/default/src/lib_protocol_compiler/bin/main_native.exe -strip -s octez-* _build/default/tezt/tests/main.exe _build/default/src/lib_protocol_compiler/bin/main_native.exe +# EXECUTABLE_FILES may contain multiple paths and so must be split. +# shellcheck disable=SC2086 +OCTEZ_EXECUTABLES="$(cat $EXECUTABLE_FILES)" +make build OCTEZ_EXECUTABLES="$OCTEZ_EXECUTABLES" + +# 3. Strip the built binaries in OCTEZ_EXECUTABLES and in BUILD_EXTRA. +# shellcheck disable=SC2086 +chmod +w ${OCTEZ_EXECUTABLES} +# shellcheck disable=SC2086 +strip -s ${OCTEZ_EXECUTABLES} + +if [ -n "${BUILD_EXTRA:-}" ]; then + paths=$(for executable in ${BUILD_EXTRA}; do echo _build/default/"$executable"; done) + # Paths must be split so disable shellcheck's SC2086 here. + # shellcheck disable=SC2086 + chmod +w $paths + # shellcheck disable=SC2086 + strip -s $paths +fi -- GitLab