From 8088230d7d3ad20f20b50c29eb04841fe20c146c Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 15 Mar 2024 13:37:52 +0100 Subject: [PATCH 01/11] CI: remove unused [bin_packages_common.yml] --- .gitlab/ci/jobs/build/bin_packages_common.yml | 47 ------------------- ci/bin/main.ml | 1 - 2 files changed, 48 deletions(-) delete mode 100644 .gitlab/ci/jobs/build/bin_packages_common.yml diff --git a/.gitlab/ci/jobs/build/bin_packages_common.yml b/.gitlab/ci/jobs/build/bin_packages_common.yml deleted file mode 100644 index 4a8d6e91765e..000000000000 --- a/.gitlab/ci/jobs/build/bin_packages_common.yml +++ /dev/null @@ -1,47 +0,0 @@ -.bin_packages_common: - extends: - - .default_settings_template - needs: [] - variables: - TARGET: "" - OCTEZ_PKGMAINTAINER: "nomadic-labs" - BLST_PORTABLE: "yes" - ARCH: "amd64" - script: - - wget https://sh.rustup.rs/rustup-init.sh - - chmod +x rustup-init.sh - - ./rustup-init.sh --profile minimal --default-toolchain $recommended_rust_version -y - - . $HOME/.cargo/env - - export OPAMYES="true" - - opam init --bare --disable-sandboxing - - make build-deps - - eval $(opam env) - - make $TARGET - artifacts: - name: "${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG" - expire_in: 1 day - when: on_success - -.bin_packages_dpkg: - image: debian:bookworm - variables: - TARGET: "dpkg" - before_script: - - . ./scripts/version.sh - - apt update - - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev protobuf-compiler libsqlite3-dev jq - artifacts: - paths: - - octez-*.deb - -.bin_packages_rpm: - image: fedora:39 - variables: - TARGET: "rpm" - before_script: - - . ./scripts/version.sh - - dnf update -y - - dnf install -y libev-devel gmp-devel hidapi-devel libffi-devel zlib-devel libpq-devel m4 perl git pkg-config rpmdevtools python3-devel python3-setuptools wget opam rsync which cargo autoconf mock systemd systemd-rpm-macros cmake python3-wheel python3-tox-current-env gcc-c++ protobuf-compiler protobuf-devel sqlite-devel jq - artifacts: - paths: - - octez-*.rpm diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 622c3c2b8661..f581db136d35 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -174,7 +174,6 @@ let () = (* Paths to exclude from generation check. As files are translated to CI-in-OCaml, they should be removed from this function *) let exclude = function - | ".gitlab/ci/jobs/build/bin_packages_common.yml" | ".gitlab/ci/jobs/build/common.yml" | ".gitlab/ci/jobs/build/oc.build_kernels.yml" | ".gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra.yml" -- GitLab From 248385896a381a8ac15086cdc4c2ac2ee8983f6c Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 15 Mar 2024 12:58:25 +0100 Subject: [PATCH 02/11] CI: remove unused templates --- .gitlab/ci/jobs/shared/templates.yml | 43 ---------------------------- 1 file changed, 43 deletions(-) diff --git a/.gitlab/ci/jobs/shared/templates.yml b/.gitlab/ci/jobs/shared/templates.yml index 953778e955d4..8c99aba097a9 100644 --- a/.gitlab/ci/jobs/shared/templates.yml +++ b/.gitlab/ci/jobs/shared/templates.yml @@ -89,25 +89,6 @@ - tzt_reference_test_suite/**/* when: on_success -# As .rules__octez_changes_or_kernels, but allows a manual trigger. -.rules__octez_changes_or_kernels_or_manual: - rules: - - changes: - - images/**/* - - scripts/ci/**/* - - kernels.mk - - etherlink.mk - - src/**/* - - etherlink/**/* - - tezt/**/* - - .gitlab/**/* - - .gitlab-ci.yml - - michelson_test_scripts/**/* - - tzt_reference_test_suite/**/* - when: on_success - - when: manual - allow_failure: true - # Only if octez source code has changed and Marge Bot is not the # trigger .rules__octez_changes_and_not_margebot: @@ -170,30 +151,6 @@ - .gitlab-ci.yml when: manual -# Only if the dependencies of the Octez Docker changed or if we're on -# master. -.rules__octez_docker_changes_or_master: - rules: - - if: '$CI_COMMIT_BRANCH == "master"' - when: always - - changes: - - scripts/**/* - - script-inputs/**/* - - src/**/* - - tezt/**/* - - vendors/**/* - - dune - - dune-project - - dune-workspace - - opam - - Makefile - - kernels.mk - - build.Dockerfile - - Dockerfile - - .gitlab/**/* - - .gitlab-ci.yml - when: on_success - .rules__octez_kaitai_e2e_changes: rules: - changes: -- GitLab From 5d768ac3057b70dac55d0bc4e282964312101a1b Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 15 Mar 2024 13:09:43 +0100 Subject: [PATCH 03/11] CIAO: refactor setting of [CI_DOCKER_HUB] --- .../build/oc.docker:amd64-test_manual.yml | 2 +- .../build/oc.docker:arm64-test_manual.yml | 2 +- .gitlab/ci/pipelines/master_branch.yml | 4 +- .gitlab/ci/pipelines/non_release_tag.yml | 4 +- .gitlab/ci/pipelines/non_release_tag_test.yml | 4 +- .../ci/pipelines/octez_beta_release_tag.yml | 4 +- .gitlab/ci/pipelines/octez_release_tag.yml | 4 +- .../ci/pipelines/octez_release_tag_test.yml | 4 +- ci/bin/common.ml | 49 +++++++++++-------- ci/bin/octez_latest_release.ml | 2 +- 10 files changed, 44 insertions(+), 35 deletions(-) diff --git a/.gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml b/.gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml index 132ed2277d53..5746c222cfbc 100644 --- a/.gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml +++ b/.gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml @@ -19,8 +19,8 @@ oc.docker:amd64: - docker:${DOCKER_VERSION}-dind variables: DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" DOCKER_BUILD_TARGET: with-evm-artifacts IMAGE_ARCH_PREFIX: amd64_ - CI_DOCKER_HUB: "false" EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables when: manual diff --git a/.gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml b/.gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml index 4cf3b1e2f151..e1f61c622a23 100644 --- a/.gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml +++ b/.gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml @@ -19,8 +19,8 @@ oc.docker:arm64: - docker:${DOCKER_VERSION}-dind variables: DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" DOCKER_BUILD_TARGET: without-evm-artifacts IMAGE_ARCH_PREFIX: arm64_ - CI_DOCKER_HUB: "false" EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables when: manual diff --git a/.gitlab/ci/pipelines/master_branch.yml b/.gitlab/ci/pipelines/master_branch.yml index b6551a0503bf..0c98288bcc3b 100644 --- a/.gitlab/ci/pipelines/master_branch.yml +++ b/.gitlab/ci/pipelines/master_branch.yml @@ -158,9 +158,9 @@ oc.docker:amd64: - docker:${DOCKER_VERSION}-dind variables: DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "true" DOCKER_BUILD_TARGET: with-evm-artifacts IMAGE_ARCH_PREFIX: amd64_ - CI_DOCKER_HUB: "true" EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables oc.docker:arm64: @@ -200,9 +200,9 @@ oc.docker:arm64: - docker:${DOCKER_VERSION}-dind variables: DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "true" DOCKER_BUILD_TARGET: without-evm-artifacts IMAGE_ARCH_PREFIX: arm64_ - CI_DOCKER_HUB: "true" EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables oc.unified_coverage: diff --git a/.gitlab/ci/pipelines/non_release_tag.yml b/.gitlab/ci/pipelines/non_release_tag.yml index 455d509283d8..9fdcf1cc4e53 100644 --- a/.gitlab/ci/pipelines/non_release_tag.yml +++ b/.gitlab/ci/pipelines/non_release_tag.yml @@ -74,9 +74,9 @@ oc.docker:amd64: - docker:${DOCKER_VERSION}-dind variables: DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "true" DOCKER_BUILD_TARGET: without-evm-artifacts IMAGE_ARCH_PREFIX: amd64_ - CI_DOCKER_HUB: "true" EXECUTABLE_FILES: script-inputs/released-executables oc.docker:arm64: @@ -96,9 +96,9 @@ oc.docker:arm64: - docker:${DOCKER_VERSION}-dind variables: DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "true" DOCKER_BUILD_TARGET: without-evm-artifacts IMAGE_ARCH_PREFIX: arm64_ - CI_DOCKER_HUB: "true" EXECUTABLE_FILES: script-inputs/released-executables oc.build:dpkg:amd64: diff --git a/.gitlab/ci/pipelines/non_release_tag_test.yml b/.gitlab/ci/pipelines/non_release_tag_test.yml index d2f5712f20cc..2768993a527d 100644 --- a/.gitlab/ci/pipelines/non_release_tag_test.yml +++ b/.gitlab/ci/pipelines/non_release_tag_test.yml @@ -74,9 +74,9 @@ oc.docker:amd64: - docker:${DOCKER_VERSION}-dind variables: DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" DOCKER_BUILD_TARGET: without-evm-artifacts IMAGE_ARCH_PREFIX: amd64_ - CI_DOCKER_HUB: "false" EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables oc.docker:arm64: @@ -96,9 +96,9 @@ oc.docker:arm64: - docker:${DOCKER_VERSION}-dind variables: DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" DOCKER_BUILD_TARGET: without-evm-artifacts IMAGE_ARCH_PREFIX: arm64_ - CI_DOCKER_HUB: "false" EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables oc.build:dpkg:amd64: diff --git a/.gitlab/ci/pipelines/octez_beta_release_tag.yml b/.gitlab/ci/pipelines/octez_beta_release_tag.yml index 693ed5a322a2..adb9355489c2 100644 --- a/.gitlab/ci/pipelines/octez_beta_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_beta_release_tag.yml @@ -74,9 +74,9 @@ oc.docker:amd64: - docker:${DOCKER_VERSION}-dind variables: DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "true" DOCKER_BUILD_TARGET: without-evm-artifacts IMAGE_ARCH_PREFIX: amd64_ - CI_DOCKER_HUB: "true" EXECUTABLE_FILES: script-inputs/released-executables oc.docker:arm64: @@ -96,9 +96,9 @@ oc.docker:arm64: - docker:${DOCKER_VERSION}-dind variables: DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "true" DOCKER_BUILD_TARGET: without-evm-artifacts IMAGE_ARCH_PREFIX: arm64_ - CI_DOCKER_HUB: "true" EXECUTABLE_FILES: script-inputs/released-executables oc.build:dpkg:amd64: diff --git a/.gitlab/ci/pipelines/octez_release_tag.yml b/.gitlab/ci/pipelines/octez_release_tag.yml index 7aacdea83d7e..10fa9aafe078 100644 --- a/.gitlab/ci/pipelines/octez_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_release_tag.yml @@ -74,9 +74,9 @@ oc.docker:amd64: - docker:${DOCKER_VERSION}-dind variables: DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "true" DOCKER_BUILD_TARGET: without-evm-artifacts IMAGE_ARCH_PREFIX: amd64_ - CI_DOCKER_HUB: "true" EXECUTABLE_FILES: script-inputs/released-executables oc.docker:arm64: @@ -96,9 +96,9 @@ oc.docker:arm64: - docker:${DOCKER_VERSION}-dind variables: DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "true" DOCKER_BUILD_TARGET: without-evm-artifacts IMAGE_ARCH_PREFIX: arm64_ - CI_DOCKER_HUB: "true" EXECUTABLE_FILES: script-inputs/released-executables oc.build:dpkg:amd64: diff --git a/.gitlab/ci/pipelines/octez_release_tag_test.yml b/.gitlab/ci/pipelines/octez_release_tag_test.yml index fd100658ac88..94b44a30a39b 100644 --- a/.gitlab/ci/pipelines/octez_release_tag_test.yml +++ b/.gitlab/ci/pipelines/octez_release_tag_test.yml @@ -74,9 +74,9 @@ oc.docker:amd64: - docker:${DOCKER_VERSION}-dind variables: DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" DOCKER_BUILD_TARGET: without-evm-artifacts IMAGE_ARCH_PREFIX: amd64_ - CI_DOCKER_HUB: "false" EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables oc.docker:arm64: @@ -96,9 +96,9 @@ oc.docker:arm64: - docker:${DOCKER_VERSION}-dind variables: DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" DOCKER_BUILD_TARGET: without-evm-artifacts IMAGE_ARCH_PREFIX: arm64_ - CI_DOCKER_HUB: "false" EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables oc.build:dpkg:amd64: diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 5725bcfa2213..0dd7c1ccc3d2 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -174,6 +174,8 @@ let before_script ?(take_ownership = false) ?(source_version = false) @ toggle install_js_deps ". ./scripts/install_build_deps.js.sh" @ before_script +let opt_var name f = function Some value -> [(name, f value)] | None -> [] + (** Add variable for bisect_ppx instrumentation. This template should be extended by jobs that build OCaml targets @@ -219,12 +221,11 @@ let enable_coverage_report job : tezos_job = which has a configured sccache Gitlab CI cache. *) let enable_sccache ?error_log ?idle_timeout ?log ?(dir = "$CI_PROJECT_DIR/_sccache") : tezos_job -> tezos_job = - let opt_var name = function Some value -> [(name, value)] | None -> [] in Tezos_ci.append_variables ([("SCCACHE_DIR", dir); ("RUSTC_WRAPPER", "sccache")] - @ opt_var "SCCACHE_ERROR_LOG" error_log - @ opt_var "SCCACHE_IDLE_TIMEOUT" idle_timeout - @ opt_var "SCCACHE_LOG" log) + @ opt_var "SCCACHE_ERROR_LOG" Fun.id error_log + @ opt_var "SCCACHE_IDLE_TIMEOUT" Fun.id idle_timeout + @ opt_var "SCCACHE_LOG" Fun.id log) (** {2 Changesets} *) @@ -298,15 +299,20 @@ let changeset_opam_jobs = (** {2 Job makers} *) -(** Helper to create jobs that uses the docker deamon. +(** Helper to create jobs that uses the Docker daemon. + + It sets the appropriate image. Furthermore, unless + [skip_docker_initialization] is [true], it: + - activates the Docker daemon as a service; + - sets up authentification with Docker registries + in the job's [before_script] section. - It: - - Sets the appropriate image. - - Activates the Docker daemon as a service. - - It sets up authentification with docker registries *) -let job_docker_authenticated ?(skip_docker_initialization = false) ?artifacts - ?variables ?rules ?dependencies ?arch ?when_ ?allow_failure ~__POS__ ~stage - ~name script : tezos_job = + If [ci_docker_hub] is set to [true], then the job will + authenticate with Docker Hub provided the environment variable + [CI_DOCKER_AUTH] contains the appropriate credentials. *) +let job_docker_authenticated ?(skip_docker_initialization = false) + ?ci_docker_hub ?artifacts ?(variables = []) ?rules ?dependencies ?arch + ?when_ ?allow_failure ~__POS__ ~stage ~name script : tezos_job = let docker_version = "24.0.6" in job ?rules @@ -318,7 +324,9 @@ let job_docker_authenticated ?(skip_docker_initialization = false) ?artifacts ~__POS__ ~image:Images.docker ~variables: - ([("DOCKER_VERSION", docker_version)] @ Option.value ~default:[] variables) + ([("DOCKER_VERSION", docker_version)] + @ opt_var "CI_DOCKER_HUB" Bool.to_string ci_docker_hub + @ variables) ~before_script: (if not skip_docker_initialization then ["./scripts/ci/docker_initialize.sh"] @@ -372,7 +380,7 @@ let job_docker_rust_toolchain ?rules ?dependencies ~__POS__ () = ~skip_docker_initialization:true ~stage:Stages.build ~name:"oc.docker:rust-toolchain" - ~variables:[("CI_DOCKER_HUB", "false")] + ~ci_docker_hub:false ~artifacts: (artifacts ~reports:(reports ~dotenv:"rust_toolchain_image_tag.env" ()) @@ -408,6 +416,11 @@ type docker_build_type = Experimental | Release | Test | Test_manual let job_docker_build ?rules ?dependencies ~__POS__ ~arch ?(external_ = false) docker_build_type : tezos_job = let arch_string = match arch with Amd64 -> "amd64" | Arm64 -> "arm64" in + let ci_docker_hub = + match docker_build_type with + | Release | Experimental -> true + | Test | Test_manual -> false + in let variables = [ ( "DOCKER_BUILD_TARGET", @@ -415,11 +428,6 @@ let job_docker_build ?rules ?dependencies ~__POS__ ~arch ?(external_ = false) | Amd64, (Test_manual | Experimental) -> "with-evm-artifacts" | _ -> "without-evm-artifacts" ); ("IMAGE_ARCH_PREFIX", arch_string ^ "_"); - ( "CI_DOCKER_HUB", - Bool.to_string - (match docker_build_type with - | Release | Experimental -> true - | Test | Test_manual -> false) ); ( "EXECUTABLE_FILES", match docker_build_type with | Release -> "script-inputs/released-executables" @@ -447,6 +455,7 @@ let job_docker_build ?rules ?dependencies ~__POS__ ~arch ?(external_ = false) ?allow_failure ?rules ?dependencies + ~ci_docker_hub ~__POS__ ~stage ~arch @@ -471,7 +480,7 @@ let job_docker_merge_manifests ~__POS__ ~ci_docker_hub ~job_docker_amd64 [docker:{amd64,arm64}] into a single multi-architecture image, and so must be run after these jobs. *) ~dependencies:(Dependent [Job job_docker_amd64; Job job_docker_arm64]) - ~variables:[("CI_DOCKER_HUB", Bool.to_string ci_docker_hub)] + ~ci_docker_hub ["./scripts/ci/docker_merge_manifests.sh"] type bin_package_target = Dpkg | Rpm diff --git a/ci/bin/octez_latest_release.ml b/ci/bin/octez_latest_release.ml index ab0370b55ef1..25e4e1c7ac2b 100644 --- a/ci/bin/octez_latest_release.ml +++ b/ci/bin/octez_latest_release.ml @@ -19,7 +19,7 @@ let job_docker_promote_to_latest ~ci_docker_hub : tezos_job = ~__POS__ ~stage:Stages.publish_release ~name:"docker:promote_to_latest" - ~variables:[("CI_DOCKER_HUB", Bool.to_string ci_docker_hub)] + ~ci_docker_hub ["./scripts/ci/docker_promote_to_latest.sh"] let jobs ?(test = false) () = -- GitLab From 94748724ad24838aab09e25a1267aaccb9582917 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Mon, 11 Mar 2024 15:43:42 +0100 Subject: [PATCH 04/11] CI: refactor, always build [client-libs-dependencies] on scheduled --- .../jobs/build/oc.docker:client-libs-dependencies-other.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-other.yml b/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-other.yml index 74e08a1107f2..0208d86eb264 100644 --- a/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-other.yml +++ b/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-other.yml @@ -2,3 +2,7 @@ include: .gitlab/ci/jobs/build/common.yml oc.docker:client-libs-dependencies: extends: .oc.docker:client-libs-dependencies_template + rules: + # This job runs on the schedule extended pipeline, so we want it + # to execute 'always'. + - when: always -- GitLab From a3a9b070f04f0ce52c88794052052ee974d8da5f Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Mon, 11 Mar 2024 15:44:19 +0100 Subject: [PATCH 05/11] CI: generate [oc.docker:client-libs-dependencies] --- .gitlab/ci/jobs/build/common.yml | 17 ----------- ...lient-libs-dependencies-before_merging.yml | 30 ++++++++++++++++--- ....docker:client-libs-dependencies-other.yml | 24 +++++++++++---- ci/bin/code_verification.ml | 24 ++++++++++++++- ci/bin/common.ml | 9 ++++++ ci/bin/main.ml | 2 -- 6 files changed, 77 insertions(+), 29 deletions(-) diff --git a/.gitlab/ci/jobs/build/common.yml b/.gitlab/ci/jobs/build/common.yml index 98a8b9b274ec..fa2b2a600f43 100644 --- a/.gitlab/ci/jobs/build/common.yml +++ b/.gitlab/ci/jobs/build/common.yml @@ -17,23 +17,6 @@ expire_in: 1 day when: on_success -.oc.docker:client-libs-dependencies_template: - extends: - - .image_template__docker - - .docker_auth_template - stage: build - variables: - # These image are not built for external use. - CI_DOCKER_HUB: "false" - # Handle docker initialization, if necessary, in - # './scripts/ci/docker_client_libs_dependencies_build.sh'. - before_script: [] - script: - - ./scripts/ci/docker_client_libs_dependencies_build.sh - artifacts: - reports: - dotenv: client_libs_dependencies_image_tag.env - .oc.build_x86_64: extends: - .oc.build diff --git a/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-before_merging.yml b/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-before_merging.yml index e38ee2a14fbb..393811f5c2c9 100644 --- a/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-before_merging.yml +++ b/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-before_merging.yml @@ -1,6 +1,28 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.docker:client-libs-dependencies: - extends: - - .oc.docker:client-libs-dependencies_template - - .rules__octez_kaitai_e2e_changes + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + stage: build + tags: + - gcp + rules: + - changes: + - images/**/* + - src/**/* + - contrib/*kaitai*/**/* + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_client_libs_dependencies_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: client_libs_dependencies_image_tag.env diff --git a/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-other.yml b/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-other.yml index 0208d86eb264..8780345daffb 100644 --- a/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-other.yml +++ b/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-other.yml @@ -1,8 +1,22 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.docker:client-libs-dependencies: - extends: .oc.docker:client-libs-dependencies_template + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + stage: build + tags: + - gcp rules: - # This job runs on the schedule extended pipeline, so we want it - # to execute 'always'. - - when: always + - when: always + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_client_libs_dependencies_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: client_libs_dependencies_image_tag.env diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index b743e455672a..28bccd1535b2 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -284,6 +284,26 @@ let jobs pipeline_type = () |> job_external_split in + let _job_docker_client_libs_dependencies = + job_docker_authenticated + ~__POS__ + ~rules:(make_rules ~changes:changeset_kaitai_e2e_files ()) + ~stage:Stages.build + ~name:"oc.docker:client-libs-dependencies" + (* These image are not built for external use. *) + ~ci_docker_hub:false + (* Handle docker initialization, if necessary, in [./scripts/ci/docker_client_libs_dependencies_build.sh]. *) + ~skip_docker_initialization:true + ["./scripts/ci/docker_client_libs_dependencies_build.sh"] + ~artifacts: + (artifacts + ~reports: + (reports ~dotenv:"client_libs_dependencies_image_tag.env" ()) + []) + |> job_external_split + ~before_merging_suffix:"before_merging" + ~scheduled_suffix:"other" + in let build = let build_arm_rules = make_rules ~label:"ci--arm64" ~manual:true () in let _job_build_arm64_release : Tezos_ci.tezos_job = @@ -317,7 +337,9 @@ let jobs pipeline_type = let _job_build_rpm_amd64 = job_build_rpm_amd64 () |> job_external in () | Before_merging -> ()) ; - (* TODO: include the jobs defined above when full pipeline is generated *) + (* TODO: include the jobs defined above when full pipeline is + generated, as well as rust tool chain and client libs docker + builds. *) [] in let packaging = diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 0dd7c1ccc3d2..e3adf86ff990 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -297,6 +297,15 @@ let changeset_opam_jobs = "scripts/version.sh"; ] +let changeset_kaitai_e2e_files = + [ + "images/**/*"; + "src/**/*"; + "contrib/*kaitai*/**/*"; + ".gitlab/**/*"; + ".gitlab-ci.yml"; + ] + (** {2 Job makers} *) (** Helper to create jobs that uses the Docker daemon. diff --git a/ci/bin/main.ml b/ci/bin/main.ml index f581db136d35..6e974e70dff1 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -178,8 +178,6 @@ let () = | ".gitlab/ci/jobs/build/oc.build_kernels.yml" | ".gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra.yml" | ".gitlab/ci/jobs/build/oc.build_x86_64-released.yml" - | ".gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-before_merging.yml" - | ".gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-other.yml" | ".gitlab/ci/jobs/build/oc.tezt:fetch-records.yml" | ".gitlab/ci/jobs/build/ocaml-check.yml" | ".gitlab/ci/jobs/build/select_tezts.yml" -- GitLab From 9fce3204e039b88dae9e7903278311efa4593b61 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Tue, 12 Mar 2024 14:52:04 +0100 Subject: [PATCH 06/11] CI: rename [oc.docker:client-libs-deps.-{other=>scheduled_...}.yml] --- ...docker:client-libs-dependencies-scheduled_extended_test.yml} | 0 .gitlab/ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/code_verification.ml | 2 -- 3 files changed, 1 insertion(+), 3 deletions(-) rename .gitlab/ci/jobs/build/{oc.docker:client-libs-dependencies-other.yml => oc.docker:client-libs-dependencies-scheduled_extended_test.yml} (100%) diff --git a/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-other.yml b/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-scheduled_extended_test.yml similarity index 100% rename from .gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-other.yml rename to .gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-scheduled_extended_test.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index e81624b5c027..4c9b28a436be 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -13,7 +13,7 @@ include: # These jobs are necessary to run flaky tezts - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-scheduled_extended_test.yml - - .gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-other.yml + - .gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-scheduled_extended_test.yml - .gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra.yml - .gitlab/ci/jobs/build/oc.build_x86_64-released.yml - .gitlab/ci/jobs/build/oc.build_kernels.yml diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 28bccd1535b2..b5ab3a196d47 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -301,8 +301,6 @@ let jobs pipeline_type = (reports ~dotenv:"client_libs_dependencies_image_tag.env" ()) []) |> job_external_split - ~before_merging_suffix:"before_merging" - ~scheduled_suffix:"other" in let build = let build_arm_rules = make_rules ~label:"ci--arm64" ~manual:true () in -- GitLab From 6e5c186bbb49ec0405a923bff151afd7c0fc78e7 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Mon, 11 Mar 2024 15:51:08 +0100 Subject: [PATCH 07/11] CI: refactor, split [ocaml-check] --- .../ci/jobs/build/ocaml-check-before_merging.yml | 14 ++++++++++++++ .../build/ocaml-check-scheduled_extended_test.yml | 7 +++++++ .gitlab/ci/jobs/build/ocaml-check.yml | 11 +---------- .gitlab/ci/pipelines/before_merging.yml | 2 +- .gitlab/ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/main.ml | 2 ++ 6 files changed, 26 insertions(+), 12 deletions(-) create mode 100644 .gitlab/ci/jobs/build/ocaml-check-before_merging.yml create mode 100644 .gitlab/ci/jobs/build/ocaml-check-scheduled_extended_test.yml diff --git a/.gitlab/ci/jobs/build/ocaml-check-before_merging.yml b/.gitlab/ci/jobs/build/ocaml-check-before_merging.yml new file mode 100644 index 000000000000..2f4b16ac5f52 --- /dev/null +++ b/.gitlab/ci/jobs/build/ocaml-check-before_merging.yml @@ -0,0 +1,14 @@ +include: .gitlab/ci/jobs/build/ocaml-check.yml + +ocaml-check: + extends: + - .ocaml-check + - .needs__trigger + rules: + - changes: + - src/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - devtools/**/* + when: on_success diff --git a/.gitlab/ci/jobs/build/ocaml-check-scheduled_extended_test.yml b/.gitlab/ci/jobs/build/ocaml-check-scheduled_extended_test.yml new file mode 100644 index 000000000000..0e1e3e6cf63c --- /dev/null +++ b/.gitlab/ci/jobs/build/ocaml-check-scheduled_extended_test.yml @@ -0,0 +1,7 @@ +include: .gitlab/ci/jobs/build/ocaml-check.yml + +ocaml-check: + extends: + - .ocaml-check + rules: + - when: always diff --git a/.gitlab/ci/jobs/build/ocaml-check.yml b/.gitlab/ci/jobs/build/ocaml-check.yml index ab8e560c3c43..420bebbcec5f 100644 --- a/.gitlab/ci/jobs/build/ocaml-check.yml +++ b/.gitlab/ci/jobs/build/ocaml-check.yml @@ -1,15 +1,6 @@ -ocaml-check: +.ocaml-check: extends: - .tags_template__build - .oc.build_template - - .needs__trigger - rules: - - changes: - - src/**/* - - tezt/**/* - - .gitlab/**/* - - .gitlab-ci.yml - - devtools/**/* - when: on_success script: - dune build @check diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 56c722216e85..910dbf48ec6a 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -14,7 +14,7 @@ include: # Stage: build - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml - .gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-before_merging.yml - - .gitlab/ci/jobs/build/ocaml-check.yml + - .gitlab/ci/jobs/build/ocaml-check-before_merging.yml - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-before_merging.yml - .gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra-before_merging.yml - .gitlab/ci/jobs/build/oc.build_arm64-released-before_merging.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 4c9b28a436be..183afda94da9 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -21,7 +21,7 @@ include: # These jobs may not have been run in before_merging pipeline # because of absence of certain changes - - .gitlab/ci/jobs/build/ocaml-check.yml + - .gitlab/ci/jobs/build/ocaml-check-scheduled_extended_test.yml - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-scheduled_extended_test.yml - .gitlab/ci/jobs/doc/documentation.yml diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 6e974e70dff1..814a6eba3e7a 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -180,6 +180,8 @@ let () = | ".gitlab/ci/jobs/build/oc.build_x86_64-released.yml" | ".gitlab/ci/jobs/build/oc.tezt:fetch-records.yml" | ".gitlab/ci/jobs/build/ocaml-check.yml" + | ".gitlab/ci/jobs/build/ocaml-check-before_merging.yml" + | ".gitlab/ci/jobs/build/ocaml-check-scheduled_extended_test.yml" | ".gitlab/ci/jobs/build/select_tezts.yml" | ".gitlab/ci/jobs/coverage/common.yml" | ".gitlab/ci/jobs/coverage/oc.unified_coverage-before_merging.yml" -- GitLab From f1479dc37216497c567e92fb8ebfcf3f7f33c0e4 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Mon, 11 Mar 2024 15:54:03 +0100 Subject: [PATCH 08/11] CI: generate [ocaml-check] --- .../jobs/build/ocaml-check-before_merging.yml | 34 +++++++++++++------ .../ocaml-check-scheduled_extended_test.yml | 18 +++++++--- .gitlab/ci/jobs/build/ocaml-check.yml | 6 ---- ci/bin/code_verification.ml | 17 ++++++++++ ci/bin/common.ml | 3 ++ ci/bin/main.ml | 3 -- 6 files changed, 57 insertions(+), 24 deletions(-) delete mode 100644 .gitlab/ci/jobs/build/ocaml-check.yml diff --git a/.gitlab/ci/jobs/build/ocaml-check-before_merging.yml b/.gitlab/ci/jobs/build/ocaml-check-before_merging.yml index 2f4b16ac5f52..59994e8289f3 100644 --- a/.gitlab/ci/jobs/build/ocaml-check-before_merging.yml +++ b/.gitlab/ci/jobs/build/ocaml-check-before_merging.yml @@ -1,14 +1,26 @@ -include: .gitlab/ci/jobs/build/ocaml-check.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. ocaml-check: - extends: - - .ocaml-check - - .needs__trigger + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp rules: - - changes: - - src/**/* - - tezt/**/* - - .gitlab/**/* - - .gitlab-ci.yml - - devtools/**/* - when: on_success + - changes: + - src/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - devtools/**/* + when: on_success + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - dune build @check diff --git a/.gitlab/ci/jobs/build/ocaml-check-scheduled_extended_test.yml b/.gitlab/ci/jobs/build/ocaml-check-scheduled_extended_test.yml index 0e1e3e6cf63c..6ca5c5892c2b 100644 --- a/.gitlab/ci/jobs/build/ocaml-check-scheduled_extended_test.yml +++ b/.gitlab/ci/jobs/build/ocaml-check-scheduled_extended_test.yml @@ -1,7 +1,17 @@ -include: .gitlab/ci/jobs/build/ocaml-check.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. ocaml-check: - extends: - - .ocaml-check + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp rules: - - when: always + - when: always + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - dune build @check diff --git a/.gitlab/ci/jobs/build/ocaml-check.yml b/.gitlab/ci/jobs/build/ocaml-check.yml deleted file mode 100644 index 420bebbcec5f..000000000000 --- a/.gitlab/ci/jobs/build/ocaml-check.yml +++ /dev/null @@ -1,6 +0,0 @@ -.ocaml-check: - extends: - - .tags_template__build - - .oc.build_template - script: - - dune build @check diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index b5ab3a196d47..9dfce9196680 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -335,6 +335,23 @@ let jobs pipeline_type = let _job_build_rpm_amd64 = job_build_rpm_amd64 () |> job_external in () | Before_merging -> ()) ; + let _job_ocaml_check : tezos_job = + job + ~__POS__ + ~name:"ocaml-check" + ~image:Images.runtime_build_dependencies + ~stage:Stages.build + ~dependencies:dependencies_needs_trigger + ~rules:(make_rules ~changes:changeset_ocaml_files ()) + ~before_script: + (before_script + ~take_ownership:true + ~source_version:true + ~eval_opam:true + []) + ["dune build @check"] + |> job_external_split + in (* TODO: include the jobs defined above when full pipeline is generated, as well as rust tool chain and client libs docker builds. *) diff --git a/ci/bin/common.ml b/ci/bin/common.ml index e3adf86ff990..65b3c6a0edaa 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -306,6 +306,9 @@ let changeset_kaitai_e2e_files = ".gitlab-ci.yml"; ] +let changeset_ocaml_files = + ["src/**/*"; "tezt/**/*"; ".gitlab/**/*"; ".gitlab-ci.yml"; "devtools/**/*"] + (** {2 Job makers} *) (** Helper to create jobs that uses the Docker daemon. diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 814a6eba3e7a..149794ba71b5 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -179,9 +179,6 @@ let () = | ".gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra.yml" | ".gitlab/ci/jobs/build/oc.build_x86_64-released.yml" | ".gitlab/ci/jobs/build/oc.tezt:fetch-records.yml" - | ".gitlab/ci/jobs/build/ocaml-check.yml" - | ".gitlab/ci/jobs/build/ocaml-check-before_merging.yml" - | ".gitlab/ci/jobs/build/ocaml-check-scheduled_extended_test.yml" | ".gitlab/ci/jobs/build/select_tezts.yml" | ".gitlab/ci/jobs/coverage/common.yml" | ".gitlab/ci/jobs/coverage/oc.unified_coverage-before_merging.yml" -- GitLab From c32be39dc22a9ca364148cf59a566ba9261a9bd7 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Thu, 14 Mar 2024 14:18:19 +0100 Subject: [PATCH 09/11] CI: generate [oc.build_x86_64-*] jobs --- .gitlab/ci/jobs/build/common.yml | 29 ------------ ...ld_x86_64-exp-dev-extra-before_merging.yml | 44 +++++++++++++++++++ ...-exp-dev-extra-scheduled_extended_test.yml | 33 ++++++++++++++ .../build/oc.build_x86_64-exp-dev-extra.yml | 16 ------- ...c.build_x86_64-released-before_merging.yml | 42 ++++++++++++++++++ ...86_64-released-scheduled_extended_test.yml | 31 +++++++++++++ .../jobs/build/oc.build_x86_64-released.yml | 12 ----- .gitlab/ci/jobs/shared/templates.yml | 10 ----- .gitlab/ci/pipelines/before_merging.yml | 4 +- .../ci/pipelines/schedule_extended_test.yml | 4 +- ci/bin/code_verification.ml | 27 ++++++++++++ ci/bin/main.ml | 3 -- 12 files changed, 181 insertions(+), 74 deletions(-) delete mode 100644 .gitlab/ci/jobs/build/common.yml create mode 100644 .gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra-before_merging.yml create mode 100644 .gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra-scheduled_extended_test.yml delete mode 100644 .gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra.yml create mode 100644 .gitlab/ci/jobs/build/oc.build_x86_64-released-before_merging.yml create mode 100644 .gitlab/ci/jobs/build/oc.build_x86_64-released-scheduled_extended_test.yml delete mode 100644 .gitlab/ci/jobs/build/oc.build_x86_64-released.yml diff --git a/.gitlab/ci/jobs/build/common.yml b/.gitlab/ci/jobs/build/common.yml deleted file mode 100644 index fa2b2a600f43..000000000000 --- a/.gitlab/ci/jobs/build/common.yml +++ /dev/null @@ -1,29 +0,0 @@ -.oc.build: - extends: - - .oc.build_template - - .rules__octez_changes - variables: - ARCH: "" - script: - - ./scripts/ci/build_full_unreleased.sh - artifacts: - name: "build-$ARCH-$CI_COMMIT_REF_SLUG" - paths: - - octez-* - - src/proto_*/parameters/*.json - - _build/default/src/lib_protocol_compiler/bin/main_native.exe - - _build/default/tezt/tests/main.exe - - _build/default/contrib/octez_injector_server/octez_injector_server.exe - expire_in: 1 day - when: on_success - -.oc.build_x86_64: - extends: - - .oc.build - # Add coverage instrumentation for the integration tests. - - .oc.template__coverage_instrumentation - # These job are the main bottleneck since a lot of tests depend on - # them. So we start them even before sanity_ci. - - .needs__trigger - variables: - ARCH: "x86_64" diff --git a/.gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra-before_merging.yml b/.gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra-before_merging.yml new file mode 100644 index 000000000000..6d416deebae4 --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra-before_merging.yml @@ -0,0 +1,44 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.build_x86_64-exp-dev-extra: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/ci/build_full_unreleased.sh + variables: + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/experimental-executables script-inputs/dev-executables + BUILD_EXTRA: src/bin_tps_evaluation/main_tps_evaluation.exe src/bin_octogram/octogram_main.exe + tezt/tests/main.exe contrib/octez_injector_server/octez_injector_server.exe + COVERAGE_OPTIONS: --instrument-with bisect_ppx + artifacts: + name: build-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-* + - src/proto_*/parameters/*.json + - _build/default/src/lib_protocol_compiler/bin/main_native.exe + - _build/default/tezt/tests/main.exe + - _build/default/contrib/octez_injector_server/octez_injector_server.exe + when: on_success diff --git a/.gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra-scheduled_extended_test.yml b/.gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra-scheduled_extended_test.yml new file mode 100644 index 000000000000..914cd5dd8985 --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra-scheduled_extended_test.yml @@ -0,0 +1,33 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.build_x86_64-exp-dev-extra: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + rules: + - when: always + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/ci/build_full_unreleased.sh + variables: + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/experimental-executables script-inputs/dev-executables + BUILD_EXTRA: src/bin_tps_evaluation/main_tps_evaluation.exe src/bin_octogram/octogram_main.exe + tezt/tests/main.exe contrib/octez_injector_server/octez_injector_server.exe + COVERAGE_OPTIONS: --instrument-with bisect_ppx + artifacts: + name: build-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-* + - src/proto_*/parameters/*.json + - _build/default/src/lib_protocol_compiler/bin/main_native.exe + - _build/default/tezt/tests/main.exe + - _build/default/contrib/octez_injector_server/octez_injector_server.exe + when: on_success diff --git a/.gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra.yml b/.gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra.yml deleted file mode 100644 index f34ee3947260..000000000000 --- a/.gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra.yml +++ /dev/null @@ -1,16 +0,0 @@ -include: .gitlab/ci/jobs/build/common.yml - -# 'oc.build_x86_64-exp-dev-extra' builds the developer and experimental -# executables, as well as the tezt test suite used by the subsequent -# 'tezt' jobs and TPS evaluation tool. -oc.build_x86_64-exp-dev-extra: - extends: - - .tags_template__build - - .oc.build_x86_64 - - .rules__octez_changes - variables: - EXECUTABLE_FILES: "script-inputs/experimental-executables script-inputs/dev-executables" - # BUILD_EXTRA contains dune targets that should be built in addition to EXECUTABLE_FILES above. - # Typically, it will contain test runners (like tezt and octogram) and binaries to test that are - # not part of any 'script-inputs/*-executables', such as `Unreleased` binaries. - BUILD_EXTRA: "src/bin_tps_evaluation/main_tps_evaluation.exe src/bin_octogram/octogram_main.exe tezt/tests/main.exe contrib/octez_injector_server/octez_injector_server.exe" diff --git a/.gitlab/ci/jobs/build/oc.build_x86_64-released-before_merging.yml b/.gitlab/ci/jobs/build/oc.build_x86_64-released-before_merging.yml new file mode 100644 index 000000000000..c4cbfb5ec6a8 --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.build_x86_64-released-before_merging.yml @@ -0,0 +1,42 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.build_x86_64-released: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/ci/build_full_unreleased.sh + variables: + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/released-executables + COVERAGE_OPTIONS: --instrument-with bisect_ppx + artifacts: + name: build-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-* + - src/proto_*/parameters/*.json + - _build/default/src/lib_protocol_compiler/bin/main_native.exe + - _build/default/tezt/tests/main.exe + - _build/default/contrib/octez_injector_server/octez_injector_server.exe + when: on_success diff --git a/.gitlab/ci/jobs/build/oc.build_x86_64-released-scheduled_extended_test.yml b/.gitlab/ci/jobs/build/oc.build_x86_64-released-scheduled_extended_test.yml new file mode 100644 index 000000000000..f187c98f56f3 --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.build_x86_64-released-scheduled_extended_test.yml @@ -0,0 +1,31 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.build_x86_64-released: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + rules: + - when: always + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/ci/build_full_unreleased.sh + variables: + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/released-executables + COVERAGE_OPTIONS: --instrument-with bisect_ppx + artifacts: + name: build-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-* + - src/proto_*/parameters/*.json + - _build/default/src/lib_protocol_compiler/bin/main_native.exe + - _build/default/tezt/tests/main.exe + - _build/default/contrib/octez_injector_server/octez_injector_server.exe + when: on_success diff --git a/.gitlab/ci/jobs/build/oc.build_x86_64-released.yml b/.gitlab/ci/jobs/build/oc.build_x86_64-released.yml deleted file mode 100644 index 5eb18ee657c9..000000000000 --- a/.gitlab/ci/jobs/build/oc.build_x86_64-released.yml +++ /dev/null @@ -1,12 +0,0 @@ -include: .gitlab/ci/jobs/build/common.yml - -# The build_x86_64 jobs are split in two to keep the artifact size -# under the 1GB hard limit set by GitLab. -# 'oc.build_x86_64-released' builds the released executables. -oc.build_x86_64-released: - extends: - - .tags_template__build - - .oc.build_x86_64 - - .rules__octez_changes - variables: - EXECUTABLE_FILES: "script-inputs/released-executables" diff --git a/.gitlab/ci/jobs/shared/templates.yml b/.gitlab/ci/jobs/shared/templates.yml index 8c99aba097a9..8be4f0dc71e0 100644 --- a/.gitlab/ci/jobs/shared/templates.yml +++ b/.gitlab/ci/jobs/shared/templates.yml @@ -201,16 +201,6 @@ .oc.template__retry_flaky: retry: 2 -.oc.build_template: - extends: - - .default_settings_template - - .image_template__runtime_build_dependencies - stage: build - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - .oc.kernels_template: extends: - .default_settings_template diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 910dbf48ec6a..3185266da8f6 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -18,8 +18,8 @@ include: - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-before_merging.yml - .gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra-before_merging.yml - .gitlab/ci/jobs/build/oc.build_arm64-released-before_merging.yml - - .gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra.yml - - .gitlab/ci/jobs/build/oc.build_x86_64-released.yml + - .gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra-before_merging.yml + - .gitlab/ci/jobs/build/oc.build_x86_64-released-before_merging.yml - .gitlab/ci/jobs/build/oc.build_kernels.yml - .gitlab/ci/jobs/doc/documentation.yml - .gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 183afda94da9..5e8ae2e566c6 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -14,8 +14,8 @@ include: # These jobs are necessary to run flaky tezts - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-scheduled_extended_test.yml - .gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-scheduled_extended_test.yml - - .gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra.yml - - .gitlab/ci/jobs/build/oc.build_x86_64-released.yml + - .gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra-scheduled_extended_test.yml + - .gitlab/ci/jobs/build/oc.build_x86_64-released-scheduled_extended_test.yml - .gitlab/ci/jobs/build/oc.build_kernels.yml - .gitlab/ci/jobs/build/oc.tezt:fetch-records.yml diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 9dfce9196680..2977a9327677 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -237,6 +237,7 @@ let jobs pipeline_type = |> job_external in (* TODO: put job_trigger here when full pipeline is generated *) + (* TODO: the dependency on job_trigger does not have to be optional *) ([], Dependent [Optional job_trigger]) in let sanity = @@ -335,6 +336,32 @@ let jobs pipeline_type = let _job_build_rpm_amd64 = job_build_rpm_amd64 () |> job_external in () | Before_merging -> ()) ; + (* The build_x86_64 jobs are split in two to keep the artifact size + under the 1GB hard limit set by GitLab. *) + (* [_job_build_x86_64_release] builds the released executables. *) + let _job_build_x86_64_release = + job_build_dynamic_binaries + ~__POS__ + ~arch:Amd64 + ~dependencies:dependencies_needs_trigger + ~release:true + ~rules:(make_rules ~changes:changeset_octez ()) + () + |> job_external_split + in + (* 'oc.build_x86_64-exp-dev-extra' builds the developer and experimental + executables, as well as the tezt test suite used by the subsequent + 'tezt' jobs and TPS evaluation tool. *) + let _job_build_x86_64_exp_dev_extra = + job_build_dynamic_binaries + ~__POS__ + ~arch:Amd64 + ~dependencies:dependencies_needs_trigger + ~release:false + ~rules:(make_rules ~changes:changeset_octez ()) + () + |> job_external_split + in let _job_ocaml_check : tezos_job = job ~__POS__ diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 149794ba71b5..72e712de2b3c 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -174,10 +174,7 @@ let () = (* Paths to exclude from generation check. As files are translated to CI-in-OCaml, they should be removed from this function *) let exclude = function - | ".gitlab/ci/jobs/build/common.yml" | ".gitlab/ci/jobs/build/oc.build_kernels.yml" - | ".gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra.yml" - | ".gitlab/ci/jobs/build/oc.build_x86_64-released.yml" | ".gitlab/ci/jobs/build/oc.tezt:fetch-records.yml" | ".gitlab/ci/jobs/build/select_tezts.yml" | ".gitlab/ci/jobs/coverage/common.yml" -- GitLab From f6eec1b04fea67a46e766ebffc3b5526e54be231 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Thu, 14 Mar 2024 14:23:09 +0100 Subject: [PATCH 10/11] CI: generate job [oc.build_kernels] --- .../build/oc.build_kernels-before_merging.yml | 58 +++++++++++++++++++ ....build_kernels-scheduled_extended_test.yml | 46 +++++++++++++++ .gitlab/ci/jobs/build/oc.build_kernels.yml | 36 ------------ .gitlab/ci/pipelines/before_merging.yml | 2 +- .../ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/code_verification.ml | 46 ++++++++++++++- ci/bin/common.ml | 9 +++ ci/bin/main.ml | 1 - 8 files changed, 158 insertions(+), 42 deletions(-) create mode 100644 .gitlab/ci/jobs/build/oc.build_kernels-before_merging.yml create mode 100644 .gitlab/ci/jobs/build/oc.build_kernels-scheduled_extended_test.yml delete mode 100644 .gitlab/ci/jobs/build/oc.build_kernels.yml diff --git a/.gitlab/ci/jobs/build/oc.build_kernels-before_merging.yml b/.gitlab/ci/jobs/build/oc.build_kernels-before_merging.yml new file mode 100644 index 000000000000..dff3c60287ea --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.build_kernels-before_merging.yml @@ -0,0 +1,58 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.build_kernels: + image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} + stage: build + tags: + - gcp + rules: + - changes: + - images/**/* + - scripts/ci/**/* + - kernels.mk + - etherlink.mk + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + cache: + - key: kernels + paths: + - cargo/ + - key: kernels-sccache + paths: + - _sccache + script: + - make -f kernels.mk build + - make -f etherlink.mk evm_kernel.wasm + - make -C src/risc_v risc-v-sandbox risc-v-dummy.elf + - make -C src/risc_v/tests/ build + variables: + CC: clang + CARGO_HOME: $CI_PROJECT_DIR/cargo + NATIVE_TARGET: x86_64-unknown-linux-musl + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + RUSTC_WRAPPER: sccache + artifacts: + name: build-kernels-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - evm_kernel.wasm + - smart-rollup-installer + - sequenced_kernel.wasm + - tx_kernel.wasm + - tx_kernel_dal.wasm + - dal_echo_kernel.wasm + - src/risc_v/risc-v-sandbox + - src/risc_v/risc-v-dummy.elf + - src/risc_v/tests/inline_asm/rv64-inline-asm-tests + when: on_success diff --git a/.gitlab/ci/jobs/build/oc.build_kernels-scheduled_extended_test.yml b/.gitlab/ci/jobs/build/oc.build_kernels-scheduled_extended_test.yml new file mode 100644 index 000000000000..8528f779c74f --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.build_kernels-scheduled_extended_test.yml @@ -0,0 +1,46 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.build_kernels: + image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} + stage: build + tags: + - gcp + rules: + - when: on_success + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + cache: + - key: kernels + paths: + - cargo/ + - key: kernels-sccache + paths: + - _sccache + script: + - make -f kernels.mk build + - make -f etherlink.mk evm_kernel.wasm + - make -C src/risc_v risc-v-sandbox risc-v-dummy.elf + - make -C src/risc_v/tests/ build + variables: + CC: clang + CARGO_HOME: $CI_PROJECT_DIR/cargo + NATIVE_TARGET: x86_64-unknown-linux-musl + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + RUSTC_WRAPPER: sccache + artifacts: + name: build-kernels-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - evm_kernel.wasm + - smart-rollup-installer + - sequenced_kernel.wasm + - tx_kernel.wasm + - tx_kernel_dal.wasm + - dal_echo_kernel.wasm + - src/risc_v/risc-v-sandbox + - src/risc_v/risc-v-dummy.elf + - src/risc_v/tests/inline_asm/rv64-inline-asm-tests + when: on_success diff --git a/.gitlab/ci/jobs/build/oc.build_kernels.yml b/.gitlab/ci/jobs/build/oc.build_kernels.yml deleted file mode 100644 index 687d32466830..000000000000 --- a/.gitlab/ci/jobs/build/oc.build_kernels.yml +++ /dev/null @@ -1,36 +0,0 @@ -oc.build_kernels: - extends: - - .tags_template__build - - .oc.kernels_template - stage: build - variables: - # We store caches in _build because GitLab CI Cache requires that - # cached files are in a sub-folder of the checkout. - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - RUSTC_WRAPPER: "sccache" - script: - - make -f kernels.mk build - - make -f etherlink.mk evm_kernel.wasm - - make -C src/risc_v risc-v-sandbox risc-v-dummy.elf - - make -C src/risc_v/tests/ build - artifacts: - name: "build-kernels-$CI_COMMIT_REF_SLUG" - paths: - - evm_kernel.wasm - - smart-rollup-installer - - sequenced_kernel.wasm - - tx_kernel.wasm - - tx_kernel_dal.wasm - - dal_echo_kernel.wasm - - src/risc_v/risc-v-sandbox - - src/risc_v/risc-v-dummy.elf - - src/risc_v/tests/inline_asm/rv64-inline-asm-tests - expire_in: 1 day - when: on_success - cache: - - key: kernels - paths: - - cargo/ - - key: kernels-sccache - paths: - - _sccache diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 3185266da8f6..6243fb2e5659 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -20,7 +20,7 @@ include: - .gitlab/ci/jobs/build/oc.build_arm64-released-before_merging.yml - .gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra-before_merging.yml - .gitlab/ci/jobs/build/oc.build_x86_64-released-before_merging.yml - - .gitlab/ci/jobs/build/oc.build_kernels.yml + - .gitlab/ci/jobs/build/oc.build_kernels-before_merging.yml - .gitlab/ci/jobs/doc/documentation.yml - .gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml - .gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 5e8ae2e566c6..9906c7f4e733 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -16,7 +16,7 @@ include: - .gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-scheduled_extended_test.yml - .gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra-scheduled_extended_test.yml - .gitlab/ci/jobs/build/oc.build_x86_64-released-scheduled_extended_test.yml - - .gitlab/ci/jobs/build/oc.build_kernels.yml + - .gitlab/ci/jobs/build/oc.build_kernels-scheduled_extended_test.yml - .gitlab/ci/jobs/build/oc.tezt:fetch-records.yml # These jobs may not have been run in before_merging pipeline diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 2977a9327677..47e1ea9acd4c 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -177,11 +177,13 @@ let jobs pipeline_type = enable the job [On_success] in the [before_merging] pipeline. This is safe, but prefer specifying a [changes] clause if possible. *) - let make_rules ?label ?changes ?(manual = false) () = + let make_rules ?label ?changes ?(manual = false) ?(dependent = false) () = match pipeline_type with | Schedule_extended_test -> - (* The scheduled pipeline always runs all tests unconditionally. *) - [job_rule ~when_:Always ()] + (* The scheduled pipeline always runs all jobs unconditionally + -- unless they are dependent on a previous, non-trigger job, in the + pipeline. *) + [job_rule ~when_:(if dependent then On_success else Always) ()] | Before_merging -> (* MR labels can be used to force tests to run. *) (match label with @@ -379,6 +381,44 @@ let jobs pipeline_type = ["dune build @check"] |> job_external_split in + let _job_build_kernels : tezos_job = + job + ~__POS__ + ~name:"oc.build_kernels" + ~image:Images.rust_toolchain + ~stage:Stages.build + ~dependencies:(Dependent [Artifacts job_docker_rust_toolchain]) + ~rules: + (make_rules ~changes:changeset_octez_or_kernels ~dependent:true ()) + [ + "make -f kernels.mk build"; + "make -f etherlink.mk evm_kernel.wasm"; + "make -C src/risc_v risc-v-sandbox risc-v-dummy.elf"; + "make -C src/risc_v/tests/ build"; + ] + ~artifacts: + (artifacts + ~name:"build-kernels-$CI_COMMIT_REF_SLUG" + ~expire_in:(Days 1) + ~when_:On_success + [ + "evm_kernel.wasm"; + "smart-rollup-installer"; + "sequenced_kernel.wasm"; + "tx_kernel.wasm"; + "tx_kernel_dal.wasm"; + "dal_echo_kernel.wasm"; + "src/risc_v/risc-v-sandbox"; + "src/risc_v/risc-v-dummy.elf"; + "src/risc_v/tests/inline_asm/rv64-inline-asm-tests"; + ]) + ~cache: + [ + {key = "kernels"; paths = ["cargo/"]}; + {key = "kernels-sccache"; paths = ["_sccache"]}; + ] + |> enable_kernels |> enable_sccache |> job_external_split + in (* TODO: include the jobs defined above when full pipeline is generated, as well as rust tool chain and client libs docker builds. *) diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 65b3c6a0edaa..bc3594b37868 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -227,6 +227,15 @@ let enable_sccache ?error_log ?idle_timeout ?log @ opt_var "SCCACHE_IDLE_TIMEOUT" Fun.id idle_timeout @ opt_var "SCCACHE_LOG" Fun.id log) +(** Add common variables used by jobs compiling kernels *) +let enable_kernels = + Tezos_ci.append_variables + [ + ("CC", "clang"); + ("CARGO_HOME", "$CI_PROJECT_DIR/cargo"); + ("NATIVE_TARGET", "x86_64-unknown-linux-musl"); + ] + (** {2 Changesets} *) let changeset_octez = diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 72e712de2b3c..32c19f91bd05 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -174,7 +174,6 @@ let () = (* Paths to exclude from generation check. As files are translated to CI-in-OCaml, they should be removed from this function *) let exclude = function - | ".gitlab/ci/jobs/build/oc.build_kernels.yml" | ".gitlab/ci/jobs/build/oc.tezt:fetch-records.yml" | ".gitlab/ci/jobs/build/select_tezts.yml" | ".gitlab/ci/jobs/coverage/common.yml" -- GitLab From 69b10f5257e71f14ee01b58586ee6532141c3050 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Thu, 14 Mar 2024 14:29:04 +0100 Subject: [PATCH 11/11] CI: generate job [oc.tezt:fetch-records] --- .../oc.tezt:fetch-records-before_merging.yml | 36 ++++++++++++++++++ ...:fetch-records-scheduled_extended_test.yml | 28 ++++++++++++++ .../ci/jobs/build/oc.tezt:fetch-records.yml | 32 ---------------- .gitlab/ci/pipelines/before_merging.yml | 2 +- .../ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/code_verification.ml | 37 +++++++++++++++++++ ci/bin/main.ml | 1 - 7 files changed, 103 insertions(+), 35 deletions(-) create mode 100644 .gitlab/ci/jobs/build/oc.tezt:fetch-records-before_merging.yml create mode 100644 .gitlab/ci/jobs/build/oc.tezt:fetch-records-scheduled_extended_test.yml delete mode 100644 .gitlab/ci/jobs/build/oc.tezt:fetch-records.yml diff --git a/.gitlab/ci/jobs/build/oc.tezt:fetch-records-before_merging.yml b/.gitlab/ci/jobs/build/oc.tezt:fetch-records-before_merging.yml new file mode 100644 index 000000000000..9dd2ae10b646 --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.tezt:fetch-records-before_merging.yml @@ -0,0 +1,36 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.tezt:fetch-records: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + dependencies: [] + allow_failure: true + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - dune exec scripts/ci/update_records/update.exe -- --log-file tezt-fetch-records.log + --from last-successful-schedule-extended-test --info + after_script: + - ./scripts/ci/filter_corrupted_records.sh + artifacts: + expire_in: 4 hours + paths: + - tezt-fetch-records.log + - tezt/records/*.json + - tezt/records/*.json.broken + when: always diff --git a/.gitlab/ci/jobs/build/oc.tezt:fetch-records-scheduled_extended_test.yml b/.gitlab/ci/jobs/build/oc.tezt:fetch-records-scheduled_extended_test.yml new file mode 100644 index 000000000000..0cd78c0ee566 --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.tezt:fetch-records-scheduled_extended_test.yml @@ -0,0 +1,28 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.tezt:fetch-records: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + rules: + - when: always + dependencies: [] + allow_failure: true + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - dune exec scripts/ci/update_records/update.exe -- --log-file tezt-fetch-records.log + --from last-successful-schedule-extended-test --info + after_script: + - ./scripts/ci/filter_corrupted_records.sh + artifacts: + expire_in: 4 hours + paths: + - tezt-fetch-records.log + - tezt/records/*.json + - tezt/records/*.json.broken + when: always diff --git a/.gitlab/ci/jobs/build/oc.tezt:fetch-records.yml b/.gitlab/ci/jobs/build/oc.tezt:fetch-records.yml deleted file mode 100644 index b7f24191aeee..000000000000 --- a/.gitlab/ci/jobs/build/oc.tezt:fetch-records.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Fetch records for Tezt generated on the last merge request pipeline -# on the most recently merged MR and makes them available in artifacts -# for future merge request pipelines. -oc.tezt:fetch-records: - extends: - - .default_settings_template - - .image_template__runtime_build_dependencies - - .rules__octez_changes - - .tags_template__build - stage: build - before_script: - - ./scripts/ci/take_ownership.sh - - . ./scripts/version.sh - - eval $(opam env) - script: - - dune exec scripts/ci/update_records/update.exe -- --log-file tezt-fetch-records.log --from last-successful-schedule-extended-test --info - after_script: - - ./scripts/ci/filter_corrupted_records.sh - # Allow failure of this job, since Tezt can use the records stored - # in the repo as backup for balancing. - allow_failure: true - artifacts: - paths: - - tezt-fetch-records.log - - tezt/records/*.json - # Keep broken records for debugging - - tezt/records/*.json.broken - # Always store artifacts to have the logs for debugging - when: always - # These artifacts are only used within this pipeline, so they - # don't need a long expiration time. - expire_in: 4 hours diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 6243fb2e5659..c746bdb1f29b 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -26,7 +26,7 @@ include: - .gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml - .gitlab/ci/jobs/build/oc.build:rpm:amd64-manual.yml - .gitlab/ci/jobs/build/oc.build:dpkg:amd64-manual.yml - - .gitlab/ci/jobs/build/oc.tezt:fetch-records.yml + - .gitlab/ci/jobs/build/oc.tezt:fetch-records-before_merging.yml - .gitlab/ci/jobs/build/select_tezts.yml # Stage: test diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 9906c7f4e733..d62f61648b31 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -17,7 +17,7 @@ include: - .gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra-scheduled_extended_test.yml - .gitlab/ci/jobs/build/oc.build_x86_64-released-scheduled_extended_test.yml - .gitlab/ci/jobs/build/oc.build_kernels-scheduled_extended_test.yml - - .gitlab/ci/jobs/build/oc.tezt:fetch-records.yml + - .gitlab/ci/jobs/build/oc.tezt:fetch-records-scheduled_extended_test.yml # These jobs may not have been run in before_merging pipeline # because of absence of certain changes diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 47e1ea9acd4c..19ee2272f4d3 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -419,6 +419,43 @@ let jobs pipeline_type = ] |> enable_kernels |> enable_sccache |> job_external_split in + (* Fetch records for Tezt generated on the last merge request pipeline + on the most recently merged MR and makes them available in artifacts + for future merge request pipelines. *) + let _job_tezt_fetch_records : tezos_job = + job + ~__POS__ + ~name:"oc.tezt:fetch-records" + ~image:Images.runtime_build_dependencies + ~stage:Stages.build + ~before_script: + (before_script + ~take_ownership:true + ~source_version:true + ~eval_opam:true + []) + ~rules:(make_rules ~changes:changeset_octez ()) + [ + "dune exec scripts/ci/update_records/update.exe -- --log-file \ + tezt-fetch-records.log --from \ + last-successful-schedule-extended-test --info"; + ] + ~after_script:["./scripts/ci/filter_corrupted_records.sh"] + (* Allow failure of this job, since Tezt can use the records + stored in the repo as backup for balancing. *) + ~allow_failure:Yes + ~artifacts: + (artifacts + ~expire_in:(Hours 4) + ~when_:Always + [ + "tezt-fetch-records.log"; + "tezt/records/*.json"; + (* Keep broken records for debugging *) + "tezt/records/*.json.broken"; + ]) + |> job_external_split + in (* TODO: include the jobs defined above when full pipeline is generated, as well as rust tool chain and client libs docker builds. *) diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 32c19f91bd05..a89c8d87bfef 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -174,7 +174,6 @@ let () = (* Paths to exclude from generation check. As files are translated to CI-in-OCaml, they should be removed from this function *) let exclude = function - | ".gitlab/ci/jobs/build/oc.tezt:fetch-records.yml" | ".gitlab/ci/jobs/build/select_tezts.yml" | ".gitlab/ci/jobs/coverage/common.yml" | ".gitlab/ci/jobs/coverage/oc.unified_coverage-before_merging.yml" -- GitLab