From 76fbbb593cda54db30bd94dfa76ddd597ab3bfe9 Mon Sep 17 00:00:00 2001 From: Killian Delarue Date: Thu, 20 Feb 2025 14:36:05 +0100 Subject: [PATCH 1/3] Teztale, CI: Build job now takes architecture as argument --- .gitlab/ci/pipelines/before_merging.yml | 45 +++++++++++++++++- .gitlab/ci/pipelines/merge_train.yml | 45 +++++++++++++++++- .gitlab/ci/pipelines/octez_monitoring.yml | 47 ++++++++++++++++++- .../ci/pipelines/schedule_extended_test.yml | 42 ++++++++++++++++- ci/bin/code_verification.ml | 6 ++- ci/bin/release_tag.ml | 3 +- teztale/ci/teztale.ml | 7 ++- 7 files changed, 186 insertions(+), 9 deletions(-) diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 340649f0e8cc..380b71f81d9b 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -1214,7 +1214,7 @@ build_grafazos_dashboards: - grafazos/output/**/*.json when: on_success -build_teztale: +teztale.build:static-x86_64: image: ${ci_image_name}/build:${ci_image_tag} stage: build tags: @@ -1257,6 +1257,49 @@ build_teztale: - octez-teztale-* when: on_success +teztale.build:static-arm64: + image: ${ci_image_name}/build:${ci_image_tag} + stage: build + tags: + - gcp_arm64 + rules: + - changes: + - teztale/**/* + when: on_success + - when: manual + allow_failure: true + dependencies: + - oc.docker:ci:arm64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + - . ./scripts/ci/sccache-start.sh + script: + - make teztale + after_script: + - ./scripts/ci/sccache-stop.sh + variables: + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 5G + artifacts: + name: teztale-binaries + expire_in: 1 day + paths: + - octez-teztale-* + when: on_success + build-layer1-profiling: image: ${ci_image_name}/build:${ci_image_tag} stage: build diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index b3bdee4ce313..abab2020f930 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -1213,7 +1213,7 @@ build_grafazos_dashboards: - grafazos/output/**/*.json when: on_success -build_teztale: +teztale.build:static-x86_64: image: ${ci_image_name}/build:${ci_image_tag} stage: build tags: @@ -1256,6 +1256,49 @@ build_teztale: - octez-teztale-* when: on_success +teztale.build:static-arm64: + image: ${ci_image_name}/build:${ci_image_tag} + stage: build + tags: + - gcp_arm64 + rules: + - changes: + - teztale/**/* + when: on_success + - when: manual + allow_failure: true + dependencies: + - oc.docker:ci:arm64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + - . ./scripts/ci/sccache-start.sh + script: + - make teztale + after_script: + - ./scripts/ci/sccache-stop.sh + variables: + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 5G + artifacts: + name: teztale-binaries + expire_in: 1 day + paths: + - octez-teztale-* + when: on_success + build-layer1-profiling: image: ${ci_image_name}/build:${ci_image_tag} stage: build diff --git a/.gitlab/ci/pipelines/octez_monitoring.yml b/.gitlab/ci/pipelines/octez_monitoring.yml index b2b34c385b25..93ccda740c59 100644 --- a/.gitlab/ci/pipelines/octez_monitoring.yml +++ b/.gitlab/ci/pipelines/octez_monitoring.yml @@ -54,6 +54,30 @@ oc.docker:jsonnet:amd64: reports: dotenv: jsonnet_image_tag.env +oc.docker:ci:arm64: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: images + tags: + - gcp_arm64 + dependencies: [] + timeout: 90 minutes + before_script: [] + script: + - ./images/ci_create_ci_images.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + CI_DOCKER_HUB: "false" + ARCH: arm64 + artifacts: + reports: + dotenv: ci_image_tag.env + retry: + max: 1 + when: + - runner_system_failure + datadog_pipeline_trace: image: datadog/ci:v2.44.0 stage: start @@ -128,7 +152,28 @@ build-layer1-profiling: paths: - ./octez-binaries/x86_64/octez-node -build_teztale: +teztale.build:static-arm64: + image: ${ci_image_name}/build:${ci_image_tag} + stage: build + tags: + - gcp_arm64 + dependencies: + - oc.docker:ci:arm64 + timeout: 60 minutes + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - make teztale + artifacts: + name: teztale-binaries + expire_in: never + paths: + - octez-teztale-* + when: on_success + +teztale.build:static-x86_64: image: ${ci_image_name}/build:${ci_image_tag} stage: build tags: diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 5e963b06c1ab..c3ad0e92c733 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -822,7 +822,7 @@ build_grafazos_dashboards: - grafazos/output/**/*.json when: on_success -build_teztale: +teztale.build:static-x86_64: image: ${ci_image_name}/build:${ci_image_tag} stage: build tags: @@ -862,6 +862,46 @@ build_teztale: - octez-teztale-* when: on_success +teztale.build:static-arm64: + image: ${ci_image_name}/build:${ci_image_tag} + stage: build + tags: + - gcp_arm64 + rules: + - when: always + dependencies: + - oc.docker:ci:arm64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + interruptible: false + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + - . ./scripts/ci/sccache-start.sh + script: + - make teztale + after_script: + - ./scripts/ci/sccache-stop.sh + variables: + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 5G + artifacts: + name: teztale-binaries + expire_in: 1 day + paths: + - octez-teztale-* + when: on_success + build-layer1-profiling: image: ${ci_image_name}/build:${ci_image_tag} stage: build diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 3eb47b220cf4..99cb0573f8fb 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -758,8 +758,9 @@ let jobs pipeline_type = | Schedule_extended_test -> Grafazos_ci.job_build_grafazos ~rules:[job_rule ~when_:Always ()] () in - let job_build_teztale = + let job_build_teztale ~arch = Teztale.job_build + ~arch ~rules:(make_rules ~manual:Yes ~changes:Teztale.changeset ()) () |> enable_cargo_cache |> enable_sccache @@ -777,7 +778,8 @@ let jobs pipeline_type = job_tezt_fetch_records; build_octez_source; job_build_grafazos; - job_build_teztale; + job_build_teztale ~arch:Amd64; + job_build_teztale ~arch:Arm64; job_build_layer1_profiling (); ] @ Option.to_list job_select_tezts diff --git a/ci/bin/release_tag.ml b/ci/bin/release_tag.ml index b93cd9d5f25e..d30c3753759d 100644 --- a/ci/bin/release_tag.ml +++ b/ci/bin/release_tag.ml @@ -52,7 +52,8 @@ let monitoring_child_pipeline = job_datadog_pipeline_trace; Grafazos_ci.job_build_grafazos (); job_build_layer1_profiling ~expire_in:Never (); - Teztale.job_build ~expire_in:Never (); + Teztale.job_build ~expire_in:Never ~arch:Arm64 (); + Teztale.job_build ~expire_in:Never ~arch:Amd64 (); ] (** Create an Octez release tag pipeline of type {!release_tag_pipeline_type}. diff --git a/teztale/ci/teztale.ml b/teztale/ci/teztale.ml index 311d6e76f5ff..b7e5e62a6f7f 100644 --- a/teztale/ci/teztale.ml +++ b/teztale/ci/teztale.ml @@ -13,10 +13,13 @@ module String_set = Set.Make (String) let changeset = Changeset.(make ["teztale/**/*"]) (** Job that builds the Teztale executables *) -let job_build ?rules ?(expire_in = Gitlab_ci.Types.(Duration (Days 1))) () = +let job_build ?rules ?(expire_in = Gitlab_ci.Types.(Duration (Days 1))) ~arch () + = + let arch_string = arch_to_string arch in job ~__POS__ - ~name:"build_teztale" + ~arch + ~name:("teztale.build:static-" ^ arch_string) ~image:Images.CI.build ~stage:Stages.build ?rules -- GitLab From 677c0ebeb30aaa70d2c3ccee8f396c7625a1a142 Mon Sep 17 00:00:00 2001 From: Killian Delarue Date: Thu, 20 Feb 2025 14:40:45 +0100 Subject: [PATCH 2/3] Teztale, CI: Add [static] profile to teztale build jobs --- .gitlab/ci/pipelines/before_merging.yml | 2 ++ .gitlab/ci/pipelines/merge_train.yml | 2 ++ .gitlab/ci/pipelines/octez_monitoring.yml | 4 ++++ .gitlab/ci/pipelines/schedule_extended_test.yml | 2 ++ teztale/ci/teztale.ml | 1 + 5 files changed, 11 insertions(+) diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 380b71f81d9b..564f6b26a796 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -1247,6 +1247,7 @@ teztale.build:static-x86_64: after_script: - ./scripts/ci/sccache-stop.sh variables: + PROFILE: static CARGO_NET_OFFLINE: "false" SCCACHE_DIR: $CI_PROJECT_DIR/_sccache SCCACHE_CACHE_SIZE: 5G @@ -1290,6 +1291,7 @@ teztale.build:static-arm64: after_script: - ./scripts/ci/sccache-stop.sh variables: + PROFILE: static CARGO_NET_OFFLINE: "false" SCCACHE_DIR: $CI_PROJECT_DIR/_sccache SCCACHE_CACHE_SIZE: 5G diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index abab2020f930..abee53c73a3f 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -1246,6 +1246,7 @@ teztale.build:static-x86_64: after_script: - ./scripts/ci/sccache-stop.sh variables: + PROFILE: static CARGO_NET_OFFLINE: "false" SCCACHE_DIR: $CI_PROJECT_DIR/_sccache SCCACHE_CACHE_SIZE: 5G @@ -1289,6 +1290,7 @@ teztale.build:static-arm64: after_script: - ./scripts/ci/sccache-stop.sh variables: + PROFILE: static CARGO_NET_OFFLINE: "false" SCCACHE_DIR: $CI_PROJECT_DIR/_sccache SCCACHE_CACHE_SIZE: 5G diff --git a/.gitlab/ci/pipelines/octez_monitoring.yml b/.gitlab/ci/pipelines/octez_monitoring.yml index 93ccda740c59..e39a976460d5 100644 --- a/.gitlab/ci/pipelines/octez_monitoring.yml +++ b/.gitlab/ci/pipelines/octez_monitoring.yml @@ -166,6 +166,8 @@ teztale.build:static-arm64: - eval $(opam env) script: - make teztale + variables: + PROFILE: static artifacts: name: teztale-binaries expire_in: never @@ -187,6 +189,8 @@ teztale.build:static-x86_64: - eval $(opam env) script: - make teztale + variables: + PROFILE: static artifacts: name: teztale-binaries expire_in: never diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index c3ad0e92c733..b09a95c622d3 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -852,6 +852,7 @@ teztale.build:static-x86_64: after_script: - ./scripts/ci/sccache-stop.sh variables: + PROFILE: static CARGO_NET_OFFLINE: "false" SCCACHE_DIR: $CI_PROJECT_DIR/_sccache SCCACHE_CACHE_SIZE: 5G @@ -892,6 +893,7 @@ teztale.build:static-arm64: after_script: - ./scripts/ci/sccache-stop.sh variables: + PROFILE: static CARGO_NET_OFFLINE: "false" SCCACHE_DIR: $CI_PROJECT_DIR/_sccache SCCACHE_CACHE_SIZE: 5G diff --git a/teztale/ci/teztale.ml b/teztale/ci/teztale.ml index b7e5e62a6f7f..23489824cd86 100644 --- a/teztale/ci/teztale.ml +++ b/teztale/ci/teztale.ml @@ -23,6 +23,7 @@ let job_build ?rules ?(expire_in = Gitlab_ci.Types.(Duration (Days 1))) ~arch () ~image:Images.CI.build ~stage:Stages.build ?rules + ~variables:[("PROFILE", "static")] ~artifacts: (artifacts ~name:"teztale-binaries" -- GitLab From d93568eaa7f259d58b60e1c5efa467e20f7f29cc Mon Sep 17 00:00:00 2001 From: Killian Delarue Date: Fri, 28 Feb 2025 09:29:45 +0100 Subject: [PATCH 3/3] CI, Teztale: Change artefacts path --- .gitlab/ci/pipelines/before_merging.yml | 8 ++++++-- .gitlab/ci/pipelines/merge_train.yml | 8 ++++++-- .gitlab/ci/pipelines/octez_monitoring.yml | 10 ++++++++-- .gitlab/ci/pipelines/schedule_extended_test.yml | 8 ++++++-- teztale/ci/teztale.ml | 7 ++++++- 5 files changed, 32 insertions(+), 9 deletions(-) diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 564f6b26a796..346fb40de231 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -1245,6 +1245,8 @@ teztale.build:static-x86_64: script: - make teztale after_script: + - mkdir -p ./teztale-binaries/x86_64 + - mv octez-teztale-* ./teztale-binaries/x86_64/ - ./scripts/ci/sccache-stop.sh variables: PROFILE: static @@ -1255,7 +1257,7 @@ teztale.build:static-x86_64: name: teztale-binaries expire_in: 1 day paths: - - octez-teztale-* + - teztale-binaries/x86_64/octez-teztale-* when: on_success teztale.build:static-arm64: @@ -1289,6 +1291,8 @@ teztale.build:static-arm64: script: - make teztale after_script: + - mkdir -p ./teztale-binaries/arm64 + - mv octez-teztale-* ./teztale-binaries/arm64/ - ./scripts/ci/sccache-stop.sh variables: PROFILE: static @@ -1299,7 +1303,7 @@ teztale.build:static-arm64: name: teztale-binaries expire_in: 1 day paths: - - octez-teztale-* + - teztale-binaries/arm64/octez-teztale-* when: on_success build-layer1-profiling: diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index abee53c73a3f..1ace66d9c3fa 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -1244,6 +1244,8 @@ teztale.build:static-x86_64: script: - make teztale after_script: + - mkdir -p ./teztale-binaries/x86_64 + - mv octez-teztale-* ./teztale-binaries/x86_64/ - ./scripts/ci/sccache-stop.sh variables: PROFILE: static @@ -1254,7 +1256,7 @@ teztale.build:static-x86_64: name: teztale-binaries expire_in: 1 day paths: - - octez-teztale-* + - teztale-binaries/x86_64/octez-teztale-* when: on_success teztale.build:static-arm64: @@ -1288,6 +1290,8 @@ teztale.build:static-arm64: script: - make teztale after_script: + - mkdir -p ./teztale-binaries/arm64 + - mv octez-teztale-* ./teztale-binaries/arm64/ - ./scripts/ci/sccache-stop.sh variables: PROFILE: static @@ -1298,7 +1302,7 @@ teztale.build:static-arm64: name: teztale-binaries expire_in: 1 day paths: - - octez-teztale-* + - teztale-binaries/arm64/octez-teztale-* when: on_success build-layer1-profiling: diff --git a/.gitlab/ci/pipelines/octez_monitoring.yml b/.gitlab/ci/pipelines/octez_monitoring.yml index e39a976460d5..38bfab2a751d 100644 --- a/.gitlab/ci/pipelines/octez_monitoring.yml +++ b/.gitlab/ci/pipelines/octez_monitoring.yml @@ -166,13 +166,16 @@ teztale.build:static-arm64: - eval $(opam env) script: - make teztale + after_script: + - mkdir -p ./teztale-binaries/arm64 + - mv octez-teztale-* ./teztale-binaries/arm64/ variables: PROFILE: static artifacts: name: teztale-binaries expire_in: never paths: - - octez-teztale-* + - teztale-binaries/arm64/octez-teztale-* when: on_success teztale.build:static-x86_64: @@ -189,11 +192,14 @@ teztale.build:static-x86_64: - eval $(opam env) script: - make teztale + after_script: + - mkdir -p ./teztale-binaries/x86_64 + - mv octez-teztale-* ./teztale-binaries/x86_64/ variables: PROFILE: static artifacts: name: teztale-binaries expire_in: never paths: - - octez-teztale-* + - teztale-binaries/x86_64/octez-teztale-* when: on_success diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index b09a95c622d3..15cd6d74f36d 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -850,6 +850,8 @@ teztale.build:static-x86_64: script: - make teztale after_script: + - mkdir -p ./teztale-binaries/x86_64 + - mv octez-teztale-* ./teztale-binaries/x86_64/ - ./scripts/ci/sccache-stop.sh variables: PROFILE: static @@ -860,7 +862,7 @@ teztale.build:static-x86_64: name: teztale-binaries expire_in: 1 day paths: - - octez-teztale-* + - teztale-binaries/x86_64/octez-teztale-* when: on_success teztale.build:static-arm64: @@ -891,6 +893,8 @@ teztale.build:static-arm64: script: - make teztale after_script: + - mkdir -p ./teztale-binaries/arm64 + - mv octez-teztale-* ./teztale-binaries/arm64/ - ./scripts/ci/sccache-stop.sh variables: PROFILE: static @@ -901,7 +905,7 @@ teztale.build:static-arm64: name: teztale-binaries expire_in: 1 day paths: - - octez-teztale-* + - teztale-binaries/arm64/octez-teztale-* when: on_success build-layer1-profiling: diff --git a/teztale/ci/teztale.ml b/teztale/ci/teztale.ml index 23489824cd86..d11b11f8011c 100644 --- a/teztale/ci/teztale.ml +++ b/teztale/ci/teztale.ml @@ -29,11 +29,16 @@ let job_build ?rules ?(expire_in = Gitlab_ci.Types.(Duration (Days 1))) ~arch () ~name:"teztale-binaries" ~expire_in ~when_:On_success - ["octez-teztale-*"]) + ["teztale-binaries/" ^ arch_string ^ "/octez-teztale-*"]) ~before_script: [ "./scripts/ci/take_ownership.sh"; ". ./scripts/version.sh"; "eval $(opam env)"; ] + ~after_script: + [ + "mkdir -p ./teztale-binaries/" ^ arch_string; + "mv octez-teztale-* ./teztale-binaries/" ^ arch_string ^ "/"; + ] ["make teztale"] -- GitLab