From 34c6a4eb83eeb94dc394a3afa848715df2d45a5b Mon Sep 17 00:00:00 2001 From: Marouane BENALLA Date: Tue, 20 May 2025 15:48:42 +0200 Subject: [PATCH 1/2] CI, CIAO: Schedule pipeline for docker images refresh --- .gitlab/ci/pipelines/master_branch.yml | 130 ++++++++++++------------- ci/bin/master_branch.ml | 59 +++++++---- ci/lib_tezos_ci/rules.ml | 3 + ci/lib_tezos_ci/rules.mli | 7 ++ 4 files changed, 116 insertions(+), 83 deletions(-) diff --git a/.gitlab/ci/pipelines/master_branch.yml b/.gitlab/ci/pipelines/master_branch.yml index 41314e5db974..418c3b7edb2c 100644 --- a/.gitlab/ci/pipelines/master_branch.yml +++ b/.gitlab/ci/pipelines/master_branch.yml @@ -285,6 +285,59 @@ oc.build_arm64-exp-dev-extra: - etherlink-governance-observer when: on_success +datadog_pipeline_trace: + image: datadog/ci:v2.44.0 + stage: start + tags: + - gcp + dependencies: [] + allow_failure: true + timeout: 60 minutes + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + script: + - CI_MERGE_REQUEST_IID=${CI_MERGE_REQUEST_IID:-none} + - DATADOG_SITE=datadoghq.eu datadog-ci tag --level pipeline --tags pipeline_type:$PIPELINE_TYPE + --tags mr_number:$CI_MERGE_REQUEST_IID + +oc.unified_coverage: + image: ${ci_image_name}/test:${ci_image_tag} + stage: test_coverage + tags: + - gcp + rules: + - when: always + dependencies: + - oc.docker:ci:amd64 + allow_failure: true + timeout: 60 minutes + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - mkdir -p _coverage_report + - dune exec scripts/ci/download_coverage/download.exe -- --from last-merged-pipeline + --info --log-file _coverage_report/download_coverage.log + - ./scripts/ci/report_coverage.sh + variables: + PROJECT: $CI_PROJECT_PATH + DEFAULT_BRANCH: $CI_COMMIT_SHA + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + SLACK_COVERAGE_CHANNEL: C02PHBE7W73 + artifacts: + expire_in: 15 days + paths: + - _coverage_report/ + - $BISECT_FILE + reports: + coverage_report: + coverage_format: cobertura + path: _coverage_report/cobertura.xml + when: always + expose_as: Coverage report + coverage: '/Coverage: ([^%]+%)/' + oc.docker:amd64: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: build @@ -334,58 +387,26 @@ oc.docker:arm64: IMAGE_ARCH_PREFIX: arm64_ EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables -datadog_pipeline_trace: - image: datadog/ci:v2.44.0 - stage: start +docker:merge_manifests: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: prepare_release tags: - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 dependencies: [] - allow_failure: true - timeout: 60 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - script: - - CI_MERGE_REQUEST_IID=${CI_MERGE_REQUEST_IID:-none} - - DATADOG_SITE=datadoghq.eu datadog-ci tag --level pipeline --tags pipeline_type:$PIPELINE_TYPE - --tags mr_number:$CI_MERGE_REQUEST_IID - -oc.unified_coverage: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test_coverage - tags: - - gcp - rules: - - when: always - dependencies: - - oc.docker:ci:amd64 - allow_failure: true timeout: 60 minutes before_script: - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/version.sh - - eval $(opam env) + - ./scripts/ci/docker_initialize.sh script: - - mkdir -p _coverage_report - - dune exec scripts/ci/download_coverage/download.exe -- --from last-merged-pipeline - --info --log-file _coverage_report/download_coverage.log - - ./scripts/ci/report_coverage.sh + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind variables: - PROJECT: $CI_PROJECT_PATH - DEFAULT_BRANCH: $CI_COMMIT_SHA - BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ - SLACK_COVERAGE_CHANNEL: C02PHBE7W73 - artifacts: - expire_in: 15 days - paths: - - _coverage_report/ - - $BISECT_FILE - reports: - coverage_report: - coverage_format: cobertura - path: _coverage_report/cobertura.xml - when: always - expose_as: Coverage report - coverage: '/Coverage: ([^%]+%)/' + DOCKER_VERSION: 24.0.7 + CI_DOCKER_HUB: "true" documentation:odoc: image: ${ci_image_name}/test:${ci_image_tag} @@ -654,27 +675,6 @@ publish:documentation: SCCACHE_DIR: $CI_PROJECT_DIR/_sccache SCCACHE_CACHE_SIZE: 5G -docker:merge_manifests: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 - stage: prepare_release - tags: - - gcp - needs: - - oc.docker:amd64 - - oc.docker:arm64 - dependencies: [] - timeout: 60 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - ./scripts/ci/docker_initialize.sh - script: - - ./scripts/ci/docker_merge_manifests.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.7 - CI_DOCKER_HUB: "true" - publish_kernel_sdk: image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} stage: manual diff --git a/ci/bin/master_branch.ml b/ci/bin/master_branch.ml index 0bb5abfa0df3..c5573e1b90f3 100644 --- a/ci/bin/master_branch.ml +++ b/ci/bin/master_branch.ml @@ -55,6 +55,33 @@ let jobs_documentation : tezos_job list = in [job_odoc; job_manuals; job_docgen; job_build_all; job_publish_documentation] +(* Defines the jobs of the [schedule_docker_build_pipeline] pipeline. + + This pipeline runs scheduled on the [master] branch. The goal + of this pipeline is to publish fresh Docker images for 'master' using the latest Alpine packages. *) + +let octez_distribution_docker_jobs = + let job_docker_amd64_experimental : tezos_job = + job_docker_build ~__POS__ ~rules:rules_always ~arch:Amd64 Experimental + in + let job_docker_arm64_experimental : tezos_job = + job_docker_build ~__POS__ ~rules:rules_always ~arch:Arm64 Experimental + in + let job_docker_merge_manifests = + job_docker_merge_manifests + ~__POS__ + ~ci_docker_hub:true + ~job_docker_amd64:job_docker_amd64_experimental + ~job_docker_arm64:job_docker_arm64_experimental + in + [ + (* Stage: build *) + job_docker_amd64_experimental; + job_docker_arm64_experimental; + (* Stage: prepare_release *) + job_docker_merge_manifests; + ] + let jobs = (* Like in the {!Schedule_extended_test} variant of {!Code_verification} pipelines, we'd like to run as many jobs as @@ -70,18 +97,18 @@ let jobs = [changes:] in different pipelines, see {{:https://docs.gitlab.com/ee/ci/jobs/job_troubleshooting.html#jobs-or-pipelines-run-unexpectedly-when-using-changes} GitLab Docs: Jobs or pipelines run unexpectedly when using changes}. *) - let job_docker_amd64_experimental : tezos_job = - job_docker_build ~__POS__ ~rules:rules_always ~arch:Amd64 Experimental + let job_static_arm64 = + job_build_static_binaries ~__POS__ ~arch:Arm64 ~rules:rules_always () in - let job_docker_arm64_experimental : tezos_job = - job_docker_build ~__POS__ ~rules:rules_always ~arch:Arm64 Experimental - in - let job_docker_merge_manifests = - job_docker_merge_manifests + let job_static_x86_64 = + job_build_static_binaries ~__POS__ - ~ci_docker_hub:true - ~job_docker_amd64:job_docker_amd64_experimental - ~job_docker_arm64:job_docker_arm64_experimental + ~arch:Amd64 + ~cpu:Very_high + ~retry: + {max = 2; when_ = [Stuck_or_timeout_failure; Runner_system_failure]} + ~rules:rules_always + () in let job_unified_coverage_default : tezos_job = job @@ -146,18 +173,14 @@ let jobs = job_static_arm64; job_build_arm64_release; job_build_arm64_exp_dev_extra; - job_docker_amd64_experimental; - job_docker_arm64_experimental; (* Stage: sanity *) job_datadog_pipeline_trace; (* Stage: test_coverage *) job_unified_coverage_default; ] + (* Jobs to build and update on Docker Hub the Octez Docker image. *) + @ octez_distribution_docker_jobs (* Stage: doc *) @ jobs_documentation - @ [ - (* Stage: prepare_release *) - job_docker_merge_manifests; - (* Stage: manual *) - job_publish_kernel_sdk; - ] + (* Stage: manual *) + @ [job_publish_kernel_sdk] diff --git a/ci/lib_tezos_ci/rules.ml b/ci/lib_tezos_ci/rules.ml index 7beb1ea84e2e..068d19c4ffed 100644 --- a/ci/lib_tezos_ci/rules.ml +++ b/ci/lib_tezos_ci/rules.ml @@ -82,6 +82,9 @@ let schedule_container_scanning_evm_node_releases = let schedule_documentation = scheduled && var "TZ_SCHEDULE_KIND" == str "DOCUMENTATION" +let schedule_docker_build = + scheduled && var "TZ_SCHEDULE_KIND" == str "DOCKER_FRESH_IMAGE_BUILD" + let on_master = Predefined_vars.ci_commit_branch == str "master" let on_branch branch = Predefined_vars.ci_commit_branch == str branch diff --git a/ci/lib_tezos_ci/rules.mli b/ci/lib_tezos_ci/rules.mli index 30d478b44b0c..b8eccd22d653 100644 --- a/ci/lib_tezos_ci/rules.mli +++ b/ci/lib_tezos_ci/rules.mli @@ -120,6 +120,13 @@ val schedule_container_scanning_evm_node_releases : If.t [TZ_SCHEDULE_KIND] set to [DOCUMENTATION]. *) val schedule_documentation : If.t +(** A rule that is true for scheduled pipelines that build + fresh Docker images (latest available Alpine packages). + + Such pipelines have [CI_PIPELINE_SOURCE] set to [scheduled] and + [TZ_SCHEDULE_KIND] set to [DOCKER_FRESH_IMAGE_BUILD]. *) +val schedule_docker_build : If.t + (** A rule that is true if [CI_COMMIT_BRANCH] is a given branch. *) val on_branch : string -> If.t -- GitLab From 2283f0f9b9530fe57d3c7f101d9d6225e09f6fce Mon Sep 17 00:00:00 2001 From: Marouane BENALLA Date: Tue, 20 May 2025 15:49:27 +0200 Subject: [PATCH 2/2] CI, CIAO: Register the new scheduled pipeline --- .gitlab-ci.yml | 9 ++ .../schedule_docker_build_pipeline.yml | 143 ++++++++++++++++++ ci/bin/main.ml | 11 +- 3 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 .gitlab/ci/pipelines/schedule_docker_build_pipeline.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8fb06e4a54e2..44feeafd279c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -119,6 +119,11 @@ workflow: variables: PIPELINE_TYPE: schedule_documentation when: always + - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "DOCKER_FRESH_IMAGE_BUILD" + variables: + PIPELINE_TYPE: schedule_docker_build_pipeline + DOCKER_FORCE_BUILD: "true" + when: always - if: $CI_PIPELINE_SOURCE == "api" && $TZ_API_KIND == "RELEASE_PAGE" && $CI_PROJECT_NAMESPACE != "tezos" variables: @@ -274,6 +279,10 @@ include: rules: - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "DOCUMENTATION" when: always +- local: .gitlab/ci/pipelines/schedule_docker_build_pipeline.yml + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "DOCKER_FRESH_IMAGE_BUILD" + when: always - local: .gitlab/ci/pipelines/publish_test_release_page.yml rules: - if: $CI_PIPELINE_SOURCE == "api" && $TZ_API_KIND == "RELEASE_PAGE" && $CI_PROJECT_NAMESPACE diff --git a/.gitlab/ci/pipelines/schedule_docker_build_pipeline.yml b/.gitlab/ci/pipelines/schedule_docker_build_pipeline.yml new file mode 100644 index 000000000000..b6bde3ad54be --- /dev/null +++ b/.gitlab/ci/pipelines/schedule_docker_build_pipeline.yml @@ -0,0 +1,143 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +stages: +- images +- build +- prepare_release + +oc.docker:ci:amd64: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: images + tags: + - gcp + dependencies: [] + timeout: 90 minutes + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + script: + - ./images/ci_create_ci_images.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + CI_DOCKER_HUB: "false" + ARCH: amd64 + artifacts: + reports: + dotenv: ci_image_tag.env + +oc.docker:rust-toolchain:amd64: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: images + tags: + - gcp + dependencies: [] + timeout: 60 minutes + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + script: + - ./scripts/ci/docker_rust_toolchain_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: rust_toolchain_image_tag.env + +oc.docker:ci:arm64: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: images + tags: + - gcp_arm64 + dependencies: [] + timeout: 90 minutes + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + script: + - ./images/ci_create_ci_images.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + CI_DOCKER_HUB: "false" + ARCH: arm64 + artifacts: + reports: + dotenv: ci_image_tag.env + retry: + max: 1 + when: + - runner_system_failure + +oc.docker:amd64: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: build + tags: + - gcp + rules: + - when: always + dependencies: + - oc.docker:rust-toolchain:amd64 + - oc.docker:ci:amd64 + timeout: 60 minutes + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + CI_DOCKER_HUB: "true" + DOCKER_BUILD_TARGET: with-evm-artifacts + IMAGE_ARCH_PREFIX: amd64_ + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + +oc.docker:arm64: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: build + tags: + - gcp_arm64 + rules: + - when: always + dependencies: + - oc.docker:ci:arm64 + timeout: 60 minutes + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + CI_DOCKER_HUB: "true" + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: arm64_ + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + +docker:merge_manifests: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: prepare_release + tags: + - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 + dependencies: [] + timeout: 60 minutes + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + CI_DOCKER_HUB: "true" diff --git a/ci/bin/main.ml b/ci/bin/main.ml index c11c2b50eb20..7586adb09141 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -378,7 +378,16 @@ let () = |> List.map (with_interruptible false)) ~description: "Scheduled pipeline that updates the octez.com/docs documentation \ - without being interrupted." + without being interrupted." ; + register + "schedule_docker_build_pipeline" + schedule_docker_build + ~jobs:Master_branch.octez_distribution_docker_jobs + ~variables:[("DOCKER_FORCE_BUILD", "true")] + ~description: + "Scheduled pipeline for forcing building fresh Docker image (skipping \ + any cache mechanism) for the current master branch of Octez. The newly \ + built images should contains the latest available Alpine packages" (** {Manual pipelines} *) -- GitLab