From 2e3181279b7c7fb2f3c00e049342ec9fe2e8819d Mon Sep 17 00:00:00 2001 From: Pietro Abate Date: Thu, 11 Dec 2025 15:59:11 +0100 Subject: [PATCH 1/2] XXX --- .gitlab/ci/pipelines/debian.daily.yml | 95 +++++++++++++------ .../pipelines/debian_repository_partial.yml | 66 ++++++++++--- .../debian_repository_partial_auto.yml | 66 ++++++++++--- .gitlab/ci/pipelines/non_release_tag.yml | 65 ++++++++++--- .gitlab/ci/pipelines/non_release_tag_test.yml | 65 ++++++++++--- .../ci/pipelines/octez_beta_release_tag.yml | 65 ++++++++++--- .../pipelines/octez_beta_release_tag_test.yml | 65 ++++++++++--- .../ci/pipelines/octez_major_release_tag.yml | 65 ++++++++++--- .../octez_major_release_tag_test.yml | 65 ++++++++++--- .../ci/pipelines/octez_minor_release_tag.yml | 65 ++++++++++--- .../octez_minor_release_tag_test.yml | 65 ++++++++++--- .../schedule_master_test_release.yml | 65 ++++++++++--- ci/lib_tezos_ci_jobs/debian_repository.ml | 23 +++-- scripts/ci/build-packages-dependencies.sh | 9 +- scripts/ci/docker-merge-base-images.sh | 22 ++++- 15 files changed, 669 insertions(+), 197 deletions(-) diff --git a/.gitlab/ci/pipelines/debian.daily.yml b/.gitlab/ci/pipelines/debian.daily.yml index dbe2793ccd73..b3f2cd3ef276 100644 --- a/.gitlab/ci/pipelines/debian.daily.yml +++ b/.gitlab/ci/pipelines/debian.daily.yml @@ -53,8 +53,8 @@ oc.docker-systemd_tests_debian: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: systemd-tests - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$DISTRIBUTION:$RELEASE parallel: @@ -90,8 +90,8 @@ oc.docker-systemd_tests_ubuntu: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: systemd-tests - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: ubuntu BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$DISTRIBUTION:$RELEASE parallel: @@ -126,8 +126,8 @@ oc.docker-build-debian-dependencies: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/${DISTRIBUTION}:${RELEASE} parallel: @@ -140,8 +140,47 @@ oc.docker-build-debian-dependencies: - gcp_very_high_cpu - gcp_arm64 +oc.docker-build-debian-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-build-debian-dependencies + dependencies: [] + timeout: 60 minutes + interruptible: false + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: debian + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - unstable + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + - gcp_arm64 + oc.build-debian: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -164,8 +203,8 @@ oc.build-debian: - ./scripts/ci/sccache-stop.sh variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian DUNE_BUILD_JOBS: -j 12 CARGO_INCREMENTAL: "0" @@ -188,12 +227,12 @@ oc.build-debian: - gcp_arm64 oc.build-data_packages: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS needs: - - oc.docker-build-debian-dependencies + - oc.docker-build-debian-dependencies.merge dependencies: [] timeout: 60 minutes interruptible: false @@ -209,8 +248,8 @@ oc.build-data_packages: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian RELEASE: trixie TAGS: gcp @@ -274,8 +313,8 @@ oc.docker-build-ubuntu-dependencies: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: ubuntu BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/${DISTRIBUTION}:${RELEASE} parallel: @@ -288,7 +327,7 @@ oc.docker-build-ubuntu-dependencies: - gcp_arm64 oc.build-ubuntu: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -311,8 +350,8 @@ oc.build-ubuntu: - ./scripts/ci/sccache-stop.sh variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: ubuntu DUNE_BUILD_JOBS: -j 12 CARGO_INCREMENTAL: "0" @@ -442,8 +481,8 @@ oc.install_bin_debian_bookworm_systemd_test: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: systemd-tests - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION PREFIX: "" DISTRIBUTION: debian RELEASE: bookworm @@ -476,8 +515,8 @@ oc.upgrade_bin_debian_bookworm-systemd: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: systemd-tests - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION PREFIX: "" DISTRIBUTION: debian RELEASE: bookworm @@ -580,8 +619,8 @@ oc.install_bin_ubuntu_noble_systemd: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: systemd-tests - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION PREFIX: "" DISTRIBUTION: ubuntu RELEASE: noble @@ -614,8 +653,8 @@ oc.upgrade_bin_ubuntu_jammy_systemd_test: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: systemd-tests - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION PREFIX: "" DISTRIBUTION: ubuntu RELEASE: jammy @@ -648,8 +687,8 @@ oc.upgrade_bin_ubuntu_noble_systemd_test: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: systemd-tests - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION PREFIX: "" DISTRIBUTION: ubuntu RELEASE: noble diff --git a/.gitlab/ci/pipelines/debian_repository_partial.yml b/.gitlab/ci/pipelines/debian_repository_partial.yml index b545773148a3..b552982239ba 100644 --- a/.gitlab/ci/pipelines/debian_repository_partial.yml +++ b/.gitlab/ci/pipelines/debian_repository_partial.yml @@ -67,8 +67,8 @@ oc.docker-systemd_tests_debian: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: systemd-tests - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$DISTRIBUTION:$RELEASE parallel: @@ -101,8 +101,8 @@ oc.docker-build-debian-dependencies: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/${DISTRIBUTION}:${RELEASE} parallel: @@ -113,8 +113,44 @@ oc.docker-build-debian-dependencies: TAGS: - gcp_very_high_cpu +oc.docker-build-debian-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-build-debian-dependencies + dependencies: [] + timeout: 60 minutes + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: debian + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + oc.build-debian: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -136,8 +172,8 @@ oc.build-debian: - ./scripts/ci/sccache-stop.sh variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -157,12 +193,12 @@ oc.build-debian: - gcp_very_high_cpu_ramfs oc.build-data_packages: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS needs: - - oc.docker-build-debian-dependencies + - oc.docker-build-debian-dependencies.merge dependencies: [] timeout: 60 minutes before_script: @@ -177,8 +213,8 @@ oc.build-data_packages: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian RELEASE: trixie TAGS: gcp @@ -291,8 +327,8 @@ oc.install_bin_debian_bookworm_systemd_test: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: systemd-tests - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION PREFIX: "" DISTRIBUTION: debian RELEASE: bookworm @@ -324,8 +360,8 @@ oc.upgrade_bin_debian_bookworm-systemd: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: systemd-tests - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION PREFIX: "" DISTRIBUTION: debian RELEASE: bookworm diff --git a/.gitlab/ci/pipelines/debian_repository_partial_auto.yml b/.gitlab/ci/pipelines/debian_repository_partial_auto.yml index b545773148a3..b552982239ba 100644 --- a/.gitlab/ci/pipelines/debian_repository_partial_auto.yml +++ b/.gitlab/ci/pipelines/debian_repository_partial_auto.yml @@ -67,8 +67,8 @@ oc.docker-systemd_tests_debian: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: systemd-tests - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$DISTRIBUTION:$RELEASE parallel: @@ -101,8 +101,8 @@ oc.docker-build-debian-dependencies: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/${DISTRIBUTION}:${RELEASE} parallel: @@ -113,8 +113,44 @@ oc.docker-build-debian-dependencies: TAGS: - gcp_very_high_cpu +oc.docker-build-debian-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-build-debian-dependencies + dependencies: [] + timeout: 60 minutes + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: debian + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + oc.build-debian: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -136,8 +172,8 @@ oc.build-debian: - ./scripts/ci/sccache-stop.sh variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian CARGO_INCREMENTAL: "0" SCCACHE_GCS_BUCKET: $GCP_SCCACHE_BUCKET @@ -157,12 +193,12 @@ oc.build-debian: - gcp_very_high_cpu_ramfs oc.build-data_packages: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS needs: - - oc.docker-build-debian-dependencies + - oc.docker-build-debian-dependencies.merge dependencies: [] timeout: 60 minutes before_script: @@ -177,8 +213,8 @@ oc.build-data_packages: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian RELEASE: trixie TAGS: gcp @@ -291,8 +327,8 @@ oc.install_bin_debian_bookworm_systemd_test: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: systemd-tests - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION PREFIX: "" DISTRIBUTION: debian RELEASE: bookworm @@ -324,8 +360,8 @@ oc.upgrade_bin_debian_bookworm-systemd: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: systemd-tests - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION PREFIX: "" DISTRIBUTION: debian RELEASE: bookworm diff --git a/.gitlab/ci/pipelines/non_release_tag.yml b/.gitlab/ci/pipelines/non_release_tag.yml index 239c3531e9d5..8274341f0709 100644 --- a/.gitlab/ci/pipelines/non_release_tag.yml +++ b/.gitlab/ci/pipelines/non_release_tag.yml @@ -374,8 +374,8 @@ oc.docker-build-debian-dependencies: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/${DISTRIBUTION}:${RELEASE} parallel: @@ -387,8 +387,45 @@ oc.docker-build-debian-dependencies: - gcp_very_high_cpu - gcp_arm64 +oc.docker-build-debian-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-build-debian-dependencies + dependencies: [] + timeout: 60 minutes + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: debian + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + - gcp_arm64 + oc.build-debian: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -410,8 +447,8 @@ oc.build-debian: - ./scripts/ci/sccache-stop.sh variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian DUNE_BUILD_JOBS: -j 12 CARGO_INCREMENTAL: "0" @@ -433,12 +470,12 @@ oc.build-debian: - gcp_arm64 oc.build-data_packages: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS needs: - - oc.docker-build-debian-dependencies + - oc.docker-build-debian-dependencies.merge dependencies: [] timeout: 60 minutes before_script: @@ -453,8 +490,8 @@ oc.build-data_packages: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian RELEASE: trixie TAGS: gcp @@ -516,8 +553,8 @@ oc.docker-build-ubuntu-dependencies: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: ubuntu BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/${DISTRIBUTION}:${RELEASE} parallel: @@ -530,7 +567,7 @@ oc.docker-build-ubuntu-dependencies: - gcp_arm64 oc.build-ubuntu: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -552,8 +589,8 @@ oc.build-ubuntu: - ./scripts/ci/sccache-stop.sh variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: ubuntu DUNE_BUILD_JOBS: -j 12 CARGO_INCREMENTAL: "0" diff --git a/.gitlab/ci/pipelines/non_release_tag_test.yml b/.gitlab/ci/pipelines/non_release_tag_test.yml index 0f2ad0f381c2..3b04132731c6 100644 --- a/.gitlab/ci/pipelines/non_release_tag_test.yml +++ b/.gitlab/ci/pipelines/non_release_tag_test.yml @@ -374,8 +374,8 @@ oc.docker-build-debian-dependencies: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/${DISTRIBUTION}:${RELEASE} parallel: @@ -387,8 +387,45 @@ oc.docker-build-debian-dependencies: - gcp_very_high_cpu - gcp_arm64 +oc.docker-build-debian-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-build-debian-dependencies + dependencies: [] + timeout: 60 minutes + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: debian + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + - gcp_arm64 + oc.build-debian: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -410,8 +447,8 @@ oc.build-debian: - ./scripts/ci/sccache-stop.sh variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian DUNE_BUILD_JOBS: -j 12 CARGO_INCREMENTAL: "0" @@ -433,12 +470,12 @@ oc.build-debian: - gcp_arm64 oc.build-data_packages: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS needs: - - oc.docker-build-debian-dependencies + - oc.docker-build-debian-dependencies.merge dependencies: [] timeout: 60 minutes before_script: @@ -453,8 +490,8 @@ oc.build-data_packages: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian RELEASE: trixie TAGS: gcp @@ -516,8 +553,8 @@ oc.docker-build-ubuntu-dependencies: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: ubuntu BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/${DISTRIBUTION}:${RELEASE} parallel: @@ -530,7 +567,7 @@ oc.docker-build-ubuntu-dependencies: - gcp_arm64 oc.build-ubuntu: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -552,8 +589,8 @@ oc.build-ubuntu: - ./scripts/ci/sccache-stop.sh variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: ubuntu DUNE_BUILD_JOBS: -j 12 CARGO_INCREMENTAL: "0" diff --git a/.gitlab/ci/pipelines/octez_beta_release_tag.yml b/.gitlab/ci/pipelines/octez_beta_release_tag.yml index 4a24f43dee66..eee3ba4b5b73 100644 --- a/.gitlab/ci/pipelines/octez_beta_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_beta_release_tag.yml @@ -376,8 +376,8 @@ oc.docker-build-debian-dependencies: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/${DISTRIBUTION}:${RELEASE} parallel: @@ -389,8 +389,45 @@ oc.docker-build-debian-dependencies: - gcp_very_high_cpu - gcp_arm64 +oc.docker-build-debian-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-build-debian-dependencies + dependencies: [] + timeout: 60 minutes + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: debian + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + - gcp_arm64 + oc.build-debian: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -412,8 +449,8 @@ oc.build-debian: - ./scripts/ci/sccache-stop.sh variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian DUNE_BUILD_JOBS: -j 12 CARGO_INCREMENTAL: "0" @@ -435,12 +472,12 @@ oc.build-debian: - gcp_arm64 oc.build-data_packages: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS needs: - - oc.docker-build-debian-dependencies + - oc.docker-build-debian-dependencies.merge dependencies: [] timeout: 60 minutes before_script: @@ -455,8 +492,8 @@ oc.build-data_packages: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian RELEASE: trixie TAGS: gcp @@ -518,8 +555,8 @@ oc.docker-build-ubuntu-dependencies: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: ubuntu BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/${DISTRIBUTION}:${RELEASE} parallel: @@ -532,7 +569,7 @@ oc.docker-build-ubuntu-dependencies: - gcp_arm64 oc.build-ubuntu: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -554,8 +591,8 @@ oc.build-ubuntu: - ./scripts/ci/sccache-stop.sh variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: ubuntu DUNE_BUILD_JOBS: -j 12 CARGO_INCREMENTAL: "0" diff --git a/.gitlab/ci/pipelines/octez_beta_release_tag_test.yml b/.gitlab/ci/pipelines/octez_beta_release_tag_test.yml index 9b190511f494..726687adaf37 100644 --- a/.gitlab/ci/pipelines/octez_beta_release_tag_test.yml +++ b/.gitlab/ci/pipelines/octez_beta_release_tag_test.yml @@ -376,8 +376,8 @@ oc.docker-build-debian-dependencies: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/${DISTRIBUTION}:${RELEASE} parallel: @@ -389,8 +389,45 @@ oc.docker-build-debian-dependencies: - gcp_very_high_cpu - gcp_arm64 +oc.docker-build-debian-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-build-debian-dependencies + dependencies: [] + timeout: 60 minutes + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: debian + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + - gcp_arm64 + oc.build-debian: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -412,8 +449,8 @@ oc.build-debian: - ./scripts/ci/sccache-stop.sh variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian DUNE_BUILD_JOBS: -j 12 CARGO_INCREMENTAL: "0" @@ -435,12 +472,12 @@ oc.build-debian: - gcp_arm64 oc.build-data_packages: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS needs: - - oc.docker-build-debian-dependencies + - oc.docker-build-debian-dependencies.merge dependencies: [] timeout: 60 minutes before_script: @@ -455,8 +492,8 @@ oc.build-data_packages: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian RELEASE: trixie TAGS: gcp @@ -518,8 +555,8 @@ oc.docker-build-ubuntu-dependencies: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: ubuntu BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/${DISTRIBUTION}:${RELEASE} parallel: @@ -532,7 +569,7 @@ oc.docker-build-ubuntu-dependencies: - gcp_arm64 oc.build-ubuntu: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -554,8 +591,8 @@ oc.build-ubuntu: - ./scripts/ci/sccache-stop.sh variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: ubuntu DUNE_BUILD_JOBS: -j 12 CARGO_INCREMENTAL: "0" diff --git a/.gitlab/ci/pipelines/octez_major_release_tag.yml b/.gitlab/ci/pipelines/octez_major_release_tag.yml index 3f7e6dbdc382..93059cbcaa3d 100644 --- a/.gitlab/ci/pipelines/octez_major_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_major_release_tag.yml @@ -376,8 +376,8 @@ oc.docker-build-debian-dependencies: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/${DISTRIBUTION}:${RELEASE} parallel: @@ -389,8 +389,45 @@ oc.docker-build-debian-dependencies: - gcp_very_high_cpu - gcp_arm64 +oc.docker-build-debian-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-build-debian-dependencies + dependencies: [] + timeout: 60 minutes + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: debian + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + - gcp_arm64 + oc.build-debian: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -412,8 +449,8 @@ oc.build-debian: - ./scripts/ci/sccache-stop.sh variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian DUNE_BUILD_JOBS: -j 12 CARGO_INCREMENTAL: "0" @@ -435,12 +472,12 @@ oc.build-debian: - gcp_arm64 oc.build-data_packages: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS needs: - - oc.docker-build-debian-dependencies + - oc.docker-build-debian-dependencies.merge dependencies: [] timeout: 60 minutes before_script: @@ -455,8 +492,8 @@ oc.build-data_packages: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian RELEASE: trixie TAGS: gcp @@ -518,8 +555,8 @@ oc.docker-build-ubuntu-dependencies: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: ubuntu BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/${DISTRIBUTION}:${RELEASE} parallel: @@ -532,7 +569,7 @@ oc.docker-build-ubuntu-dependencies: - gcp_arm64 oc.build-ubuntu: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -554,8 +591,8 @@ oc.build-ubuntu: - ./scripts/ci/sccache-stop.sh variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: ubuntu DUNE_BUILD_JOBS: -j 12 CARGO_INCREMENTAL: "0" diff --git a/.gitlab/ci/pipelines/octez_major_release_tag_test.yml b/.gitlab/ci/pipelines/octez_major_release_tag_test.yml index 71dfc68476f3..488b0ad98dd2 100644 --- a/.gitlab/ci/pipelines/octez_major_release_tag_test.yml +++ b/.gitlab/ci/pipelines/octez_major_release_tag_test.yml @@ -376,8 +376,8 @@ oc.docker-build-debian-dependencies: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/${DISTRIBUTION}:${RELEASE} parallel: @@ -389,8 +389,45 @@ oc.docker-build-debian-dependencies: - gcp_very_high_cpu - gcp_arm64 +oc.docker-build-debian-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-build-debian-dependencies + dependencies: [] + timeout: 60 minutes + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: debian + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + - gcp_arm64 + oc.build-debian: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -412,8 +449,8 @@ oc.build-debian: - ./scripts/ci/sccache-stop.sh variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian DUNE_BUILD_JOBS: -j 12 CARGO_INCREMENTAL: "0" @@ -435,12 +472,12 @@ oc.build-debian: - gcp_arm64 oc.build-data_packages: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS needs: - - oc.docker-build-debian-dependencies + - oc.docker-build-debian-dependencies.merge dependencies: [] timeout: 60 minutes before_script: @@ -455,8 +492,8 @@ oc.build-data_packages: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian RELEASE: trixie TAGS: gcp @@ -518,8 +555,8 @@ oc.docker-build-ubuntu-dependencies: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: ubuntu BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/${DISTRIBUTION}:${RELEASE} parallel: @@ -532,7 +569,7 @@ oc.docker-build-ubuntu-dependencies: - gcp_arm64 oc.build-ubuntu: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -554,8 +591,8 @@ oc.build-ubuntu: - ./scripts/ci/sccache-stop.sh variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: ubuntu DUNE_BUILD_JOBS: -j 12 CARGO_INCREMENTAL: "0" diff --git a/.gitlab/ci/pipelines/octez_minor_release_tag.yml b/.gitlab/ci/pipelines/octez_minor_release_tag.yml index c7849056c78b..1d5db87305b6 100644 --- a/.gitlab/ci/pipelines/octez_minor_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_minor_release_tag.yml @@ -350,8 +350,8 @@ oc.docker-build-debian-dependencies: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/${DISTRIBUTION}:${RELEASE} parallel: @@ -363,8 +363,45 @@ oc.docker-build-debian-dependencies: - gcp_very_high_cpu - gcp_arm64 +oc.docker-build-debian-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-build-debian-dependencies + dependencies: [] + timeout: 60 minutes + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: debian + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + - gcp_arm64 + oc.build-debian: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -386,8 +423,8 @@ oc.build-debian: - ./scripts/ci/sccache-stop.sh variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian DUNE_BUILD_JOBS: -j 12 CARGO_INCREMENTAL: "0" @@ -409,12 +446,12 @@ oc.build-debian: - gcp_arm64 oc.build-data_packages: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS needs: - - oc.docker-build-debian-dependencies + - oc.docker-build-debian-dependencies.merge dependencies: [] timeout: 60 minutes before_script: @@ -429,8 +466,8 @@ oc.build-data_packages: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian RELEASE: trixie TAGS: gcp @@ -492,8 +529,8 @@ oc.docker-build-ubuntu-dependencies: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: ubuntu BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/${DISTRIBUTION}:${RELEASE} parallel: @@ -506,7 +543,7 @@ oc.docker-build-ubuntu-dependencies: - gcp_arm64 oc.build-ubuntu: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -528,8 +565,8 @@ oc.build-ubuntu: - ./scripts/ci/sccache-stop.sh variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: ubuntu DUNE_BUILD_JOBS: -j 12 CARGO_INCREMENTAL: "0" diff --git a/.gitlab/ci/pipelines/octez_minor_release_tag_test.yml b/.gitlab/ci/pipelines/octez_minor_release_tag_test.yml index 54c7c418c43e..80eba5ed0c9c 100644 --- a/.gitlab/ci/pipelines/octez_minor_release_tag_test.yml +++ b/.gitlab/ci/pipelines/octez_minor_release_tag_test.yml @@ -350,8 +350,8 @@ oc.docker-build-debian-dependencies: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/${DISTRIBUTION}:${RELEASE} parallel: @@ -363,8 +363,45 @@ oc.docker-build-debian-dependencies: - gcp_very_high_cpu - gcp_arm64 +oc.docker-build-debian-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-build-debian-dependencies + dependencies: [] + timeout: 60 minutes + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: debian + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + - gcp_arm64 + oc.build-debian: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -386,8 +423,8 @@ oc.build-debian: - ./scripts/ci/sccache-stop.sh variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian DUNE_BUILD_JOBS: -j 12 CARGO_INCREMENTAL: "0" @@ -409,12 +446,12 @@ oc.build-debian: - gcp_arm64 oc.build-data_packages: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS needs: - - oc.docker-build-debian-dependencies + - oc.docker-build-debian-dependencies.merge dependencies: [] timeout: 60 minutes before_script: @@ -429,8 +466,8 @@ oc.build-data_packages: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian RELEASE: trixie TAGS: gcp @@ -492,8 +529,8 @@ oc.docker-build-ubuntu-dependencies: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: ubuntu BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/${DISTRIBUTION}:${RELEASE} parallel: @@ -506,7 +543,7 @@ oc.docker-build-ubuntu-dependencies: - gcp_arm64 oc.build-ubuntu: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -528,8 +565,8 @@ oc.build-ubuntu: - ./scripts/ci/sccache-stop.sh variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: ubuntu DUNE_BUILD_JOBS: -j 12 CARGO_INCREMENTAL: "0" diff --git a/.gitlab/ci/pipelines/schedule_master_test_release.yml b/.gitlab/ci/pipelines/schedule_master_test_release.yml index f558fea93011..d1b04bfb17ce 100644 --- a/.gitlab/ci/pipelines/schedule_master_test_release.yml +++ b/.gitlab/ci/pipelines/schedule_master_test_release.yml @@ -377,8 +377,8 @@ oc.docker-build-debian-dependencies: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/${DISTRIBUTION}:${RELEASE} parallel: @@ -390,8 +390,45 @@ oc.docker-build-debian-dependencies: - gcp_very_high_cpu - gcp_arm64 +oc.docker-build-debian-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-build-debian-dependencies + dependencies: [] + timeout: 60 minutes + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: debian + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + - gcp_arm64 + oc.build-debian: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -413,8 +450,8 @@ oc.build-debian: - ./scripts/ci/sccache-stop.sh variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian DUNE_BUILD_JOBS: -j 12 CARGO_INCREMENTAL: "0" @@ -436,12 +473,12 @@ oc.build-debian: - gcp_arm64 oc.build-data_packages: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS needs: - - oc.docker-build-debian-dependencies + - oc.docker-build-debian-dependencies.merge dependencies: [] timeout: 60 minutes before_script: @@ -456,8 +493,8 @@ oc.build-data_packages: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: debian RELEASE: trixie TAGS: gcp @@ -519,8 +556,8 @@ oc.docker-build-ubuntu-dependencies: variables: DOCKER_VERSION: 24.0.7 FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: ubuntu BASE_IMAGE: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/${DISTRIBUTION}:${RELEASE} parallel: @@ -533,7 +570,7 @@ oc.docker-build-ubuntu-dependencies: - gcp_arm64 oc.build-ubuntu: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -555,8 +592,8 @@ oc.build-ubuntu: - ./scripts/ci/sccache-stop.sh variables: FLAVOUR: build - DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE - DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION DISTRIBUTION: ubuntu DUNE_BUILD_JOBS: -j 12 CARGO_INCREMENTAL: "0" diff --git a/ci/lib_tezos_ci_jobs/debian_repository.ml b/ci/lib_tezos_ci_jobs/debian_repository.ml index d3027ea3006e..6f72507ef3d7 100644 --- a/ci/lib_tezos_ci_jobs/debian_repository.ml +++ b/ci/lib_tezos_ci_jobs/debian_repository.ml @@ -20,7 +20,8 @@ open Common.Helpers let build_debian_packages_image = Image.mk_external - ~image_path:"$DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" + ~image_path: + "$DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" (* the name of the image depends on the variables in the parallel matrix. we delcare it once, but we use it in two different contexts *) @@ -105,13 +106,11 @@ let jobs ?(limit_dune_build_jobs = false) pipeline_type = let variables ?(kind = "build") add = ("FLAVOUR", kind) :: ( "DEP_IMAGE", - "${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE" - ) + "${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION" ) (* this second variable is for a read only registry and we want it to be tezos/tezos *) :: ( "DEP_IMAGE_PROTECTED", - "${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION-$RELEASE" - ) + "${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION" ) :: add in let make_job_docker_systemd_tests ~__POS__ ~name ~matrix ~distribution = @@ -222,6 +221,17 @@ let jobs ?(limit_dune_build_jobs = false) pipeline_type = ] |> enable_sccache in + let job_merge_debian_build_deps = + job_docker_authenticated + ~__POS__ + ~name:"oc.docker-build-debian-dependencies.merge" + ~stage:Stages.images + ~dependencies:(Dependent [Job job_docker_build_debian_dependencies]) + ~variables: + (variables [("DISTRIBUTION", "debian"); ("IMAGE_NAME", "$DEP_IMAGE")]) + ~parallel:(Matrix (debian_package_release_matrix pipeline_type)) + ["scripts/ci/docker-merge-base-images.sh"] + in (* data packages. we build them once *) let job_build_data_packages : tezos_job = @@ -233,7 +243,7 @@ let jobs ?(limit_dune_build_jobs = false) pipeline_type = ~variables: (variables [("DISTRIBUTION", "debian"); ("RELEASE", "trixie"); ("TAGS", "gcp")]) - ~dependencies:(Dependent [Job job_docker_build_debian_dependencies]) + ~dependencies:(Dependent [Job job_merge_debian_build_deps]) ~tag:Dynamic ~artifacts:(artifacts ["packages/$DISTRIBUTION/$RELEASE"]) [ @@ -492,6 +502,7 @@ let jobs ?(limit_dune_build_jobs = false) pipeline_type = let debian_jobs = [ job_docker_build_debian_dependencies; + job_merge_debian_build_deps; job_build_debian_package; job_build_data_packages; job_apt_repo_debian; diff --git a/scripts/ci/build-packages-dependencies.sh b/scripts/ci/build-packages-dependencies.sh index 18cc6465bc7a..20729ce1b7f7 100755 --- a/scripts/ci/build-packages-dependencies.sh +++ b/scripts/ci/build-packages-dependencies.sh @@ -25,7 +25,10 @@ gcp | gcp_dev | gcp_high_cpu | gcp_very_high_cpu | gcp_very_high_cpu_ramfs | gcp ;; esac -LOCAL_IMAGE_NAME="$DEP_IMAGE:${ARCHITECTURE}-${CI_COMMIT_REF_SLUG}" +ARCH_IMAGE_PREFIX="$DEP_IMAGE" +ARCH_IAMGE_TAG="${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}-${ARCHITECTURE}" +ARCH_IMAGE_NAME="$ARCH_IMAGE_PREFIX:$ARCH_IAMGE_TAG" + # Set as a variable in the job definition BASE_IMAGE="${BASE_IMAGE:?Must set base image to run this script}" @@ -52,10 +55,12 @@ docker build \ --build-arg OPAM_VERSION="${opam_version}" \ --cache-from="${DEP_IMAGE}:${ARCHITECTURE}-${CI_COMMIT_REF_SLUG}" \ --cache-from="${DEP_IMAGE_PROTECTED}:master" \ - -t "$LOCAL_IMAGE_NAME" \ + -t "$ARCH_IMAGE_NAME" \ $(if [ "$skip_registry_cache_check" = "true" ]; then echo "--no-cache"; fi) \ . +exit 0 + # The LATEST_TAG is used to compile the images in the CI pipelines # where we want to make sure to always have the latest available # image ( we use IfNotPresent as pull policy ) diff --git a/scripts/ci/docker-merge-base-images.sh b/scripts/ci/docker-merge-base-images.sh index d772cf9274c3..95081642c647 100755 --- a/scripts/ci/docker-merge-base-images.sh +++ b/scripts/ci/docker-merge-base-images.sh @@ -26,8 +26,8 @@ set -euo pipefail # Assumes IMAGE_NAME, LATEST_TAG, and platform-specific images are already # set and pushed. -LATEST_TAG="${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" -LATEST_TAG_GENERIC="${RELEASE}-${CI_COMMIT_REF_SLUG}" +LATEST_TAG="${RELEASE}-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA" +LATEST_TAG_GENERIC="$RELEASE-$CI_COMMIT_REF_SLUG" create_and_push_manifest() { local TAG="$1" @@ -35,18 +35,30 @@ create_and_push_manifest() { echo "Inspect ${IMAGE_NAME}:${LATEST_TAG}-amd64" docker buildx imagetools inspect "${IMAGE_NAME}:${LATEST_TAG}-amd64" + set +e # Disable exit on error temporarily + echo "Inspect ${IMAGE_NAME}:${LATEST_TAG}-arm64" docker buildx imagetools inspect "${IMAGE_NAME}:${LATEST_TAG}-arm64" + ARM64_INSPECT=$? + + set -e # Re-enable exit on error echo "📦 Creating multi-arch manifest for tag: ${TAG}" - docker manifest create "${IMAGE_NAME}:${TAG}" \ - --amend "${IMAGE_NAME}:${LATEST_TAG}-amd64" \ - --amend "${IMAGE_NAME}:${LATEST_TAG}-arm64" + echo "Adding ${IMAGE_NAME}:${LATEST_TAG}-amd64 to the manifest." + docker manifest create "${IMAGE_NAME}:${TAG}" --amend "${IMAGE_NAME}:${LATEST_TAG}-amd64" + # Amend the manifest if the arm64 image exists + if [ $ARM64_INSPECT -eq 0 ]; then + echo "Adding ${IMAGE_NAME}:${LATEST_TAG}-arm64 to the manifest." + docker manifest amend "${IMAGE_NAME}:${TAG}" "${IMAGE_NAME}:${LATEST_TAG}-arm64" + fi + + # Push the manifest docker manifest push "${IMAGE_NAME}:${TAG}" echo "Pushed manifest: ${IMAGE_NAME}:${TAG}" + } create_and_push_manifest "${LATEST_TAG}" -- GitLab From baf6c0bd09f12c0ceeee1e28a534d4924b8d4cc1 Mon Sep 17 00:00:00 2001 From: Pietro Abate Date: Fri, 12 Dec 2025 14:54:44 +0100 Subject: [PATCH 2/2] packages: add merge manifest job for build images --- .gitlab/ci/pipelines/debian.daily.yml | 133 ++++++++++++++++-- .../pipelines/debian_repository_partial.yml | 44 +++++- .../debian_repository_partial_auto.yml | 44 +++++- .gitlab/ci/pipelines/non_release_tag.yml | 43 +++++- .gitlab/ci/pipelines/non_release_tag_test.yml | 43 +++++- .../ci/pipelines/octez_beta_release_tag.yml | 43 +++++- .../pipelines/octez_beta_release_tag_test.yml | 43 +++++- .../ci/pipelines/octez_major_release_tag.yml | 43 +++++- .../octez_major_release_tag_test.yml | 43 +++++- .../ci/pipelines/octez_minor_release_tag.yml | 43 +++++- .../octez_minor_release_tag_test.yml | 43 +++++- .../schedule_master_test_release.yml | 43 +++++- ci/lib_tezos_ci_jobs/debian_repository.ml | 67 +++++++-- scripts/ci/build-packages-dependencies.sh | 51 +------ scripts/ci/docker-merge-base-images.sh | 11 +- .../packaging/tests/systemd-docker-test.sh | 2 +- 16 files changed, 632 insertions(+), 107 deletions(-) diff --git a/.gitlab/ci/pipelines/debian.daily.yml b/.gitlab/ci/pipelines/debian.daily.yml index b3f2cd3ef276..735be576653f 100644 --- a/.gitlab/ci/pipelines/debian.daily.yml +++ b/.gitlab/ci/pipelines/debian.daily.yml @@ -103,6 +103,84 @@ oc.docker-systemd_tests_ubuntu: - gcp_very_high_cpu - gcp_arm64 +oc.docker-systemd-test-debian-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-systemd_tests_debian + dependencies: [] + timeout: 60 minutes + interruptible: false + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: systemd-tests + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: debian + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - unstable + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + - gcp_arm64 + +oc.docker-systemd-test-ubuntu-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-systemd_tests_ubuntu + dependencies: [] + timeout: 60 minutes + interruptible: false + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: ubuntu + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - unstable + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + - gcp_arm64 + oc.docker-build-debian-dependencies: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 stage: images @@ -180,7 +258,7 @@ oc.docker-build-debian-dependencies.merge: - gcp_arm64 oc.build-debian: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -227,7 +305,7 @@ oc.build-debian: - gcp_arm64 oc.build-data_packages: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -326,8 +404,47 @@ oc.docker-build-ubuntu-dependencies: - gcp_very_high_cpu - gcp_arm64 +oc.docker-build-ubuntu-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-build-debian-dependencies + dependencies: [] + timeout: 60 minutes + interruptible: false + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: ubuntu + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - unstable + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + - gcp_arm64 + oc.build-ubuntu: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -459,7 +576,7 @@ oc.install_bin_debian_bookworm_systemd_test: tags: - gcp needs: - - oc.docker-systemd_tests_debian + - oc.docker-systemd-test-debian-dependencies.merge - apt_repo_debian dependencies: [] allow_failure: true @@ -493,7 +610,7 @@ oc.upgrade_bin_debian_bookworm-systemd: tags: - gcp needs: - - oc.docker-systemd_tests_debian + - oc.docker-systemd-test-debian-dependencies.merge - apt_repo_debian dependencies: [] allow_failure: true @@ -597,7 +714,7 @@ oc.install_bin_ubuntu_noble_systemd: tags: - gcp needs: - - oc.docker-systemd_tests_ubuntu + - oc.docker-systemd-test-ubuntu-dependencies.merge - apt_repo_ubuntu dependencies: [] allow_failure: true @@ -631,7 +748,7 @@ oc.upgrade_bin_ubuntu_jammy_systemd_test: tags: - gcp needs: - - oc.docker-systemd_tests_ubuntu + - oc.docker-systemd-test-ubuntu-dependencies.merge - apt_repo_ubuntu dependencies: [] allow_failure: true @@ -665,7 +782,7 @@ oc.upgrade_bin_ubuntu_noble_systemd_test: tags: - gcp needs: - - oc.docker-systemd_tests_ubuntu + - oc.docker-systemd-test-ubuntu-dependencies.merge - apt_repo_ubuntu dependencies: [] allow_failure: true diff --git a/.gitlab/ci/pipelines/debian_repository_partial.yml b/.gitlab/ci/pipelines/debian_repository_partial.yml index b552982239ba..14d0ead99c87 100644 --- a/.gitlab/ci/pipelines/debian_repository_partial.yml +++ b/.gitlab/ci/pipelines/debian_repository_partial.yml @@ -79,6 +79,42 @@ oc.docker-systemd_tests_debian: TAGS: - gcp_very_high_cpu +oc.docker-systemd-test-debian-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-systemd_tests_debian + dependencies: [] + timeout: 60 minutes + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: systemd-tests + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: debian + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + oc.docker-build-debian-dependencies: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 stage: images @@ -150,7 +186,7 @@ oc.docker-build-debian-dependencies.merge: - gcp_very_high_cpu oc.build-debian: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -193,7 +229,7 @@ oc.build-debian: - gcp_very_high_cpu_ramfs oc.build-data_packages: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -306,7 +342,7 @@ oc.install_bin_debian_bookworm_systemd_test: tags: - gcp needs: - - oc.docker-systemd_tests_debian + - oc.docker-systemd-test-debian-dependencies.merge - apt_repo_debian dependencies: [] allow_failure: true @@ -339,7 +375,7 @@ oc.upgrade_bin_debian_bookworm-systemd: tags: - gcp needs: - - oc.docker-systemd_tests_debian + - oc.docker-systemd-test-debian-dependencies.merge - apt_repo_debian dependencies: [] allow_failure: true diff --git a/.gitlab/ci/pipelines/debian_repository_partial_auto.yml b/.gitlab/ci/pipelines/debian_repository_partial_auto.yml index b552982239ba..14d0ead99c87 100644 --- a/.gitlab/ci/pipelines/debian_repository_partial_auto.yml +++ b/.gitlab/ci/pipelines/debian_repository_partial_auto.yml @@ -79,6 +79,42 @@ oc.docker-systemd_tests_debian: TAGS: - gcp_very_high_cpu +oc.docker-systemd-test-debian-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-systemd_tests_debian + dependencies: [] + timeout: 60 minutes + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: systemd-tests + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: debian + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + oc.docker-build-debian-dependencies: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 stage: images @@ -150,7 +186,7 @@ oc.docker-build-debian-dependencies.merge: - gcp_very_high_cpu oc.build-debian: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -193,7 +229,7 @@ oc.build-debian: - gcp_very_high_cpu_ramfs oc.build-data_packages: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -306,7 +342,7 @@ oc.install_bin_debian_bookworm_systemd_test: tags: - gcp needs: - - oc.docker-systemd_tests_debian + - oc.docker-systemd-test-debian-dependencies.merge - apt_repo_debian dependencies: [] allow_failure: true @@ -339,7 +375,7 @@ oc.upgrade_bin_debian_bookworm-systemd: tags: - gcp needs: - - oc.docker-systemd_tests_debian + - oc.docker-systemd-test-debian-dependencies.merge - apt_repo_debian dependencies: [] allow_failure: true diff --git a/.gitlab/ci/pipelines/non_release_tag.yml b/.gitlab/ci/pipelines/non_release_tag.yml index 8274341f0709..a24908ac9a57 100644 --- a/.gitlab/ci/pipelines/non_release_tag.yml +++ b/.gitlab/ci/pipelines/non_release_tag.yml @@ -425,7 +425,7 @@ oc.docker-build-debian-dependencies.merge: - gcp_arm64 oc.build-debian: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -470,7 +470,7 @@ oc.build-debian: - gcp_arm64 oc.build-data_packages: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -566,8 +566,45 @@ oc.docker-build-ubuntu-dependencies: - gcp_very_high_cpu - gcp_arm64 +oc.docker-build-ubuntu-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-build-debian-dependencies + dependencies: [] + timeout: 60 minutes + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: ubuntu + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + - gcp_arm64 + oc.build-ubuntu: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS diff --git a/.gitlab/ci/pipelines/non_release_tag_test.yml b/.gitlab/ci/pipelines/non_release_tag_test.yml index 3b04132731c6..7a0bc7444341 100644 --- a/.gitlab/ci/pipelines/non_release_tag_test.yml +++ b/.gitlab/ci/pipelines/non_release_tag_test.yml @@ -425,7 +425,7 @@ oc.docker-build-debian-dependencies.merge: - gcp_arm64 oc.build-debian: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -470,7 +470,7 @@ oc.build-debian: - gcp_arm64 oc.build-data_packages: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -566,8 +566,45 @@ oc.docker-build-ubuntu-dependencies: - gcp_very_high_cpu - gcp_arm64 +oc.docker-build-ubuntu-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-build-debian-dependencies + dependencies: [] + timeout: 60 minutes + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: ubuntu + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + - gcp_arm64 + oc.build-ubuntu: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS diff --git a/.gitlab/ci/pipelines/octez_beta_release_tag.yml b/.gitlab/ci/pipelines/octez_beta_release_tag.yml index eee3ba4b5b73..1f1d9d155954 100644 --- a/.gitlab/ci/pipelines/octez_beta_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_beta_release_tag.yml @@ -427,7 +427,7 @@ oc.docker-build-debian-dependencies.merge: - gcp_arm64 oc.build-debian: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -472,7 +472,7 @@ oc.build-debian: - gcp_arm64 oc.build-data_packages: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -568,8 +568,45 @@ oc.docker-build-ubuntu-dependencies: - gcp_very_high_cpu - gcp_arm64 +oc.docker-build-ubuntu-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-build-debian-dependencies + dependencies: [] + timeout: 60 minutes + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: ubuntu + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + - gcp_arm64 + oc.build-ubuntu: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS diff --git a/.gitlab/ci/pipelines/octez_beta_release_tag_test.yml b/.gitlab/ci/pipelines/octez_beta_release_tag_test.yml index 726687adaf37..8023f38c79b5 100644 --- a/.gitlab/ci/pipelines/octez_beta_release_tag_test.yml +++ b/.gitlab/ci/pipelines/octez_beta_release_tag_test.yml @@ -427,7 +427,7 @@ oc.docker-build-debian-dependencies.merge: - gcp_arm64 oc.build-debian: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -472,7 +472,7 @@ oc.build-debian: - gcp_arm64 oc.build-data_packages: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -568,8 +568,45 @@ oc.docker-build-ubuntu-dependencies: - gcp_very_high_cpu - gcp_arm64 +oc.docker-build-ubuntu-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-build-debian-dependencies + dependencies: [] + timeout: 60 minutes + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: ubuntu + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + - gcp_arm64 + oc.build-ubuntu: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS diff --git a/.gitlab/ci/pipelines/octez_major_release_tag.yml b/.gitlab/ci/pipelines/octez_major_release_tag.yml index 93059cbcaa3d..64cabbb36f58 100644 --- a/.gitlab/ci/pipelines/octez_major_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_major_release_tag.yml @@ -427,7 +427,7 @@ oc.docker-build-debian-dependencies.merge: - gcp_arm64 oc.build-debian: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -472,7 +472,7 @@ oc.build-debian: - gcp_arm64 oc.build-data_packages: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -568,8 +568,45 @@ oc.docker-build-ubuntu-dependencies: - gcp_very_high_cpu - gcp_arm64 +oc.docker-build-ubuntu-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-build-debian-dependencies + dependencies: [] + timeout: 60 minutes + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: ubuntu + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + - gcp_arm64 + oc.build-ubuntu: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS diff --git a/.gitlab/ci/pipelines/octez_major_release_tag_test.yml b/.gitlab/ci/pipelines/octez_major_release_tag_test.yml index 488b0ad98dd2..31f0cf123974 100644 --- a/.gitlab/ci/pipelines/octez_major_release_tag_test.yml +++ b/.gitlab/ci/pipelines/octez_major_release_tag_test.yml @@ -427,7 +427,7 @@ oc.docker-build-debian-dependencies.merge: - gcp_arm64 oc.build-debian: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -472,7 +472,7 @@ oc.build-debian: - gcp_arm64 oc.build-data_packages: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -568,8 +568,45 @@ oc.docker-build-ubuntu-dependencies: - gcp_very_high_cpu - gcp_arm64 +oc.docker-build-ubuntu-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-build-debian-dependencies + dependencies: [] + timeout: 60 minutes + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: ubuntu + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + - gcp_arm64 + oc.build-ubuntu: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS diff --git a/.gitlab/ci/pipelines/octez_minor_release_tag.yml b/.gitlab/ci/pipelines/octez_minor_release_tag.yml index 1d5db87305b6..47d05d1910a1 100644 --- a/.gitlab/ci/pipelines/octez_minor_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_minor_release_tag.yml @@ -401,7 +401,7 @@ oc.docker-build-debian-dependencies.merge: - gcp_arm64 oc.build-debian: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -446,7 +446,7 @@ oc.build-debian: - gcp_arm64 oc.build-data_packages: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -542,8 +542,45 @@ oc.docker-build-ubuntu-dependencies: - gcp_very_high_cpu - gcp_arm64 +oc.docker-build-ubuntu-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-build-debian-dependencies + dependencies: [] + timeout: 60 minutes + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: ubuntu + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + - gcp_arm64 + oc.build-ubuntu: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS diff --git a/.gitlab/ci/pipelines/octez_minor_release_tag_test.yml b/.gitlab/ci/pipelines/octez_minor_release_tag_test.yml index 80eba5ed0c9c..797fa82fad37 100644 --- a/.gitlab/ci/pipelines/octez_minor_release_tag_test.yml +++ b/.gitlab/ci/pipelines/octez_minor_release_tag_test.yml @@ -401,7 +401,7 @@ oc.docker-build-debian-dependencies.merge: - gcp_arm64 oc.build-debian: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -446,7 +446,7 @@ oc.build-debian: - gcp_arm64 oc.build-data_packages: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -542,8 +542,45 @@ oc.docker-build-ubuntu-dependencies: - gcp_very_high_cpu - gcp_arm64 +oc.docker-build-ubuntu-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-build-debian-dependencies + dependencies: [] + timeout: 60 minutes + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: ubuntu + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + - gcp_arm64 + oc.build-ubuntu: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS diff --git a/.gitlab/ci/pipelines/schedule_master_test_release.yml b/.gitlab/ci/pipelines/schedule_master_test_release.yml index d1b04bfb17ce..e0f8282f43b3 100644 --- a/.gitlab/ci/pipelines/schedule_master_test_release.yml +++ b/.gitlab/ci/pipelines/schedule_master_test_release.yml @@ -428,7 +428,7 @@ oc.docker-build-debian-dependencies.merge: - gcp_arm64 oc.build-debian: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -473,7 +473,7 @@ oc.build-debian: - gcp_arm64 oc.build-data_packages: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS @@ -569,8 +569,45 @@ oc.docker-build-ubuntu-dependencies: - gcp_very_high_cpu - gcp_arm64 +oc.docker-build-ubuntu-dependencies.merge: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + needs: + - oc.docker-build-debian-dependencies + dependencies: [] + timeout: 60 minutes + before_script: + - SCRIPT_STEP_BEGIN=$(date +%s) + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - scripts/ci/docker-merge-base-images.sh + - . ./scripts/ci/datadog_send_job_script_step_time.sh || true + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + FLAVOUR: build + DEP_IMAGE: ${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$FLAVOUR-$DISTRIBUTION + DEP_IMAGE_PROTECTED: ${GCP_PROTECTED_REGISTRY}/tezos/tezos/$FLAVOUR-$DISTRIBUTION + DISTRIBUTION: ubuntu + IMAGE_NAME: $DEP_IMAGE + parallel: + matrix: + - RELEASE: + - bookworm + - trixie + TAGS: + - gcp_very_high_cpu + - gcp_arm64 + oc.build-ubuntu: - image: $DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} + image: ${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} stage: build tags: - $TAGS diff --git a/ci/lib_tezos_ci_jobs/debian_repository.ml b/ci/lib_tezos_ci_jobs/debian_repository.ml index 6f72507ef3d7..020143c08632 100644 --- a/ci/lib_tezos_ci_jobs/debian_repository.ml +++ b/ci/lib_tezos_ci_jobs/debian_repository.ml @@ -21,7 +21,7 @@ open Common.Helpers let build_debian_packages_image = Image.mk_external ~image_path: - "$DEP_IMAGE:$RELEASE-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" + "${DEP_IMAGE}:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" (* the name of the image depends on the variables in the parallel matrix. we delcare it once, but we use it in two different contexts *) @@ -147,6 +147,32 @@ let jobs ?(limit_dune_build_jobs = false) pipeline_type = ~distribution:"ubuntu" ~matrix:(ubuntu_package_release_matrix pipeline_type) in + let job_merge_systemd_test_debian_dependencies = + job_docker_authenticated + ~__POS__ + ~name:"oc.docker-systemd-test-debian-dependencies.merge" + ~stage:Stages.images + ~dependencies: + (Dependent [Job job_docker_systemd_test_debian_dependencies]) + ~variables: + (variables + ~kind:"systemd-tests" + [("DISTRIBUTION", "debian"); ("IMAGE_NAME", "$DEP_IMAGE")]) + ~parallel:(Matrix (debian_package_release_matrix pipeline_type)) + ["scripts/ci/docker-merge-base-images.sh"] + in + let job_merge_systemd_test_ubuntu_dependencies = + job_docker_authenticated + ~__POS__ + ~name:"oc.docker-systemd-test-ubuntu-dependencies.merge" + ~stage:Stages.images + ~dependencies: + (Dependent [Job job_docker_systemd_test_ubuntu_dependencies]) + ~variables: + (variables [("DISTRIBUTION", "ubuntu"); ("IMAGE_NAME", "$DEP_IMAGE")]) + ~parallel:(Matrix (debian_package_release_matrix pipeline_type)) + ["scripts/ci/docker-merge-base-images.sh"] + in let make_job_docker_build_debian_dependencies ~__POS__ ~name ~matrix ~distribution = @@ -232,6 +258,17 @@ let jobs ?(limit_dune_build_jobs = false) pipeline_type = ~parallel:(Matrix (debian_package_release_matrix pipeline_type)) ["scripts/ci/docker-merge-base-images.sh"] in + let job_merge_ubuntu_build_deps = + job_docker_authenticated + ~__POS__ + ~name:"oc.docker-build-ubuntu-dependencies.merge" + ~stage:Stages.images + ~dependencies:(Dependent [Job job_docker_build_debian_dependencies]) + ~variables: + (variables [("DISTRIBUTION", "ubuntu"); ("IMAGE_NAME", "$DEP_IMAGE")]) + ~parallel:(Matrix (debian_package_release_matrix pipeline_type)) + ["scripts/ci/docker-merge-base-images.sh"] + in (* data packages. we build them once *) let job_build_data_packages : tezos_job = @@ -384,7 +421,7 @@ let jobs ?(limit_dune_build_jobs = false) pipeline_type = ~dependencies: (Dependent [ - Job job_docker_systemd_test_ubuntu_dependencies; + Job job_merge_systemd_test_ubuntu_dependencies; Job job_apt_repo_ubuntu; ]) ~variables: @@ -403,7 +440,7 @@ let jobs ?(limit_dune_build_jobs = false) pipeline_type = ~dependencies: (Dependent [ - Job job_docker_systemd_test_ubuntu_dependencies; + Job job_merge_systemd_test_ubuntu_dependencies; Job job_apt_repo_ubuntu; ]) ~variables: @@ -422,7 +459,7 @@ let jobs ?(limit_dune_build_jobs = false) pipeline_type = ~dependencies: (Dependent [ - Job job_docker_systemd_test_ubuntu_dependencies; + Job job_merge_systemd_test_ubuntu_dependencies; Job job_apt_repo_ubuntu; ]) ~variables: @@ -458,7 +495,7 @@ let jobs ?(limit_dune_build_jobs = false) pipeline_type = ~dependencies: (Dependent [ - Job job_docker_systemd_test_debian_dependencies; + Job job_merge_systemd_test_debian_dependencies; Job job_apt_repo_debian; ]) ~variables: @@ -481,7 +518,7 @@ let jobs ?(limit_dune_build_jobs = false) pipeline_type = ~dependencies: (Dependent [ - Job job_docker_systemd_test_debian_dependencies; + Job job_merge_systemd_test_debian_dependencies; Job job_apt_repo_debian; ]) ~variables: @@ -511,18 +548,28 @@ let jobs ?(limit_dune_build_jobs = false) pipeline_type = let ubuntu_jobs = [ job_docker_build_ubuntu_dependencies; + job_merge_ubuntu_build_deps; job_build_ubuntu_package; job_apt_repo_ubuntu; ] in match pipeline_type with | Partial -> - (job_docker_systemd_test_debian_dependencies :: debian_jobs) + ([ + job_docker_systemd_test_debian_dependencies; + job_merge_systemd_test_debian_dependencies; + ] + @ debian_jobs) @ test_debian_packages_jobs | Full -> - job_docker_systemd_test_debian_dependencies - :: job_docker_systemd_test_ubuntu_dependencies :: debian_jobs - @ ubuntu_jobs @ test_debian_packages_jobs @ test_ubuntu_packages_jobs + [ + job_docker_systemd_test_debian_dependencies; + job_docker_systemd_test_ubuntu_dependencies; + job_merge_systemd_test_debian_dependencies; + job_merge_systemd_test_ubuntu_dependencies; + ] + @ debian_jobs @ ubuntu_jobs @ test_debian_packages_jobs + @ test_ubuntu_packages_jobs | Release -> debian_jobs @ ubuntu_jobs let register ~auto ~description pipeline_type = diff --git a/scripts/ci/build-packages-dependencies.sh b/scripts/ci/build-packages-dependencies.sh index 20729ce1b7f7..63876349bbc7 100755 --- a/scripts/ci/build-packages-dependencies.sh +++ b/scripts/ci/build-packages-dependencies.sh @@ -25,9 +25,9 @@ gcp | gcp_dev | gcp_high_cpu | gcp_very_high_cpu | gcp_very_high_cpu_ramfs | gcp ;; esac -ARCH_IMAGE_PREFIX="$DEP_IMAGE" +ARCH_IMAGE_PREFIX="${DEP_IMAGE}" ARCH_IAMGE_TAG="${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}-${ARCHITECTURE}" -ARCH_IMAGE_NAME="$ARCH_IMAGE_PREFIX:$ARCH_IAMGE_TAG" +ARCH_IMAGE_NAME="${ARCH_IMAGE_PREFIX}:${ARCH_IAMGE_TAG}" # Set as a variable in the job definition BASE_IMAGE="${BASE_IMAGE:?Must set base image to run this script}" @@ -58,50 +58,3 @@ docker build \ -t "$ARCH_IMAGE_NAME" \ $(if [ "$skip_registry_cache_check" = "true" ]; then echo "--no-cache"; fi) \ . - -exit 0 - -# The LATEST_TAG is used to compile the images in the CI pipelines -# where we want to make sure to always have the latest available -# image ( we use IfNotPresent as pull policy ) -# The LATEST_TAG_GENERIC tag is used for the docker cache and -# general use where we can control the pull policy -LATEST_TAG="${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" -LATEST_TAG_GENERIC="${CI_COMMIT_REF_SLUG}" - -if [ "$skip_registry_cache_check" != "true" ]; then - echo "Checking for existence of image $DEP_IMAGE:$LATEST_TAG" - docker buildx imagetools inspect "$DEP_IMAGE:$LATEST_TAG" || export IMAGE_EXISTS="false" -else - export IMAGE_EXISTS="false" - echo "Force rebuild of CI images: we use the locally build image for the manifest" -fi - -if [ "$IMAGE_EXISTS" = "false" ]; then - echo "Creating image manifesto for $DEP_IMAGE:$LATEST_TAG" - docker buildx imagetools create -t "$DEP_IMAGE:$LATEST_TAG" "$LOCAL_IMAGE_NAME" -else - echo "Appending to image manifesto for $DEP_IMAGE:$LATEST_TAG with new ${ARCHITECTURE} image" - - # we get the sha of the image of the "other" architecture - OTHER_SHA=$(docker manifest inspect "${DEP_IMAGE}:$LATEST_TAG" | jq -r ".manifests[] | select(.platform.architecture != \"$ARCHITECTURE\") | .digest" | head -n1) - - if [ "${OTHER_SHA}" != "" ]; then - # we create a new manifest ( override the old one ) with both the - # local image and the image that was in the old manifest - docker buildx imagetools create \ - -t "${DEP_IMAGE}:$LATEST_TAG" \ - -t "${DEP_IMAGE}:$LATEST_TAG_GENERIC" \ - "$LOCAL_IMAGE_NAME" \ - "$DEP_IMAGE@$OTHER_SHA" - else - # we associate to "$DEP_IMAGE:$LATEST_TAG" the latest image with tag - # "$LOCAL_IMAGE_NAME" and replace the old one. - docker buildx imagetools create \ - -t "$DEP_IMAGE:$LATEST_TAG" \ - -t "$DEP_IMAGE:$LATEST_TAG_GENERIC" \ - "$LOCAL_IMAGE_NAME" - fi -fi - -docker buildx imagetools inspect "${DEP_IMAGE}:${LATEST_TAG}" diff --git a/scripts/ci/docker-merge-base-images.sh b/scripts/ci/docker-merge-base-images.sh index 95081642c647..aa43889c3780 100755 --- a/scripts/ci/docker-merge-base-images.sh +++ b/scripts/ci/docker-merge-base-images.sh @@ -26,11 +26,12 @@ set -euo pipefail # Assumes IMAGE_NAME, LATEST_TAG, and platform-specific images are already # set and pushed. -LATEST_TAG="${RELEASE}-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA" -LATEST_TAG_GENERIC="$RELEASE-$CI_COMMIT_REF_SLUG" +LATEST_TAG="${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" +LATEST_TAG_GENERIC="${RELEASE}-${CI_COMMIT_REF_SLUG}" create_and_push_manifest() { local TAG="$1" + set -x echo "Inspect ${IMAGE_NAME}:${LATEST_TAG}-amd64" docker buildx imagetools inspect "${IMAGE_NAME}:${LATEST_TAG}-amd64" @@ -46,14 +47,16 @@ create_and_push_manifest() { echo "📦 Creating multi-arch manifest for tag: ${TAG}" echo "Adding ${IMAGE_NAME}:${LATEST_TAG}-amd64 to the manifest." - docker manifest create "${IMAGE_NAME}:${TAG}" --amend "${IMAGE_NAME}:${LATEST_TAG}-amd64" + AMEND="--amend ${IMAGE_NAME}:${LATEST_TAG}-amd64" # Amend the manifest if the arm64 image exists if [ $ARM64_INSPECT -eq 0 ]; then echo "Adding ${IMAGE_NAME}:${LATEST_TAG}-arm64 to the manifest." - docker manifest amend "${IMAGE_NAME}:${TAG}" "${IMAGE_NAME}:${LATEST_TAG}-arm64" + AMEND="$AMEND --amend ${IMAGE_NAME}:${LATEST_TAG}-arm64" fi + docker manifest create "${IMAGE_NAME}:${TAG}" ${AMEND} + # Push the manifest docker manifest push "${IMAGE_NAME}:${TAG}" diff --git a/scripts/packaging/tests/systemd-docker-test.sh b/scripts/packaging/tests/systemd-docker-test.sh index 8a33ce287af6..377abab30c48 100755 --- a/scripts/packaging/tests/systemd-docker-test.sh +++ b/scripts/packaging/tests/systemd-docker-test.sh @@ -20,7 +20,7 @@ GCP_LINUX_PACKAGES_BUCKET=${GCP_LINUX_PACKAGES_BUCKET:-tezos-linux-repo} #shellcheck disable=SC2236 if [ ! -z ${DEP_IMAGE+x} ]; then - IMAGE="$DEP_IMAGE:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" + IMAGE="$DEP_IMAGE:${RELEASE}-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" docker pull "$IMAGE" else # for local execution, we assume the image name is "systemd" -- GitLab