From 26402d15074c5812dbb88984696a91bf813aa6e1 Mon Sep 17 00:00:00 2001 From: Pietro Abate Date: Wed, 11 Jun 2025 11:25:11 +0200 Subject: [PATCH] images: Add base images for debian and ubuntu --- .gitlab/ci/pipelines/base_images.yml | 82 +++++++++++++++++++ .gitlab/ci/pipelines/before_merging.yml | 12 +++ .gitlab/ci/pipelines/merge_train.yml | 12 +++ .../ci/pipelines/schedule_extended_test.yml | 11 +++ ci/bin/base_images.ml | 49 +++++++++++ ci/bin/code_verification.ml | 36 +++++++- ci/bin/common.ml | 3 + images/base-images/Dockerfile.debian | 21 +++++ scripts/ci/build-base-images.sh | 44 ++++++++++ 9 files changed, 266 insertions(+), 4 deletions(-) create mode 100644 .gitlab/ci/pipelines/base_images.yml create mode 100644 ci/bin/base_images.ml create mode 100644 images/base-images/Dockerfile.debian create mode 100755 scripts/ci/build-base-images.sh diff --git a/.gitlab/ci/pipelines/base_images.yml b/.gitlab/ci/pipelines/base_images.yml new file mode 100644 index 000000000000..8f88a44f90cb --- /dev/null +++ b/.gitlab/ci/pipelines/base_images.yml @@ -0,0 +1,82 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +workflow: + rules: + - if: $foo != "bar" || $foo == "bar" + when: always + +stages: +- start +- images + +datadog_pipeline_trace: + image: datadog/ci:v2.44.0 + stage: start + tags: + - gcp + dependencies: [] + allow_failure: true + timeout: 60 minutes + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + script: + - CI_MERGE_REQUEST_IID=${CI_MERGE_REQUEST_IID:-none} + - DATADOG_SITE=datadoghq.eu datadog-ci tag --level pipeline --tags pipeline_type:$PIPELINE_TYPE + --tags mr_number:$CI_MERGE_REQUEST_IID + +oc.base-images.debian: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp_very_high_cpu + dependencies: [] + timeout: 60 minutes + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + script: + - scripts/ci/build-base-images.sh images/base-images/Dockerfile.debian + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + DISTRIBUTION: debian + retry: + max: 2 + when: + - stuck_or_timeout_failure + - runner_system_failure + parallel: + matrix: + - RELEASE: + - unstable + - bookworm + +oc.base-images.ubuntu: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp_very_high_cpu + dependencies: [] + timeout: 60 minutes + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/docker_initialize.sh + script: + - scripts/ci/build-base-images.sh images/base-images/Dockerfile.debian + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + DISTRIBUTION: ubuntu + retry: + max: 2 + when: + - stuck_or_timeout_failure + - runner_system_failure + parallel: + matrix: + - RELEASE: + - noble + - jammy diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 745d6d9a74a4..e1af00d905ba 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -4739,3 +4739,15 @@ oc.script.docker_verify_image_amd64: variables: DOCKER_VERSION: 24.0.7 IMAGE_ARCH_PREFIX: amd64_ + +trigger:base_images: + stage: manual + variables: + PIPELINE_TYPE: base_images + rules: + - when: manual + allow_failure: true + needs: [] + trigger: + include: .gitlab/ci/pipelines/base_images.yml + strategy: depend diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index a386dd2092de..b92b5173bc0c 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -4702,3 +4702,15 @@ oc.script.docker_verify_image_amd64: variables: DOCKER_VERSION: 24.0.7 IMAGE_ARCH_PREFIX: amd64_ + +trigger:base_images: + stage: manual + variables: + PIPELINE_TYPE: base_images + rules: + - when: manual + allow_failure: true + needs: [] + trigger: + include: .gitlab/ci/pipelines/base_images.yml + strategy: depend diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 7f2d757cf665..a1fec02d2a9d 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -1875,6 +1875,17 @@ trigger:homebrew: include: .gitlab/ci/pipelines/homebrew.yml strategy: depend +trigger:base_images: + stage: images + variables: + PIPELINE_TYPE: base_images + rules: + - when: always + needs: [] + trigger: + include: .gitlab/ci/pipelines/base_images.yml + strategy: depend + kaitai_checks: image: ${ci_image_name}/build:${ci_image_tag} stage: test diff --git a/ci/bin/base_images.ml b/ci/bin/base_images.ml new file mode 100644 index 000000000000..b6058d1fd0f2 --- /dev/null +++ b/ci/bin/base_images.ml @@ -0,0 +1,49 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2025 Nomadic Labs. *) +(* *) +(*****************************************************************************) + +open Gitlab_ci.Types +open Tezos_ci +open Common + +let debian_matrix = [[("RELEASE", ["unstable"; "bookworm"])]] + +let ubuntu_matrix = [[("RELEASE", ["noble"; "jammy"])]] + +let jobs = + let make_job_base_images ~__POS__ ~name ~matrix ~distribution = + job_docker_authenticated + ~__POS__ + ~name + ~stage:Stages.images + ~variables:[("DISTRIBUTION", distribution)] + ~parallel:(Matrix matrix) + ~retry: + {max = 2; when_ = [Stuck_or_timeout_failure; Runner_system_failure]} + ~tag:Gcp_very_high_cpu + ["scripts/ci/build-base-images.sh images/base-images/Dockerfile.debian"] + in + let job_debian_based_images = + make_job_base_images + ~__POS__ + ~name:"oc.base-images.debian" + ~distribution:"debian" + ~matrix:debian_matrix + in + let job_ubuntu_based_images = + make_job_base_images + ~__POS__ + ~name:"oc.base-images.ubuntu" + ~distribution:"ubuntu" + ~matrix:ubuntu_matrix + in + [job_debian_based_images; job_ubuntu_based_images] + +let child_pipeline = + Pipeline.register_child + "base_images" + ~description:"Build CI base images" + ~jobs:(job_datadog_pipeline_trace :: jobs) diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 2e962e09bf03..e452dd96e7c3 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -970,6 +970,16 @@ let jobs pipeline_type = ~dependencies:(Dependent []) Homebrew.child_pipeline_full in + + let job_base_images_trigger = + trigger_job + ~__POS__ + ~rules:(make_rules ~manual:No ~changes:changeset_base_images ()) + ~stage:Stages.images + ~dependencies:(Dependent []) + Base_images.child_pipeline + in + (* check that ksy files are still up-to-date with octez *) let job_kaitai_checks : tezos_job = job @@ -1865,6 +1875,7 @@ let jobs pipeline_type = job_debian_repository_trigger_full; job_rpm_repository_trigger_full; job_homebrew_trigger_full; + job_base_images_trigger; ] in jobs_debian @ jobs_misc @ jobs_sdk_rust @ jobs_sdk_bindings @ jobs_kernels @@ -2084,6 +2095,14 @@ let jobs pipeline_type = ~stage:Stages.manual Homebrew.child_pipeline_full in + let job_base_images_trigger = + trigger_job + ~__POS__ + ~rules:(make_rules ~manual:Yes ()) + ~stage:Stages.manual + ~dependencies:(Dependent []) + Base_images.child_pipeline + in match pipeline_type with | Before_merging | Merge_train -> (* Note: manual jobs in stage [manual] (which is the final @@ -2132,13 +2151,22 @@ let jobs pipeline_type = ["./scripts/ci/docker_verify_signature.sh"] in let jobs = - [job_docker_amd64_test_manual; job_docker_arm64_test_manual] - @ [job_docker_verify_test_arm64; job_docker_verify_test_amd64] + [ + job_docker_amd64_test_manual; + job_docker_arm64_test_manual; + job_docker_verify_test_arm64; + job_docker_verify_test_amd64; + job_base_images_trigger; + ] in if pipeline_type = Merge_train then jobs else - job_homebrew_repository_trigger :: job_rpm_repository_trigger_partial - :: job_debian_repository_trigger_partial :: jobs + [ + job_homebrew_repository_trigger; + job_rpm_repository_trigger_partial; + job_debian_repository_trigger_partial; + ] + @ jobs (* No manual jobs on the scheduled pipeline *) | Schedule_extended_test -> [] in diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 02530dc7e955..8c36b441e32a 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -255,6 +255,9 @@ let changeset_base = Changeset.make [".gitlab/**/*"; ".gitlab-ci.yml"] let changeset_images = Changeset.make ["images/**/*"] +let changeset_base_images = + Changeset.make ["images/base-images/**/*"; "scripts/ci/build-base-images.sh"] + (** Only if octez source code has changed *) let changeset_octez = let octez_source_content = diff --git a/images/base-images/Dockerfile.debian b/images/base-images/Dockerfile.debian new file mode 100644 index 000000000000..5490ee914f34 --- /dev/null +++ b/images/base-images/Dockerfile.debian @@ -0,0 +1,21 @@ +ARG IMAGE=invalid +# the image with proper version is set as ARG +#hadolint ignore=DL3006 +FROM ${IMAGE} + +ARG DEBIAN_FRONTEND=noninteractive +ENV TZ=Etc/UTC + +# Setup proxy +ARG APT_PROXY +ENV APT_PROXY=${APT_PROXY:-false} +RUN echo "Acquire::http::Proxy \"$APT_PROXY\";" > /etc/apt/apt.conf.d/01proxy + +# install datadog static binary +COPY images/scripts/install_datadog_static.sh /tmp +#hadolint ignore=DL3008,DL3015 +RUN apt-get update && \ + apt-get -y install curl && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + /tmp/install_datadog_static.sh diff --git a/scripts/ci/build-base-images.sh b/scripts/ci/build-base-images.sh new file mode 100755 index 000000000000..56209edb822f --- /dev/null +++ b/scripts/ci/build-base-images.sh @@ -0,0 +1,44 @@ +#!/bin/sh +set -eu + +DOCKERFILE=${1:?"Dockerfile missing"} + +BINFMT_VERSION=9.2.2-52 + +LATEST_TAG="${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}" +LATEST_TAG_GENERIC="${CI_COMMIT_REF_SLUG}" + +## Build multi-arch Docker image +gcp="${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$DISTRIBUTION:$RELEASE-${LATEST_TAG}" +gcp_generic="${GCP_REGISTRY}/$CI_PROJECT_NAMESPACE/tezos/$DISTRIBUTION:$RELEASE-${LATEST_TAG_GENERIC}" +protected_gcp="${GCP_PROTECTED_REGISTRY}/tezos/tezos/$DISTRIBUTION:$RELEASE" + +# Cross-platform emulator collection +# We cannot make this installation permanent inside the GitLab executor GCP image yet +# https://github.com/tonistiigi/binfmt/issues/75 +docker run --privileged --rm "tonistiigi/binfmt:qemu-v${BINFMT_VERSION}" --install all + +docker buildx create --use --name tezos || echo "Warning: Docker builder instance already exists" + +echo "Building images to GCP registries ..." +# shellcheck disable=SC2046 +docker buildx build --push \ + --platform linux/amd64,linux/arm64 \ + --label "com.tezos.build-pipeline-id"="${CI_PIPELINE_ID}" \ + --label "com.tezos.build-pipeline-url"="${CI_PIPELINE_URL}" \ + --label "com.tezos.build-job-id"="${CI_JOB_ID}" \ + --label "com.tezos.build-job-url"="${CI_JOB_URL}" \ + --label "com.tezos.build-tezos-revision"="${CI_COMMIT_SHA}" \ + --cache-from="type=registry,mode=max,compression=zstd,ref=${gcp_generic}" \ + --cache-from="type=registry,mode=max,compression=zstd,ref=${protected_gcp}" \ + --build-arg=BUILDKIT_INLINE_CACHE=1 \ + --secret "id=npm_token,src=/tmp/npm_token.txt" \ + --build-arg IMAGE="$DISTRIBUTION:$RELEASE" \ + --build-arg APT_PROXY="${APT_PROXY_DEB:-}" \ + --build-arg NPM_REGISTRY_DOMAIN="${NPM_REGISTRY_DOMAIN:-}" \ + --build-arg NPM_REGISTRY="${NPM_REGISTRY:-}" \ + --tag "${gcp}" \ + --tag "${gcp_generic}" \ + $(if [ "$CI_COMMIT_REF_PROTECTED" = "true" ]; then echo "--tag=${protected_gcp}"; fi) \ + $(if [ "${DOCKER_FORCE_BUILD:-false}" = "true" ]; then echo "--no-cache"; fi) \ + -f "$DOCKERFILE" . -- GitLab