From d528682f7fa307f2e519e2963f9ab2200eb53597 Mon Sep 17 00:00:00 2001 From: Neo <11726174-neo.nomadic@users.noreply.gitlab.com> Date: Wed, 10 Apr 2024 18:11:14 +0000 Subject: [PATCH 1/2] CI: Setting up the GCP protected docker registry with manual authentication --- .../build/oc.docker:amd64-test_manual.yml | 27 ++++ .../build/oc.docker:arm64-test_manual.yml | 25 ++++ ...lient-libs-dependencies-before_merging.yml | 29 ++++ ...s-dependencies-scheduled_extended_test.yml | 22 +++ ...c.docker:rust-toolchain-before_merging.yml | 38 +++++ ...rust-toolchain-scheduled_extended_test.yml | 22 +++ .../ci/jobs/packaging/debian_repository.yml | 133 ++++++++++++++++++ .gitlab/ci/jobs/shared/images.yml | 45 ++++++ .../ci/pipelines/etherlink_release_tag.yml | 4 +- .gitlab/ci/pipelines/master_branch.yml | 8 +- .gitlab/ci/pipelines/non_release_tag.yml | 8 +- .gitlab/ci/pipelines/non_release_tag_test.yml | 8 +- .../ci/pipelines/octez_beta_release_tag.yml | 8 +- .gitlab/ci/pipelines/octez_latest_release.yml | 2 +- .../pipelines/octez_latest_release_test.yml | 2 +- .gitlab/ci/pipelines/octez_release_tag.yml | 8 +- .../ci/pipelines/octez_release_tag_test.yml | 8 +- ci/bin/common.ml | 4 +- scripts/ci/docker_registry_auth.sh | 33 ++++- 19 files changed, 400 insertions(+), 34 deletions(-) create mode 100644 .gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml create mode 100644 .gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml create mode 100644 .gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-before_merging.yml create mode 100644 .gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-scheduled_extended_test.yml create mode 100644 .gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml create mode 100644 .gitlab/ci/jobs/build/oc.docker:rust-toolchain-scheduled_extended_test.yml create mode 100644 .gitlab/ci/jobs/packaging/debian_repository.yml create mode 100644 .gitlab/ci/jobs/shared/images.yml diff --git a/.gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml b/.gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml new file mode 100644 index 000000000000..664db2f48495 --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml @@ -0,0 +1,27 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.docker:amd64: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + stage: manual + tags: + - gcp + needs: [] + dependencies: [] + allow_failure: true + before_script: [] + script: + - ./scripts/ci/docker_rust_toolchain_build.sh + - source rust_toolchain_image_tag.env + - export rust_toolchain_image_tag + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + DOCKER_BUILD_TARGET: with-evm-artifacts + IMAGE_ARCH_PREFIX: amd64_ + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + RUST_TOOLCHAIN_ALWAYS_REBUILD: "true" + when: manual diff --git a/.gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml b/.gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml new file mode 100644 index 000000000000..bf534da18a37 --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml @@ -0,0 +1,25 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.docker:arm64: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + stage: manual + tags: + - gcp_arm64 + needs: [] + dependencies: [] + allow_failure: true + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: arm64_ + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + rust_toolchain_image_tag: is-never-pulled + when: manual diff --git a/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-before_merging.yml b/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-before_merging.yml new file mode 100644 index 000000000000..4f163b6289b6 --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-before_merging.yml @@ -0,0 +1,29 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.docker:client-libs-dependencies: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + stage: build + tags: + - gcp + rules: + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - client-libs/*kaitai*/**/* + - images/**/* + - scripts/ci/**/* + - src/**/* + when: on_success + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_client_libs_dependencies_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: client_libs_dependencies_image_tag.env diff --git a/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-scheduled_extended_test.yml b/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-scheduled_extended_test.yml new file mode 100644 index 000000000000..a18779e15e6b --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-scheduled_extended_test.yml @@ -0,0 +1,22 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.docker:client-libs-dependencies: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + stage: build + tags: + - gcp + rules: + - when: always + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_client_libs_dependencies_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: client_libs_dependencies_image_tag.env diff --git a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml new file mode 100644 index 000000000000..895b69cf1356 --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml @@ -0,0 +1,38 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.docker:rust-toolchain: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + stage: build + tags: + - gcp + rules: + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - etherlink.mk + - etherlink/**/* + - images/**/* + - kernels.mk + - michelson_test_scripts/**/* + - scripts/ci/**/* + - src/**/* + - tezt/**/* + - tzt_reference_test_suite/**/* + when: on_success + - when: manual + allow_failure: true + needs: + - trigger + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_rust_toolchain_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: rust_toolchain_image_tag.env diff --git a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-scheduled_extended_test.yml b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-scheduled_extended_test.yml new file mode 100644 index 000000000000..c0512e9d9cec --- /dev/null +++ b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-scheduled_extended_test.yml @@ -0,0 +1,22 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.docker:rust-toolchain: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + stage: build + tags: + - gcp + rules: + - when: always + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_rust_toolchain_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: rust_toolchain_image_tag.env diff --git a/.gitlab/ci/jobs/packaging/debian_repository.yml b/.gitlab/ci/jobs/packaging/debian_repository.yml new file mode 100644 index 000000000000..2e011b4e0108 --- /dev/null +++ b/.gitlab/ci/jobs/packaging/debian_repository.yml @@ -0,0 +1,133 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.docker-build-debian-dependencies: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + stage: build + tags: + - $TAGS + rules: + - changes: + - .gitlab-ci.yml + - debian-deps-build.Dockerfile + - scripts/version.sh + when: on_success + - when: manual + allow_failure: true + needs: + - trigger + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - .gitlab/ci/jobs/packaging/build-debian-packages-dependencies.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DEP_IMAGE: registry.gitlab.com/tezos/tezos/build-$DISTRIBUTION-$RELEASE + DISTRIBUTION: debian + parallel: + matrix: + - RELEASE: + - unstable + - bookworm + TAGS: + - gcp + - gcp_arm64 + +oc.docker-build-ubuntu-dependencies: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + stage: build + tags: + - $TAGS + rules: + - changes: + - .gitlab-ci.yml + - debian-deps-build.Dockerfile + - scripts/version.sh + when: on_success + - when: manual + allow_failure: true + needs: + - trigger + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - .gitlab/ci/jobs/packaging/build-debian-packages-dependencies.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DEP_IMAGE: registry.gitlab.com/tezos/tezos/build-$DISTRIBUTION-$RELEASE + DISTRIBUTION: ubuntu + parallel: + matrix: + - RELEASE: + - focal + - jammy + TAGS: + - gcp + - gcp_arm64 + +oc.build-debian-based-packages: + image: alpine:3.18 + stage: manual + tags: + - gcp + needs: [] + dependencies: [] + script: + - echo 'Trigger build debian packages' + when: manual + +oc.build-debian: + image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG} + stage: manual + tags: + - $TAGS + needs: + - oc.build-debian-based-packages + dependencies: [] + script: + - .gitlab/ci/jobs/packaging/build-debian-packages.sh + variables: + DEP_IMAGE: registry.gitlab.com/tezos/tezos/build-$DISTRIBUTION-$RELEASE + DISTRIBUTION: debian + artifacts: + paths: + - packages/$DISTRIBUTION/$RELEASE + parallel: + matrix: + - RELEASE: + - unstable + - bookworm + TAGS: + - gcp + - gcp_arm64 + +oc.build-ubuntu: + image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG} + stage: manual + tags: + - $TAGS + needs: + - oc.build-debian-based-packages + dependencies: [] + script: + - .gitlab/ci/jobs/packaging/build-debian-packages.sh + variables: + DEP_IMAGE: registry.gitlab.com/tezos/tezos/build-$DISTRIBUTION-$RELEASE + DISTRIBUTION: ubuntu + artifacts: + paths: + - packages/$DISTRIBUTION/$RELEASE + parallel: + matrix: + - RELEASE: + - focal + - jammy + TAGS: + - gcp + - gcp_arm64 diff --git a/.gitlab/ci/jobs/shared/images.yml b/.gitlab/ci/jobs/shared/images.yml new file mode 100644 index 000000000000..b017d17abc5e --- /dev/null +++ b/.gitlab/ci/jobs/shared/images.yml @@ -0,0 +1,45 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +.image_template__alpine: + image: alpine:3.18 +.image_template__ci_release: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.5.0 +.image_template__client_libs_dependencies: + image: ${client_libs_dependencies_image_name}:${client_libs_dependencies_image_tag} +.image_template__debian_bookworm: + image: debian:bookworm +.image_template__debian_bullseye: + image: debian:bullseye +.image_template__debian_dependencies_image: + image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG} +.image_template__docker: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 +.image_template__fedora_37: + image: fedora:37 +.image_template__fedora_39: + image: fedora:39 +.image_template__hadolint: + image: hadolint/hadolint:2.9.3-debian +.image_template__opam_debian_bullseye: + image: ocaml/opam:debian-11 +.image_template__opam_ubuntu_focal: + image: ocaml/opam:ubuntu-20.04 +.image_template__opam_ubuntu_mantic: + image: ocaml/opam:ubuntu-23.10 +.image_template__runtime_build_dependencies: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} +.image_template__runtime_build_test_dependencies: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} +.image_template__runtime_e2etest_dependencies: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} +.image_template__runtime_prebuild_dependencies: + image: ${build_deps_image_name}:runtime-prebuild-dependencies--${build_deps_image_version} +.image_template__rust_toolchain: + image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} +.image_template__semgrep_agent: + image: returntocorp/semgrep-agent:sha-c6cd7cf +.image_template__ubuntu_focal: + image: public.ecr.aws/lts/ubuntu:20.04_stable +.image_template__ubuntu_jammy: + image: public.ecr.aws/lts/ubuntu:22.04_stable diff --git a/.gitlab/ci/pipelines/etherlink_release_tag.yml b/.gitlab/ci/pipelines/etherlink_release_tag.yml index bb9334466c12..3483bcad26db 100644 --- a/.gitlab/ci/pipelines/etherlink_release_tag.yml +++ b/.gitlab/ci/pipelines/etherlink_release_tag.yml @@ -2,7 +2,7 @@ # Edit file ci/bin/main.ml instead. docker:prepare-etherlink-release: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 stage: prepare_release tags: - gcp @@ -21,7 +21,7 @@ docker:prepare-etherlink-release: - kernels.tar.gz gitlab:etherlink-release: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.4.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.5.0 stage: publish_package_gitlab tags: - gcp diff --git a/.gitlab/ci/pipelines/master_branch.yml b/.gitlab/ci/pipelines/master_branch.yml index 7c37491f1f01..b4480e6d3292 100644 --- a/.gitlab/ci/pipelines/master_branch.yml +++ b/.gitlab/ci/pipelines/master_branch.yml @@ -2,7 +2,7 @@ # Edit file ci/bin/main.ml instead. oc.docker:rust-toolchain: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 stage: build tags: - gcp @@ -123,7 +123,7 @@ oc.build_arm64-exp-dev-extra: when: on_success oc.docker:amd64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 stage: build tags: - gcp @@ -147,7 +147,7 @@ oc.docker:amd64: RUST_TOOLCHAIN_ALWAYS_REBUILD: "true" oc.docker:arm64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 stage: build tags: - gcp_arm64 @@ -233,7 +233,7 @@ publish:documentation: - ./scripts/ci/doc_publish.sh docker:merge_manifests: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 stage: prepare_release tags: - gcp diff --git a/.gitlab/ci/pipelines/non_release_tag.yml b/.gitlab/ci/pipelines/non_release_tag.yml index 95dc73d228aa..10c1b71f1cca 100644 --- a/.gitlab/ci/pipelines/non_release_tag.yml +++ b/.gitlab/ci/pipelines/non_release_tag.yml @@ -40,7 +40,7 @@ oc.build:static-arm64-linux-binaries: - octez-binaries/$ARCH/* oc.docker:amd64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 stage: build tags: - gcp @@ -60,7 +60,7 @@ oc.docker:amd64: rust_toolchain_image_tag: is-never-pulled oc.docker:arm64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 stage: build tags: - gcp_arm64 @@ -169,7 +169,7 @@ oc.build:rpm:amd64: - rockylinux:9.3 docker:merge_manifests: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 stage: prepare_release tags: - gcp @@ -188,7 +188,7 @@ docker:merge_manifests: CI_DOCKER_HUB: "true" gitlab:publish: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.4.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.5.0 stage: publish_package_gitlab tags: - gcp diff --git a/.gitlab/ci/pipelines/non_release_tag_test.yml b/.gitlab/ci/pipelines/non_release_tag_test.yml index e6c30d86e7c2..d1c62d98f353 100644 --- a/.gitlab/ci/pipelines/non_release_tag_test.yml +++ b/.gitlab/ci/pipelines/non_release_tag_test.yml @@ -40,7 +40,7 @@ oc.build:static-arm64-linux-binaries: - octez-binaries/$ARCH/* oc.docker:amd64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 stage: build tags: - gcp @@ -60,7 +60,7 @@ oc.docker:amd64: rust_toolchain_image_tag: is-never-pulled oc.docker:arm64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 stage: build tags: - gcp_arm64 @@ -169,7 +169,7 @@ oc.build:rpm:amd64: - rockylinux:9.3 docker:merge_manifests: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 stage: prepare_release tags: - gcp @@ -188,7 +188,7 @@ docker:merge_manifests: CI_DOCKER_HUB: "false" gitlab:publish: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.4.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.5.0 stage: publish_package_gitlab tags: - gcp diff --git a/.gitlab/ci/pipelines/octez_beta_release_tag.yml b/.gitlab/ci/pipelines/octez_beta_release_tag.yml index 5a2db2902b43..d9b9be16dc1b 100644 --- a/.gitlab/ci/pipelines/octez_beta_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_beta_release_tag.yml @@ -40,7 +40,7 @@ oc.build:static-arm64-linux-binaries: - octez-binaries/$ARCH/* oc.docker:amd64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 stage: build tags: - gcp @@ -60,7 +60,7 @@ oc.docker:amd64: rust_toolchain_image_tag: is-never-pulled oc.docker:arm64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 stage: build tags: - gcp_arm64 @@ -169,7 +169,7 @@ oc.build:rpm:amd64: - rockylinux:9.3 docker:merge_manifests: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 stage: prepare_release tags: - gcp @@ -188,7 +188,7 @@ docker:merge_manifests: CI_DOCKER_HUB: "true" gitlab:release: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.4.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.5.0 stage: publish_release_gitlab tags: - gcp diff --git a/.gitlab/ci/pipelines/octez_latest_release.yml b/.gitlab/ci/pipelines/octez_latest_release.yml index a735a2a81e88..21190818b143 100644 --- a/.gitlab/ci/pipelines/octez_latest_release.yml +++ b/.gitlab/ci/pipelines/octez_latest_release.yml @@ -2,7 +2,7 @@ # Edit file ci/bin/main.ml instead. docker:promote_to_latest: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 stage: publish_release tags: - gcp diff --git a/.gitlab/ci/pipelines/octez_latest_release_test.yml b/.gitlab/ci/pipelines/octez_latest_release_test.yml index a9c8a797376f..78c8669cc533 100644 --- a/.gitlab/ci/pipelines/octez_latest_release_test.yml +++ b/.gitlab/ci/pipelines/octez_latest_release_test.yml @@ -2,7 +2,7 @@ # Edit file ci/bin/main.ml instead. docker:promote_to_latest: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 stage: publish_release tags: - gcp diff --git a/.gitlab/ci/pipelines/octez_release_tag.yml b/.gitlab/ci/pipelines/octez_release_tag.yml index d15ce45457d0..11e787cd392f 100644 --- a/.gitlab/ci/pipelines/octez_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_release_tag.yml @@ -40,7 +40,7 @@ oc.build:static-arm64-linux-binaries: - octez-binaries/$ARCH/* oc.docker:amd64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 stage: build tags: - gcp @@ -60,7 +60,7 @@ oc.docker:amd64: rust_toolchain_image_tag: is-never-pulled oc.docker:arm64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 stage: build tags: - gcp_arm64 @@ -169,7 +169,7 @@ oc.build:rpm:amd64: - rockylinux:9.3 docker:merge_manifests: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 stage: prepare_release tags: - gcp @@ -188,7 +188,7 @@ docker:merge_manifests: CI_DOCKER_HUB: "true" gitlab:release: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.4.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.5.0 stage: publish_release_gitlab tags: - gcp diff --git a/.gitlab/ci/pipelines/octez_release_tag_test.yml b/.gitlab/ci/pipelines/octez_release_tag_test.yml index 461c2c32b089..70d1fc8b905d 100644 --- a/.gitlab/ci/pipelines/octez_release_tag_test.yml +++ b/.gitlab/ci/pipelines/octez_release_tag_test.yml @@ -40,7 +40,7 @@ oc.build:static-arm64-linux-binaries: - octez-binaries/$ARCH/* oc.docker:amd64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 stage: build tags: - gcp @@ -60,7 +60,7 @@ oc.docker:amd64: rust_toolchain_image_tag: is-never-pulled oc.docker:arm64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 stage: build tags: - gcp_arm64 @@ -169,7 +169,7 @@ oc.build:rpm:amd64: - rockylinux:9.3 docker:merge_manifests: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 stage: prepare_release tags: - gcp @@ -188,7 +188,7 @@ docker:merge_manifests: CI_DOCKER_HUB: "false" gitlab:release: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.4.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.5.0 stage: publish_release_gitlab tags: - gcp diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 58cd57a00845..8846bbe4b4f7 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -132,7 +132,7 @@ module Images = struct let docker = Image.register ~name:"docker" - ~image_path:"${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0" + ~image_path:"${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0" (* The Alpine version should be kept up to date with the version used for the [build_deps_image_name] images and specified in the @@ -179,7 +179,7 @@ module Images = struct let ci_release = Image.register ~name:"ci_release" - ~image_path:"${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.4.0" + ~image_path:"${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.5.0" let hadolint = Image.register ~name:"hadolint" ~image_path:"hadolint/hadolint:2.9.3-debian" diff --git a/scripts/ci/docker_registry_auth.sh b/scripts/ci/docker_registry_auth.sh index 3649dfa83f24..b540dca2d93b 100755 --- a/scripts/ci/docker_registry_auth.sh +++ b/scripts/ci/docker_registry_auth.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -eu current_dir=$(cd "$(dirname "${0}")" && pwd) @@ -29,6 +29,7 @@ echo "CI_PROJECT_NAMESPACE=${CI_PROJECT_NAMESPACE}" echo "IMAGE_ARCH_PREFIX=${IMAGE_ARCH_PREFIX:-}" echo "DOCKER_BUILD_TARGET=${DOCKER_BUILD_TARGET:-}" echo "RUST_TOOLCHAIN_IMAGE=${RUST_TOOLCHAIN_IMAGE:-}" +echo "CI_COMMIT_REF_PROTECTED=${CI_COMMIT_REF_PROTECTED}" # CI_DOCKER_HUB is used to switch to Docker Hub if credentials are available with CI_DOCKER_AUTH # /!\ CI_DOCKER_HUB can be unset, CI_DOCKER_AUTH is only available on protected branches @@ -64,9 +65,33 @@ fi # Allow to push to private GCP Artifact Registry if the CI/CD variable is defined if [ -n "${GCP_REGISTRY:-}" ]; then - echo "### Logging into GCP Artifact Registry for pushing images" - GCP_ARTIFACT_REGISTRY_TOKEN=$(curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token | cut -d'"' -f4) - echo "${GCP_ARTIFACT_REGISTRY_TOKEN}" | docker login us-central1-docker.pkg.dev -u oauth2accesstoken --password-stdin + + # There are two registry for storing docker images. The first allows push from + # a Tezos CI job from a no protected branch, the second is accessible for push + # operation only from protected branches for security reasons. Finally both + # registries can be pull in public access. + if [ "${CI_COMMIT_REF_PROTECTED:-false}" = true ]; then + echo "### Logging into protected GCP Artifact Registry for pushing images" + + echo "${GCP_PROTECTED_SERVICE_ACCOUNT}" | base64 -d > protected_sa.json + + PRIVATE_KEY=$(jq -r '.private_key' protected_sa.json) + CLIENT_EMAIL=$(jq -r '.client_email' protected_sa.json) + + JWT_HEADER=$(echo -n '{"alg":"RS256","typ":"JWT"}' | base64 | tr -d '\n=') + # shellcheck disable=SC2153 + CURRENT_TIMESTAMP=`date +%s` + JWT_CLAIM_SET=$(echo -n '{"iss":"'"${CLIENT_EMAIL}"'","scope":"https://www.googleapis.com/auth/cloud-platform","aud":"https://oauth2.googleapis.com/token","exp":'$(expr $CURRENT_TIMESTAMP + 3600)',"iat":'$(date +%s)'}' | base64 | tr -d '\n=') + JWT_SIGNATURE=$(echo -n "${JWT_HEADER}.${JWT_CLAIM_SET}" | openssl dgst -sha256 -sign <(echo -n "${PRIVATE_KEY}") -binary | base64 -w 0 | tr '+/' '-_' | tr -d '=') + JWT="${JWT_HEADER}.${JWT_CLAIM_SET}.${JWT_SIGNATURE}" + + ACCESS_TOKEN=$(curl -s -X POST -H "Content-Type: application/x-www-form-urlencoded" -d "grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer&assertion=${JWT}" https://oauth2.googleapis.com/token | jq -r '.access_token') + echo "$ACCESS_TOKEN" | docker login us-central1-docker.pkg.dev -u oauth2accesstoken --password-stdin + else + echo "### Logging into standard GCP Artifact Registry for pushing images" + GCP_ARTIFACT_REGISTRY_TOKEN=$(curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token | cut -d'"' -f4) + echo "${GCP_ARTIFACT_REGISTRY_TOKEN}" | docker login us-central1-docker.pkg.dev -u oauth2accesstoken --password-stdin + fi fi # shellcheck source=scripts/ci/docker_registry.inc.sh -- GitLab From 7b6b2a5b834d8ef6b83643bff33a360bb56a25bd Mon Sep 17 00:00:00 2001 From: Neo <11726174-neo.nomadic@users.noreply.gitlab.com> Date: Fri, 19 Apr 2024 12:11:42 +0000 Subject: [PATCH 2/2] CI: Setting up the GCP protected docker registry with gcloud authentication --- .../build/oc.docker:amd64-test_manual.yml | 27 ---- .../build/oc.docker:arm64-test_manual.yml | 25 ---- ...lient-libs-dependencies-before_merging.yml | 29 ---- ...s-dependencies-scheduled_extended_test.yml | 22 --- ...c.docker:rust-toolchain-before_merging.yml | 38 ----- ...rust-toolchain-scheduled_extended_test.yml | 22 --- .../ci/jobs/packaging/debian_repository.yml | 133 ------------------ .gitlab/ci/jobs/shared/images.yml | 45 ------ .gitlab/ci/pipelines/before_merging.yml | 12 +- .../ci/pipelines/etherlink_release_tag.yml | 4 +- .gitlab/ci/pipelines/master_branch.yml | 8 +- .gitlab/ci/pipelines/non_release_tag.yml | 8 +- .gitlab/ci/pipelines/non_release_tag_test.yml | 8 +- .../ci/pipelines/octez_beta_release_tag.yml | 8 +- .gitlab/ci/pipelines/octez_latest_release.yml | 2 +- .../pipelines/octez_latest_release_test.yml | 2 +- .gitlab/ci/pipelines/octez_release_tag.yml | 8 +- .../ci/pipelines/octez_release_tag_test.yml | 8 +- .../ci/pipelines/schedule_extended_test.yml | 4 +- ci/bin/common.ml | 4 +- ci/bin/main.ml | 6 + scripts/ci/docker_registry_auth.sh | 31 ++-- 22 files changed, 54 insertions(+), 400 deletions(-) delete mode 100644 .gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml delete mode 100644 .gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml delete mode 100644 .gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-before_merging.yml delete mode 100644 .gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-scheduled_extended_test.yml delete mode 100644 .gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml delete mode 100644 .gitlab/ci/jobs/build/oc.docker:rust-toolchain-scheduled_extended_test.yml delete mode 100644 .gitlab/ci/jobs/packaging/debian_repository.yml delete mode 100644 .gitlab/ci/jobs/shared/images.yml diff --git a/.gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml b/.gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml deleted file mode 100644 index 664db2f48495..000000000000 --- a/.gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml +++ /dev/null @@ -1,27 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.docker:amd64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 - stage: manual - tags: - - gcp - needs: [] - dependencies: [] - allow_failure: true - before_script: [] - script: - - ./scripts/ci/docker_rust_toolchain_build.sh - - source rust_toolchain_image_tag.env - - export rust_toolchain_image_tag - - ./scripts/ci/docker_release.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - CI_DOCKER_HUB: "false" - DOCKER_BUILD_TARGET: with-evm-artifacts - IMAGE_ARCH_PREFIX: amd64_ - EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables - RUST_TOOLCHAIN_ALWAYS_REBUILD: "true" - when: manual diff --git a/.gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml b/.gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml deleted file mode 100644 index bf534da18a37..000000000000 --- a/.gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml +++ /dev/null @@ -1,25 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.docker:arm64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 - stage: manual - tags: - - gcp_arm64 - needs: [] - dependencies: [] - allow_failure: true - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - ./scripts/ci/docker_release.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - CI_DOCKER_HUB: "false" - DOCKER_BUILD_TARGET: without-evm-artifacts - IMAGE_ARCH_PREFIX: arm64_ - EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables - rust_toolchain_image_tag: is-never-pulled - when: manual diff --git a/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-before_merging.yml b/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-before_merging.yml deleted file mode 100644 index 4f163b6289b6..000000000000 --- a/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-before_merging.yml +++ /dev/null @@ -1,29 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.docker:client-libs-dependencies: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 - stage: build - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - client-libs/*kaitai*/**/* - - images/**/* - - scripts/ci/**/* - - src/**/* - when: on_success - dependencies: [] - before_script: [] - script: - - ./scripts/ci/docker_client_libs_dependencies_build.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - CI_DOCKER_HUB: "false" - artifacts: - reports: - dotenv: client_libs_dependencies_image_tag.env diff --git a/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-scheduled_extended_test.yml b/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-scheduled_extended_test.yml deleted file mode 100644 index a18779e15e6b..000000000000 --- a/.gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-scheduled_extended_test.yml +++ /dev/null @@ -1,22 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.docker:client-libs-dependencies: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 - stage: build - tags: - - gcp - rules: - - when: always - dependencies: [] - before_script: [] - script: - - ./scripts/ci/docker_client_libs_dependencies_build.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - CI_DOCKER_HUB: "false" - artifacts: - reports: - dotenv: client_libs_dependencies_image_tag.env diff --git a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml deleted file mode 100644 index 895b69cf1356..000000000000 --- a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-before_merging.yml +++ /dev/null @@ -1,38 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.docker:rust-toolchain: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 - stage: build - tags: - - gcp - rules: - - changes: - - .gitlab-ci.yml - - .gitlab/**/* - - etherlink.mk - - etherlink/**/* - - images/**/* - - kernels.mk - - michelson_test_scripts/**/* - - scripts/ci/**/* - - src/**/* - - tezt/**/* - - tzt_reference_test_suite/**/* - when: on_success - - when: manual - allow_failure: true - needs: - - trigger - dependencies: [] - before_script: [] - script: - - ./scripts/ci/docker_rust_toolchain_build.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - CI_DOCKER_HUB: "false" - artifacts: - reports: - dotenv: rust_toolchain_image_tag.env diff --git a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-scheduled_extended_test.yml b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-scheduled_extended_test.yml deleted file mode 100644 index c0512e9d9cec..000000000000 --- a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-scheduled_extended_test.yml +++ /dev/null @@ -1,22 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.docker:rust-toolchain: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 - stage: build - tags: - - gcp - rules: - - when: always - dependencies: [] - before_script: [] - script: - - ./scripts/ci/docker_rust_toolchain_build.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - CI_DOCKER_HUB: "false" - artifacts: - reports: - dotenv: rust_toolchain_image_tag.env diff --git a/.gitlab/ci/jobs/packaging/debian_repository.yml b/.gitlab/ci/jobs/packaging/debian_repository.yml deleted file mode 100644 index 2e011b4e0108..000000000000 --- a/.gitlab/ci/jobs/packaging/debian_repository.yml +++ /dev/null @@ -1,133 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.docker-build-debian-dependencies: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 - stage: build - tags: - - $TAGS - rules: - - changes: - - .gitlab-ci.yml - - debian-deps-build.Dockerfile - - scripts/version.sh - when: on_success - - when: manual - allow_failure: true - needs: - - trigger - dependencies: [] - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - .gitlab/ci/jobs/packaging/build-debian-packages-dependencies.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - DEP_IMAGE: registry.gitlab.com/tezos/tezos/build-$DISTRIBUTION-$RELEASE - DISTRIBUTION: debian - parallel: - matrix: - - RELEASE: - - unstable - - bookworm - TAGS: - - gcp - - gcp_arm64 - -oc.docker-build-ubuntu-dependencies: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 - stage: build - tags: - - $TAGS - rules: - - changes: - - .gitlab-ci.yml - - debian-deps-build.Dockerfile - - scripts/version.sh - when: on_success - - when: manual - allow_failure: true - needs: - - trigger - dependencies: [] - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - .gitlab/ci/jobs/packaging/build-debian-packages-dependencies.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - DEP_IMAGE: registry.gitlab.com/tezos/tezos/build-$DISTRIBUTION-$RELEASE - DISTRIBUTION: ubuntu - parallel: - matrix: - - RELEASE: - - focal - - jammy - TAGS: - - gcp - - gcp_arm64 - -oc.build-debian-based-packages: - image: alpine:3.18 - stage: manual - tags: - - gcp - needs: [] - dependencies: [] - script: - - echo 'Trigger build debian packages' - when: manual - -oc.build-debian: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG} - stage: manual - tags: - - $TAGS - needs: - - oc.build-debian-based-packages - dependencies: [] - script: - - .gitlab/ci/jobs/packaging/build-debian-packages.sh - variables: - DEP_IMAGE: registry.gitlab.com/tezos/tezos/build-$DISTRIBUTION-$RELEASE - DISTRIBUTION: debian - artifacts: - paths: - - packages/$DISTRIBUTION/$RELEASE - parallel: - matrix: - - RELEASE: - - unstable - - bookworm - TAGS: - - gcp - - gcp_arm64 - -oc.build-ubuntu: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG} - stage: manual - tags: - - $TAGS - needs: - - oc.build-debian-based-packages - dependencies: [] - script: - - .gitlab/ci/jobs/packaging/build-debian-packages.sh - variables: - DEP_IMAGE: registry.gitlab.com/tezos/tezos/build-$DISTRIBUTION-$RELEASE - DISTRIBUTION: ubuntu - artifacts: - paths: - - packages/$DISTRIBUTION/$RELEASE - parallel: - matrix: - - RELEASE: - - focal - - jammy - TAGS: - - gcp - - gcp_arm64 diff --git a/.gitlab/ci/jobs/shared/images.yml b/.gitlab/ci/jobs/shared/images.yml deleted file mode 100644 index b017d17abc5e..000000000000 --- a/.gitlab/ci/jobs/shared/images.yml +++ /dev/null @@ -1,45 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -.image_template__alpine: - image: alpine:3.18 -.image_template__ci_release: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.5.0 -.image_template__client_libs_dependencies: - image: ${client_libs_dependencies_image_name}:${client_libs_dependencies_image_tag} -.image_template__debian_bookworm: - image: debian:bookworm -.image_template__debian_bullseye: - image: debian:bullseye -.image_template__debian_dependencies_image: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG} -.image_template__docker: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 -.image_template__fedora_37: - image: fedora:37 -.image_template__fedora_39: - image: fedora:39 -.image_template__hadolint: - image: hadolint/hadolint:2.9.3-debian -.image_template__opam_debian_bullseye: - image: ocaml/opam:debian-11 -.image_template__opam_ubuntu_focal: - image: ocaml/opam:ubuntu-20.04 -.image_template__opam_ubuntu_mantic: - image: ocaml/opam:ubuntu-23.10 -.image_template__runtime_build_dependencies: - image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} -.image_template__runtime_build_test_dependencies: - image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} -.image_template__runtime_e2etest_dependencies: - image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} -.image_template__runtime_prebuild_dependencies: - image: ${build_deps_image_name}:runtime-prebuild-dependencies--${build_deps_image_version} -.image_template__rust_toolchain: - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} -.image_template__semgrep_agent: - image: returntocorp/semgrep-agent:sha-c6cd7cf -.image_template__ubuntu_focal: - image: public.ecr.aws/lts/ubuntu:20.04_stable -.image_template__ubuntu_jammy: - image: public.ecr.aws/lts/ubuntu:22.04_stable diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 71a0d90fc828..e01f2464f773 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -76,7 +76,7 @@ nix: when: on_failure oc.docker:rust-toolchain: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: build tags: - gcp @@ -114,7 +114,7 @@ oc.docker:rust-toolchain: dotenv: rust_toolchain_image_tag.env oc.docker:client-libs-dependencies: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: build tags: - gcp @@ -6423,7 +6423,7 @@ opam:tezt-tezos: retry: 2 oc.docker-build-debian-dependencies: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: build tags: - $TAGS @@ -6458,7 +6458,7 @@ oc.docker-build-debian-dependencies: - gcp_arm64 oc.docker-build-ubuntu-dependencies: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: build tags: - $TAGS @@ -8275,7 +8275,7 @@ documentation:linkcheck: - make -C docs linkcheck oc.docker:amd64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: manual tags: - gcp @@ -8300,7 +8300,7 @@ oc.docker:amd64: when: manual oc.docker:arm64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: manual tags: - gcp_arm64 diff --git a/.gitlab/ci/pipelines/etherlink_release_tag.yml b/.gitlab/ci/pipelines/etherlink_release_tag.yml index 3483bcad26db..d545ad8dd8c1 100644 --- a/.gitlab/ci/pipelines/etherlink_release_tag.yml +++ b/.gitlab/ci/pipelines/etherlink_release_tag.yml @@ -2,7 +2,7 @@ # Edit file ci/bin/main.ml instead. docker:prepare-etherlink-release: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: prepare_release tags: - gcp @@ -21,7 +21,7 @@ docker:prepare-etherlink-release: - kernels.tar.gz gitlab:etherlink-release: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.5.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.6.0 stage: publish_package_gitlab tags: - gcp diff --git a/.gitlab/ci/pipelines/master_branch.yml b/.gitlab/ci/pipelines/master_branch.yml index b4480e6d3292..25a6f8132cd3 100644 --- a/.gitlab/ci/pipelines/master_branch.yml +++ b/.gitlab/ci/pipelines/master_branch.yml @@ -2,7 +2,7 @@ # Edit file ci/bin/main.ml instead. oc.docker:rust-toolchain: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: build tags: - gcp @@ -123,7 +123,7 @@ oc.build_arm64-exp-dev-extra: when: on_success oc.docker:amd64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: build tags: - gcp @@ -147,7 +147,7 @@ oc.docker:amd64: RUST_TOOLCHAIN_ALWAYS_REBUILD: "true" oc.docker:arm64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: build tags: - gcp_arm64 @@ -233,7 +233,7 @@ publish:documentation: - ./scripts/ci/doc_publish.sh docker:merge_manifests: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: prepare_release tags: - gcp diff --git a/.gitlab/ci/pipelines/non_release_tag.yml b/.gitlab/ci/pipelines/non_release_tag.yml index 10c1b71f1cca..695d7dac3473 100644 --- a/.gitlab/ci/pipelines/non_release_tag.yml +++ b/.gitlab/ci/pipelines/non_release_tag.yml @@ -40,7 +40,7 @@ oc.build:static-arm64-linux-binaries: - octez-binaries/$ARCH/* oc.docker:amd64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: build tags: - gcp @@ -60,7 +60,7 @@ oc.docker:amd64: rust_toolchain_image_tag: is-never-pulled oc.docker:arm64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: build tags: - gcp_arm64 @@ -169,7 +169,7 @@ oc.build:rpm:amd64: - rockylinux:9.3 docker:merge_manifests: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: prepare_release tags: - gcp @@ -188,7 +188,7 @@ docker:merge_manifests: CI_DOCKER_HUB: "true" gitlab:publish: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.5.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.6.0 stage: publish_package_gitlab tags: - gcp diff --git a/.gitlab/ci/pipelines/non_release_tag_test.yml b/.gitlab/ci/pipelines/non_release_tag_test.yml index d1c62d98f353..6966a9cc653e 100644 --- a/.gitlab/ci/pipelines/non_release_tag_test.yml +++ b/.gitlab/ci/pipelines/non_release_tag_test.yml @@ -40,7 +40,7 @@ oc.build:static-arm64-linux-binaries: - octez-binaries/$ARCH/* oc.docker:amd64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: build tags: - gcp @@ -60,7 +60,7 @@ oc.docker:amd64: rust_toolchain_image_tag: is-never-pulled oc.docker:arm64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: build tags: - gcp_arm64 @@ -169,7 +169,7 @@ oc.build:rpm:amd64: - rockylinux:9.3 docker:merge_manifests: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: prepare_release tags: - gcp @@ -188,7 +188,7 @@ docker:merge_manifests: CI_DOCKER_HUB: "false" gitlab:publish: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.5.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.6.0 stage: publish_package_gitlab tags: - gcp diff --git a/.gitlab/ci/pipelines/octez_beta_release_tag.yml b/.gitlab/ci/pipelines/octez_beta_release_tag.yml index d9b9be16dc1b..a3bb6d67d800 100644 --- a/.gitlab/ci/pipelines/octez_beta_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_beta_release_tag.yml @@ -40,7 +40,7 @@ oc.build:static-arm64-linux-binaries: - octez-binaries/$ARCH/* oc.docker:amd64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: build tags: - gcp @@ -60,7 +60,7 @@ oc.docker:amd64: rust_toolchain_image_tag: is-never-pulled oc.docker:arm64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: build tags: - gcp_arm64 @@ -169,7 +169,7 @@ oc.build:rpm:amd64: - rockylinux:9.3 docker:merge_manifests: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: prepare_release tags: - gcp @@ -188,7 +188,7 @@ docker:merge_manifests: CI_DOCKER_HUB: "true" gitlab:release: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.5.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.6.0 stage: publish_release_gitlab tags: - gcp diff --git a/.gitlab/ci/pipelines/octez_latest_release.yml b/.gitlab/ci/pipelines/octez_latest_release.yml index 21190818b143..2dbeeab7fd82 100644 --- a/.gitlab/ci/pipelines/octez_latest_release.yml +++ b/.gitlab/ci/pipelines/octez_latest_release.yml @@ -2,7 +2,7 @@ # Edit file ci/bin/main.ml instead. docker:promote_to_latest: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: publish_release tags: - gcp diff --git a/.gitlab/ci/pipelines/octez_latest_release_test.yml b/.gitlab/ci/pipelines/octez_latest_release_test.yml index 78c8669cc533..8410cd0a9fbe 100644 --- a/.gitlab/ci/pipelines/octez_latest_release_test.yml +++ b/.gitlab/ci/pipelines/octez_latest_release_test.yml @@ -2,7 +2,7 @@ # Edit file ci/bin/main.ml instead. docker:promote_to_latest: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: publish_release tags: - gcp diff --git a/.gitlab/ci/pipelines/octez_release_tag.yml b/.gitlab/ci/pipelines/octez_release_tag.yml index 11e787cd392f..fdde43c82f90 100644 --- a/.gitlab/ci/pipelines/octez_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_release_tag.yml @@ -40,7 +40,7 @@ oc.build:static-arm64-linux-binaries: - octez-binaries/$ARCH/* oc.docker:amd64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: build tags: - gcp @@ -60,7 +60,7 @@ oc.docker:amd64: rust_toolchain_image_tag: is-never-pulled oc.docker:arm64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: build tags: - gcp_arm64 @@ -169,7 +169,7 @@ oc.build:rpm:amd64: - rockylinux:9.3 docker:merge_manifests: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: prepare_release tags: - gcp @@ -188,7 +188,7 @@ docker:merge_manifests: CI_DOCKER_HUB: "true" gitlab:release: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.5.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.6.0 stage: publish_release_gitlab tags: - gcp diff --git a/.gitlab/ci/pipelines/octez_release_tag_test.yml b/.gitlab/ci/pipelines/octez_release_tag_test.yml index 70d1fc8b905d..5c8a16fa1499 100644 --- a/.gitlab/ci/pipelines/octez_release_tag_test.yml +++ b/.gitlab/ci/pipelines/octez_release_tag_test.yml @@ -40,7 +40,7 @@ oc.build:static-arm64-linux-binaries: - octez-binaries/$ARCH/* oc.docker:amd64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: build tags: - gcp @@ -60,7 +60,7 @@ oc.docker:amd64: rust_toolchain_image_tag: is-never-pulled oc.docker:arm64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: build tags: - gcp_arm64 @@ -169,7 +169,7 @@ oc.build:rpm:amd64: - rockylinux:9.3 docker:merge_manifests: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: prepare_release tags: - gcp @@ -188,7 +188,7 @@ docker:merge_manifests: CI_DOCKER_HUB: "false" gitlab:release: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.5.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.6.0 stage: publish_release_gitlab tags: - gcp diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index abb7a238f5ef..b2439a9a1e1b 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -29,7 +29,7 @@ docker:hadolint-schedule_extended_test: - hadolint Dockerfile oc.docker:rust-toolchain: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: build tags: - gcp @@ -49,7 +49,7 @@ oc.docker:rust-toolchain: dotenv: rust_toolchain_image_tag.env oc.docker:client-libs-dependencies: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: build tags: - gcp diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 8846bbe4b4f7..f3c8293261ba 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -132,7 +132,7 @@ module Images = struct let docker = Image.register ~name:"docker" - ~image_path:"${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.11.0" + ~image_path:"${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0" (* The Alpine version should be kept up to date with the version used for the [build_deps_image_name] images and specified in the @@ -179,7 +179,7 @@ module Images = struct let ci_release = Image.register ~name:"ci_release" - ~image_path:"${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.5.0" + ~image_path:"${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.6.0" let hadolint = Image.register ~name:"hadolint" ~image_path:"hadolint/hadolint:2.9.3-debian" diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 5c03973ee15e..892a2016ae30 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -25,6 +25,12 @@ let variables : variables = [ (* /!\ This value MUST be the same as `opam_repository_tag` in `scripts/version.sh` *) ("build_deps_image_version", Common.build_deps_image_version); + (* /!\ GCP_REGISTRY is the variable containing the name of the registry to and from + which docker images are produced and consumed. This variable is defined at tezos + level with the value unprotected registry and at tezos/tezos level in its protected + version. This mechanism allows pipelines from a protected tezos/tezos branch to + read the protected variable from tezos/tezos and for others to not have access to + the variable tezos/tezos but tezos. *) ("build_deps_image_name", "${GCP_REGISTRY}/tezos/opam-repository"); ( "rust_toolchain_image_name", "${GCP_REGISTRY}/${CI_PROJECT_PATH}/rust-toolchain" ); diff --git a/scripts/ci/docker_registry_auth.sh b/scripts/ci/docker_registry_auth.sh index b540dca2d93b..c5fd4b60f876 100755 --- a/scripts/ci/docker_registry_auth.sh +++ b/scripts/ci/docker_registry_auth.sh @@ -29,7 +29,7 @@ echo "CI_PROJECT_NAMESPACE=${CI_PROJECT_NAMESPACE}" echo "IMAGE_ARCH_PREFIX=${IMAGE_ARCH_PREFIX:-}" echo "DOCKER_BUILD_TARGET=${DOCKER_BUILD_TARGET:-}" echo "RUST_TOOLCHAIN_IMAGE=${RUST_TOOLCHAIN_IMAGE:-}" -echo "CI_COMMIT_REF_PROTECTED=${CI_COMMIT_REF_PROTECTED}" +echo "CI_COMMIT_REF_PROTECTED=${CI_COMMIT_REF_PROTECTED:-}" # CI_DOCKER_HUB is used to switch to Docker Hub if credentials are available with CI_DOCKER_AUTH # /!\ CI_DOCKER_HUB can be unset, CI_DOCKER_AUTH is only available on protected branches @@ -65,28 +65,17 @@ fi # Allow to push to private GCP Artifact Registry if the CI/CD variable is defined if [ -n "${GCP_REGISTRY:-}" ]; then - - # There are two registry for storing docker images. The first allows push from - # a Tezos CI job from a no protected branch, the second is accessible for push - # operation only from protected branches for security reasons. Finally both - # registries can be pull in public access. - if [ "${CI_COMMIT_REF_PROTECTED:-false}" = true ]; then + # There are two registries for storing Docker images. The first allows pushes from + # Tezos CI jobs on unprotected branches. The second is accessible for push + # operation only from protected branches for security reasons. Finally, both + # registries are publicly accessible for pulls. + if [ "${CI_COMMIT_REF_PROTECTED:-false}" = true ]; then echo "### Logging into protected GCP Artifact Registry for pushing images" - echo "${GCP_PROTECTED_SERVICE_ACCOUNT}" | base64 -d > protected_sa.json - - PRIVATE_KEY=$(jq -r '.private_key' protected_sa.json) - CLIENT_EMAIL=$(jq -r '.client_email' protected_sa.json) - - JWT_HEADER=$(echo -n '{"alg":"RS256","typ":"JWT"}' | base64 | tr -d '\n=') - # shellcheck disable=SC2153 - CURRENT_TIMESTAMP=`date +%s` - JWT_CLAIM_SET=$(echo -n '{"iss":"'"${CLIENT_EMAIL}"'","scope":"https://www.googleapis.com/auth/cloud-platform","aud":"https://oauth2.googleapis.com/token","exp":'$(expr $CURRENT_TIMESTAMP + 3600)',"iat":'$(date +%s)'}' | base64 | tr -d '\n=') - JWT_SIGNATURE=$(echo -n "${JWT_HEADER}.${JWT_CLAIM_SET}" | openssl dgst -sha256 -sign <(echo -n "${PRIVATE_KEY}") -binary | base64 -w 0 | tr '+/' '-_' | tr -d '=') - JWT="${JWT_HEADER}.${JWT_CLAIM_SET}.${JWT_SIGNATURE}" - - ACCESS_TOKEN=$(curl -s -X POST -H "Content-Type: application/x-www-form-urlencoded" -d "grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer&assertion=${JWT}" https://oauth2.googleapis.com/token | jq -r '.access_token') - echo "$ACCESS_TOKEN" | docker login us-central1-docker.pkg.dev -u oauth2accesstoken --password-stdin + gcloud auth activate-service-account --key-file=protected_sa.json + gcloud auth configure-docker us.gcr.io + gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://us-central1-docker.pkg.dev + rm protected_sa.json else echo "### Logging into standard GCP Artifact Registry for pushing images" GCP_ARTIFACT_REGISTRY_TOKEN=$(curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token | cut -d'"' -f4) -- GitLab