From db2ef4e79502fd406f3ac4c2b45f8c5311fdbc09 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Thu, 28 Mar 2024 11:44:27 +0100 Subject: [PATCH] CI: generate [debian_repository.yml] --- .../ci/jobs/packaging/debian_repository.yml | 155 +++++++++++------- .gitlab/ci/jobs/shared/images.yml | 2 + ci/bin/code_verification.ml | 119 +++++++++++++- ci/bin/common.ml | 6 +- ci/bin/main.ml | 1 - 5 files changed, 224 insertions(+), 59 deletions(-) diff --git a/.gitlab/ci/jobs/packaging/debian_repository.yml b/.gitlab/ci/jobs/packaging/debian_repository.yml index be2853a81e77..f5549c8028f5 100644 --- a/.gitlab/ci/jobs/packaging/debian_repository.yml +++ b/.gitlab/ci/jobs/packaging/debian_repository.yml @@ -1,88 +1,133 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. -variables: - DEP_IMAGE: registry.gitlab.com/tezos/tezos/build-$DISTRIBUTION-$RELEASE - -# This is a generic template to build both Debian- and Ubuntu- -# packages. We use the variable $DISTRIBUTION to discriminate -# between the two. -.docker-build-debian-dependencies: - extends: - - .image_template__docker - - .docker_auth_template - - .needs__trigger +oc.docker-build-debian-dependencies: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 stage: build - rules: - - changes: - - scripts/version.sh - - .gitlab-ci.yml - - debian-deps-build.Dockerfile - when: on_success - - when: manual - allow_failure: true tags: - - $TAGS + - $TAGS + rules: + - changes: + - .gitlab-ci.yml + - debian-deps-build.Dockerfile + - scripts/version.sh + when: on_success + - when: manual + allow_failure: true + needs: + - trigger + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh script: - - .gitlab/ci/jobs/packaging/build-debian-packages-dependencies.sh - -oc.docker-build-debian-dependencies: - extends: - - .docker-build-debian-dependencies + - .gitlab/ci/jobs/packaging/build-debian-packages-dependencies.sh + services: + - docker:${DOCKER_VERSION}-dind variables: + DOCKER_VERSION: 24.0.6 + DEP_IMAGE: registry.gitlab.com/tezos/tezos/build-$DISTRIBUTION-$RELEASE DISTRIBUTION: debian parallel: matrix: - - RELEASE: [unstable, bookworm] - TAGS: [gcp, gcp_arm64] + - RELEASE: + - unstable + - bookworm + TAGS: + - gcp + - gcp_arm64 oc.docker-build-ubuntu-dependencies: - extends: - - .docker-build-debian-dependencies + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + stage: build + tags: + - $TAGS + rules: + - changes: + - .gitlab-ci.yml + - debian-deps-build.Dockerfile + - scripts/version.sh + when: on_success + - when: manual + allow_failure: true + needs: + - trigger + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - .gitlab/ci/jobs/packaging/build-debian-packages-dependencies.sh + services: + - docker:${DOCKER_VERSION}-dind variables: + DOCKER_VERSION: 24.0.6 + DEP_IMAGE: registry.gitlab.com/tezos/tezos/build-$DISTRIBUTION-$RELEASE DISTRIBUTION: ubuntu parallel: matrix: - - RELEASE: [focal, jammy] - TAGS: [gcp, gcp_arm64] + - RELEASE: + - focal + - jammy + TAGS: + - gcp + - gcp_arm64 oc.build-debian-based-packages: + image: alpine:3.18 stage: manual + tags: + - gcp needs: [] - when: manual + dependencies: [] script: - echo "Trigger build debian packages" + - echo 'Trigger build debian packages' + when: manual -.build-debian-packages: - stage: manual - needs: [oc.build-debian-based-packages] +oc.build-debian: image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG} + stage: manual tags: - - $TAGS - artifacts: - paths: - - packages/$DISTRIBUTION/$RELEASE + - $TAGS + needs: + - oc.build-debian-based-packages + dependencies: [] script: - - .gitlab/ci/jobs/packaging/build-debian-packages.sh - -# Since we use parallel/matrix and we don't want to create -# false associations (i.e. debian:jammy). We have to create -# two different jobs for debian and ubuntu even if they -# share the same template. -oc.build-debian: - extends: - - .build-debian-packages + - .gitlab/ci/jobs/packaging/build-debian-packages.sh variables: + DEP_IMAGE: registry.gitlab.com/tezos/tezos/build-$DISTRIBUTION-$RELEASE DISTRIBUTION: debian + artifacts: + paths: + - packages/$DISTRIBUTION/$RELEASE parallel: matrix: - - RELEASE: [unstable, bookworm] - TAGS: [gcp, gcp_arm64] + - RELEASE: + - unstable + - bookworm + TAGS: + - gcp + - gcp_arm64 oc.build-ubuntu: - extends: - - .build-debian-packages + image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG} + stage: manual + tags: + - $TAGS + needs: + - oc.build-debian-based-packages + dependencies: [] + script: + - .gitlab/ci/jobs/packaging/build-debian-packages.sh variables: + DEP_IMAGE: registry.gitlab.com/tezos/tezos/build-$DISTRIBUTION-$RELEASE DISTRIBUTION: ubuntu + artifacts: + paths: + - packages/$DISTRIBUTION/$RELEASE parallel: matrix: - - RELEASE: [jammy, focal] - TAGS: [gcp, gcp_arm64] + - RELEASE: + - focal + - jammy + TAGS: + - gcp + - gcp_arm64 diff --git a/.gitlab/ci/jobs/shared/images.yml b/.gitlab/ci/jobs/shared/images.yml index 43cdde0eb3b7..6affd59cdf19 100644 --- a/.gitlab/ci/jobs/shared/images.yml +++ b/.gitlab/ci/jobs/shared/images.yml @@ -9,6 +9,8 @@ image: ${client_libs_dependencies_image_name}:${client_libs_dependencies_image_tag} .image_template__debian_bookworm: image: debian:bookworm +.image_template__debian_dependencies_image: + image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG} .image_template__docker: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 .image_template__fedora_37: diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index af82c3e9ae2b..2a423a843a76 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -368,6 +368,9 @@ let jobs_unit_tests ~job_build_x86_64_release ~job_build_x86_64_exp_dev_extra oc_unit_protocol_compiles; ] +(* These are the set of Linux distributions and their release for + which we test installation of the deprecated Serokell PPA binary + packages. *) type install_octez_distribution = Ubuntu_focal | Ubuntu_jammy | Fedora_37 let image_of_distribution = function @@ -375,6 +378,20 @@ let image_of_distribution = function | Ubuntu_jammy -> Images.ubuntu_jammy | Fedora_37 -> Images.fedora_37 +(** These are the set of Debian release-architecture combinations for + which we build deb packages in the job + [job_build_debian_package]. A dependency image will be built once + for each combination of [RELEASE] and [TAGS]. *) +let debian_package_release_matrix = + [[("RELEASE", ["unstable"; "bookworm"]); ("TAGS", ["gcp"; "gcp_arm64"])]] + +(** These are the set of Ubuntu release-architecture combinations for + which we build deb packages in the job + [job_build_ubuntu_package]. See {!debian_package_release_matrix} + for more information. *) +let ubuntu_package_release_matrix = + [[("RELEASE", ["focal"; "jammy"]); ("TAGS", ["gcp"; "gcp_arm64"])]] + let job_tezt ~__POS__ ?rules ?parallel ?(tags = ["gcp_tezt"]) ~name ~(tezt_tests : Tezt_core.TSL_AST.t) ?(retry = 2) ?(tezt_retry = 1) ?(tezt_parallel = 1) ?(tezt_variant = "") @@ -511,6 +528,11 @@ let tezt_tests ?(memory_3k = false) ?(memory_4k = false) @ List.map (fun tag -> TSL_AST.Not (Has_tag tag)) negative @ and_) +let build_debian_packages_image = + Image.register + ~name:"debian_dependencies_image" + ~image_path:"$DEP_IMAGE:${CI_COMMIT_REF_SLUG}" + (* Encodes the conditional [before_merging] pipeline and its unconditional variant [schedule_extended_test]. *) let jobs pipeline_type = @@ -887,7 +909,101 @@ let jobs pipeline_type = ~dependencies:(Dependent [Artifacts job_opam_prepare])) |> jobs_external_once ~path:"packaging/opam_package.yml" in - job_opam_prepare :: jobs_opam_packages + let debian_repository : tezos_job list = + let variables add = + ( "DEP_IMAGE", + "registry.gitlab.com/tezos/tezos/build-$DISTRIBUTION-$RELEASE" ) + :: add + in + let changeset_debian_dependencies = + Changeset.make + [ + "scripts/version.sh"; + ".gitlab-ci.yml"; + "debian-deps-build.Dockerfile"; + ] + in + let make_job_docker_build_debian_dependencies ~__POS__ ~name ~matrix + ~distribution = + job_docker_authenticated + ~__POS__ + ~name + ~dependencies:dependencies_needs_trigger + ~rules: + (make_rules ~changes:changeset_debian_dependencies ~manual:Yes ()) + ~stage:Stages.build + ~variables:(variables [("DISTRIBUTION", distribution)]) + ~parallel:(Matrix matrix) + ~tags:["$TAGS"] + [".gitlab/ci/jobs/packaging/build-debian-packages-dependencies.sh"] + in + let job_docker_build_debian_dependencies : tezos_job = + make_job_docker_build_debian_dependencies + ~__POS__ + ~name:"oc.docker-build-debian-dependencies" + ~distribution:"debian" + ~matrix:debian_package_release_matrix + in + let job_docker_build_ubuntu_dependencies : tezos_job = + make_job_docker_build_debian_dependencies + ~__POS__ + ~name:"oc.docker-build-ubuntu-dependencies" + ~distribution:"ubuntu" + ~matrix:ubuntu_package_release_matrix + in + let job_trigger_build_debian_based_packages : tezos_job = + job + ~__POS__ + ~name:"oc.build-debian-based-packages" + ~image:Images.alpine + ~stage:Stages.manual + ~dependencies:(Dependent []) + ~when_:Manual + ["echo 'Trigger build debian packages'"] + in + let make_job_build_debian_packages ~__POS__ ~name ~matrix ~distribution = + job + ~__POS__ + ~name + ~image:build_debian_packages_image + ~stage:Stages.manual + ~variables:(variables [("DISTRIBUTION", distribution)]) + ~dependencies: + (Dependent [Job job_trigger_build_debian_based_packages]) + ~parallel:(Matrix matrix) + ~tags:["$TAGS"] + ~artifacts:(artifacts ["packages/$DISTRIBUTION/$RELEASE"]) + [".gitlab/ci/jobs/packaging/build-debian-packages.sh"] + in + let job_build_debian_package : tezos_job = + make_job_build_debian_packages + ~__POS__ + ~name:"oc.build-debian" + ~distribution:"debian" + ~matrix:debian_package_release_matrix + in + let job_build_ubuntu_package : tezos_job = + make_job_build_debian_packages + ~__POS__ + ~name:"oc.build-ubuntu" + ~distribution:"ubuntu" + ~matrix:ubuntu_package_release_matrix + in + [ + job_docker_build_debian_dependencies; + job_docker_build_ubuntu_dependencies; + job_trigger_build_debian_based_packages; + job_build_debian_package; + job_build_ubuntu_package; + ] + |> jobs_external_once ~path:"packaging/debian_repository.yml" + in + (job_opam_prepare :: jobs_opam_packages) + @ + (* Debian packages can only be built manually on the [Before_merging] pipelines *) + match pipeline_type with + | Before_merging -> debian_repository + | Schedule_extended_test -> [] in (* Dependencies for jobs that should run immediately after jobs [job_build_x86_64] in [Before_merging] if they are present @@ -1165,6 +1281,7 @@ let jobs pipeline_type = let install_octez_rules = make_rules ~changes:changeset_install_jobs ~manual:Yes () in + (* Test installation of the deprecated Serokell PPA binary packages. *) let job_install_bin ~__POS__ ~name ?allow_failure ?(rc = false) distribution = let distribution_string = diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 45aaf1a3e8b3..0f06a9864b50 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -515,16 +515,18 @@ let changeset_test_evm_compatibility = authenticate with Docker Hub provided the environment variable [CI_DOCKER_AUTH] contains the appropriate credentials. *) let job_docker_authenticated ?(skip_docker_initialization = false) - ?ci_docker_hub ?artifacts ?(variables = []) ?rules ?dependencies ?arch - ?when_ ?allow_failure ~__POS__ ~stage ~name script : tezos_job = + ?ci_docker_hub ?artifacts ?(variables = []) ?rules ?dependencies ?arch ?tags + ?when_ ?allow_failure ?parallel ~__POS__ ~stage ~name script : tezos_job = let docker_version = "24.0.6" in job ?rules ?dependencies ?artifacts ?arch + ?tags ?when_ ?allow_failure + ?parallel ~__POS__ ~image:Images.docker ~variables: diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 58c66e5ec148..6e56a8864fe1 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -190,7 +190,6 @@ let () = | ".gitlab/ci/jobs/doc/documentation.yml" | ".gitlab/ci/jobs/doc/documentation:linkcheck.yml" | ".gitlab/ci/jobs/doc/oc.install_python.yml" - | ".gitlab/ci/jobs/packaging/debian_repository.yml" | ".gitlab/ci/jobs/shared/images.yml" | ".gitlab/ci/jobs/shared/templates.yml" | ".gitlab/ci/jobs/test/common.yml" | ".gitlab/ci/pipelines/before_merging.yml" -- GitLab