From 5ba6b1c80cf62f87338a9c9e0751974e8f8af550 Mon Sep 17 00:00:00 2001 From: Pietro Abate Date: Fri, 7 Jun 2024 10:37:53 +0200 Subject: [PATCH] packaging: reduce packages tests matrix for merge pipelines --- .gitlab/ci/pipelines/debian_repository.yml | 119 ------------------ .../ci/pipelines/schedule_extended_test.yml | 4 +- ci/bin/code_verification.ml | 17 +-- ci/bin/common.ml | 4 + ci/bin/debian_repository.ml | 100 +++++++++------ 5 files changed, 80 insertions(+), 164 deletions(-) diff --git a/.gitlab/ci/pipelines/debian_repository.yml b/.gitlab/ci/pipelines/debian_repository.yml index 76ad0f8fba28..ff1321d397cf 100644 --- a/.gitlab/ci/pipelines/debian_repository.yml +++ b/.gitlab/ci/pipelines/debian_repository.yml @@ -31,36 +31,9 @@ oc.docker-build-debian-dependencies: parallel: matrix: - RELEASE: - - unstable - bookworm TAGS: - gcp - - gcp_arm64 - -oc.docker-build-ubuntu-dependencies: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 - stage: images - tags: - - $TAGS - dependencies: [] - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - ./scripts/ci/build-debian-packages-dependencies.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - DEP_IMAGE: registry.gitlab.com/tezos/tezos/build-$DISTRIBUTION-$RELEASE - DISTRIBUTION: ubuntu - parallel: - matrix: - - RELEASE: - - focal - - jammy - TAGS: - - gcp - - gcp_arm64 oc.build-debian: image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG} @@ -80,35 +53,9 @@ oc.build-debian: parallel: matrix: - RELEASE: - - unstable - bookworm TAGS: - gcp - - gcp_arm64 - -oc.build-ubuntu: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG} - stage: build - tags: - - $TAGS - dependencies: [] - script: - - export CARGO_NET_OFFLINE=false - - ./scripts/ci/build-debian-packages.sh - variables: - DEP_IMAGE: registry.gitlab.com/tezos/tezos/build-$DISTRIBUTION-$RELEASE - DISTRIBUTION: ubuntu - artifacts: - paths: - - packages/$DISTRIBUTION/$RELEASE - parallel: - matrix: - - RELEASE: - - focal - - jammy - TAGS: - - gcp - - gcp_arm64 oc.build-debian-current: image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG} @@ -128,35 +75,9 @@ oc.build-debian-current: parallel: matrix: - RELEASE: - - unstable - bookworm TAGS: - gcp - - gcp_arm64 - -oc.build-ubuntu-current: - image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG} - stage: build - tags: - - $TAGS - dependencies: [] - script: - - export CARGO_NET_OFFLINE=false - - ./scripts/ci/build-debian-packages_current.sh - variables: - DEP_IMAGE: registry.gitlab.com/tezos/tezos/build-$DISTRIBUTION-$RELEASE - DISTRIBUTION: ubuntu - artifacts: - paths: - - packages/$DISTRIBUTION/$RELEASE - parallel: - matrix: - - RELEASE: - - focal - - jammy - TAGS: - - gcp - - gcp_arm64 apt_repo_debian_current: image: debian:bookworm @@ -176,46 +97,6 @@ apt_repo_debian_current: ARCHITECTURES: amd64 GNUPGHOME: $CI_PROJECT_DIR/.gnupg -apt_repo_ubuntu_current: - image: public.ecr.aws/lts/ubuntu:20.04_stable - stage: publishing - tags: - - gcp - needs: - - oc.build-ubuntu-current - dependencies: - - oc.build-ubuntu-current - before_script: - - . ./scripts/version.sh - - ./scripts/ci/prepare-apt-repo.sh - script: - - ./scripts/ci/create_debian_repo.sh ubuntu focal jammy - variables: - ARCHITECTURES: amd64 - GNUPGHOME: $CI_PROJECT_DIR/.gnupg - -oc.install_bin_ubuntu_focal: - image: public.ecr.aws/lts/ubuntu:20.04_stable - stage: publishing_tests - tags: - - gcp - needs: - - apt_repo_ubuntu_current - dependencies: [] - script: - - ./docs/introduction/install-bin-deb.sh ubuntu focal - -oc.install_bin_ubuntu_jammy: - image: public.ecr.aws/lts/ubuntu:22.04_stable - stage: publishing_tests - tags: - - gcp - needs: - - apt_repo_ubuntu_current - dependencies: [] - script: - - ./docs/introduction/install-bin-deb.sh ubuntu jammy - oc.install_bin_debian_bookworm: image: debian:bookworm stage: publishing_tests diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 94a87ba5272f..390db64cf332 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -2917,10 +2917,10 @@ documentation:linkcheck: - make -C docs redirectcheck - make -C docs linkcheck -trigger:debian_repository: +trigger:debian_repository_scheduled: stage: manual rules: - when: always needs: [] trigger: - include: .gitlab/ci/pipelines/debian_repository.yml + include: .gitlab/ci/pipelines/debian_repository_scheduled.yml diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 4b8807f442bb..cdcbe8492f21 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -36,10 +36,6 @@ open Gitlab_ci.Util open Tezos_ci open Common -(* Encodes the conditional [before_merging] pipeline and its - unconditional variant [schedule_extended_test]. *) -type code_verification_pipeline = Before_merging | Schedule_extended_test - (** Configuration of manual jobs for [make_rules] *) type manual = | No (** Do not add rule for manual start. *) @@ -311,8 +307,15 @@ let tezt_tests ?(memory_3k = false) ?(memory_4k = false) @ List.map (fun tag -> TSL_AST.Not (Has_tag tag)) negative @ and_) -let debian_repository_child_pipeline = - Pipeline.register_child "debian_repository" ~jobs:Debian_repository.jobs +let debian_repository_child_pipeline pipeline_type = + let pipeline_name = + match pipeline_type with + | Before_merging -> "debian_repository" + | Schedule_extended_test -> "debian_repository_scheduled" + in + Pipeline.register_child + pipeline_name + ~jobs:(Debian_repository.jobs pipeline_type) (* Encodes the conditional [before_merging] pipeline and its unconditional variant [schedule_extended_test]. *) @@ -1812,7 +1815,7 @@ let jobs pipeline_type = ~rules:(make_rules ~manual:Yes ()) ~dependencies:(Dependent []) ~stage:Stages.manual - debian_repository_child_pipeline + (debian_repository_child_pipeline pipeline_type) in match pipeline_type with | Before_merging -> diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 3e06c79a4308..665b15eb640a 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -19,6 +19,10 @@ open Gitlab_ci.Types open Gitlab_ci.Util open Tezos_ci +(* Encodes the conditional [before_merging] pipeline and its + unconditional variant [schedule_extended_test]. *) +type code_verification_pipeline = Before_merging | Schedule_extended_test + let cargo_home = (* Note: - We want [CARGO_HOME] to be in a sub-folder of diff --git a/ci/bin/debian_repository.ml b/ci/bin/debian_repository.ml index 0d7cdeb71242..6068d799fb95 100644 --- a/ci/bin/debian_repository.ml +++ b/ci/bin/debian_repository.ml @@ -22,16 +22,26 @@ let build_debian_packages_image = (** These are the set of Debian release-architecture combinations for which we build deb packages in the job [job_build_debian_package]. A dependency image will be built once - for each combination of [RELEASE] and [TAGS]. *) -let debian_package_release_matrix = - [[("RELEASE", ["unstable"; "bookworm"]); ("TAGS", ["gcp"; "gcp_arm64"])]] + for each combination of [RELEASE] and [TAGS]. + + If [release_pipeline] is false, we only tests a subset of the matrix, + one release, and one architecture. *) +let debian_package_release_matrix = function + | Before_merging -> [[("RELEASE", ["bookworm"]); ("TAGS", ["gcp"])]] + | Schedule_extended_test -> + [[("RELEASE", ["unstable"; "bookworm"]); ("TAGS", ["gcp"; "gcp_arm64"])]] (** These are the set of Ubuntu release-architecture combinations for which we build deb packages in the job [job_build_ubuntu_package]. See {!debian_package_release_matrix} - for more information. *) -let ubuntu_package_release_matrix = - [[("RELEASE", ["focal"; "jammy"]); ("TAGS", ["gcp"; "gcp_arm64"])]] + for more information. + + If [release_pipeline] is false, we only tests a subset of the matrix, + one release, and one architecture. *) +let ubuntu_package_release_matrix = function + | Before_merging -> [[("RELEASE", ["jammy"]); ("TAGS", ["gcp"])]] + | Schedule_extended_test -> + [[("RELEASE", ["focal"; "jammy"]); ("TAGS", ["gcp"; "gcp_arm64"])]] (* Push .deb artifacts to storagecloud apt repository. *) let job_apt_repo ?rules ~__POS__ ~name ?(stage = Stages.publishing) @@ -55,7 +65,9 @@ let job_apt_repo ?rules ~__POS__ ~name ?(stage = Stages.publishing) ~variables script -let jobs = +(* The entire Debian packages pipeline. When [pipeline_type] is [Before_merging] + we test only on Debian stable. *) +let jobs pipeline_type = let variables add = ("DEP_IMAGE", "registry.gitlab.com/tezos/tezos/build-$DISTRIBUTION-$RELEASE") :: add @@ -76,14 +88,14 @@ let jobs = ~__POS__ ~name:"oc.docker-build-debian-dependencies" ~distribution:"debian" - ~matrix:debian_package_release_matrix + ~matrix:(debian_package_release_matrix pipeline_type) in let job_docker_build_ubuntu_dependencies : tezos_job = make_job_docker_build_debian_dependencies ~__POS__ ~name:"oc.docker-build-ubuntu-dependencies" ~distribution:"ubuntu" - ~matrix:ubuntu_package_release_matrix + ~matrix:(ubuntu_package_release_matrix pipeline_type) in let make_job_build_debian_packages ~__POS__ ~name ~matrix ~distribution ~script = @@ -122,16 +134,16 @@ let jobs = ~__POS__ ~name:"oc.build-debian-current" ~distribution:"debian" - ~matrix:debian_package_release_matrix ~script:"./scripts/ci/build-debian-packages_current.sh" + ~matrix:(debian_package_release_matrix pipeline_type) in let job_build_ubuntu_package_current : tezos_job = make_job_build_debian_packages ~__POS__ ~name:"oc.build-ubuntu-current" ~distribution:"ubuntu" - ~matrix:ubuntu_package_release_matrix ~script:"./scripts/ci/build-debian-packages_current.sh" + ~matrix:(ubuntu_package_release_matrix pipeline_type) in (* These jobs build the next packages in a matrix using the @@ -141,16 +153,16 @@ let jobs = ~__POS__ ~name:"oc.build-debian" ~distribution:"debian" - ~matrix:debian_package_release_matrix ~script:"./scripts/ci/build-debian-packages.sh" + ~matrix:(debian_package_release_matrix pipeline_type) in let job_build_ubuntu_package : tezos_job = make_job_build_debian_packages ~__POS__ ~name:"oc.build-ubuntu" ~distribution:"ubuntu" - ~matrix:ubuntu_package_release_matrix ~script:"./scripts/ci/build-debian-packages.sh" + ~matrix:(ubuntu_package_release_matrix pipeline_type) in (* These create the apt repository for the current packages *) @@ -172,18 +184,20 @@ let jobs = in (* These test the installability of the current packages *) - let test_current_packages_jobs = - let job_install_bin ~__POS__ ~name ~dependencies ~image ?allow_failure - script = - job - ?allow_failure - ~__POS__ - ~name - ~image - ~dependencies - ~stage:Stages.publishing_tests - script - in + let job_install_bin ~__POS__ ~name ~dependencies ~image ?allow_failure script + = + job + ?allow_failure + ~__POS__ + ~name + ~image + ~dependencies + ~stage:Stages.publishing_tests + script + in + let test_current_ubuntu_packages_jobs = + (* in merge pipelines we tests only debian. release pipelines + test the entire matrix *) [ job_install_bin ~__POS__ @@ -197,6 +211,10 @@ let jobs = ~dependencies:(Dependent [Job job_apt_repo_ubuntu_current]) ~image:Images.ubuntu_jammy ["./docs/introduction/install-bin-deb.sh ubuntu jammy"]; + ] + in + let test_current_debian_packages_jobs = + [ job_install_bin ~__POS__ ~name:"oc.install_bin_debian_bookworm" @@ -205,14 +223,24 @@ let jobs = ["./docs/introduction/install-bin-deb.sh debian bookworm"]; ] in - [ - job_docker_build_debian_dependencies; - job_docker_build_ubuntu_dependencies; - job_build_debian_package; - job_build_ubuntu_package; - job_build_debian_package_current; - job_build_ubuntu_package_current; - job_apt_repo_debian_current; - job_apt_repo_ubuntu_current; - ] - @ test_current_packages_jobs + let debian_jobs = + [ + job_docker_build_debian_dependencies; + job_build_debian_package; + job_build_debian_package_current; + job_apt_repo_debian_current; + ] + @ test_current_debian_packages_jobs + in + let ubuntu_jobs = + [ + job_docker_build_ubuntu_dependencies; + job_build_ubuntu_package; + job_build_ubuntu_package_current; + job_apt_repo_ubuntu_current; + ] + @ test_current_ubuntu_packages_jobs + in + match pipeline_type with + | Before_merging -> debian_jobs + | Schedule_extended_test -> debian_jobs @ ubuntu_jobs -- GitLab