From 5d826ddd0dbcfef9ed760ad6d0fb37bb60d973d1 Mon Sep 17 00:00:00 2001 From: Pietro Abate Date: Mon, 10 Jun 2024 14:18:04 +0200 Subject: [PATCH] packages next: add apt repo for next packages --- .../ci/pipelines/debian_repository_full.yml | 38 +++++++++++++++++++ .../pipelines/debian_repository_partial.yml | 19 ++++++++++ .gitlab/ci/pipelines/non_release_tag.yml | 38 +++++++++++++++++++ .gitlab/ci/pipelines/non_release_tag_test.yml | 38 +++++++++++++++++++ .../ci/pipelines/octez_beta_release_tag.yml | 38 +++++++++++++++++++ .gitlab/ci/pipelines/octez_release_tag.yml | 38 +++++++++++++++++++ .../ci/pipelines/octez_release_tag_test.yml | 38 +++++++++++++++++++ ci/bin/debian_repository.ml | 28 ++++++++++++-- scripts/ci/create_debian_repo.sh | 16 ++++---- 9 files changed, 280 insertions(+), 11 deletions(-) diff --git a/.gitlab/ci/pipelines/debian_repository_full.yml b/.gitlab/ci/pipelines/debian_repository_full.yml index dd933e9e6578..319eb1d4f947 100644 --- a/.gitlab/ci/pipelines/debian_repository_full.yml +++ b/.gitlab/ci/pipelines/debian_repository_full.yml @@ -135,6 +135,25 @@ apt_repo_debian_current: ARCHITECTURES: amd64 GNUPGHOME: $CI_PROJECT_DIR/.gnupg +apt_repo_debian: + image: debian:bookworm + stage: publishing + tags: + - gcp + needs: + - oc.build-debian + dependencies: + - oc.build-debian + before_script: + - . ./scripts/version.sh + - ./scripts/ci/prepare-apt-repo.sh + script: + - ./scripts/ci/create_debian_repo.sh debian bookworm + variables: + ARCHITECTURES: amd64 + GNUPGHOME: $CI_PROJECT_DIR/.gnupg + PREFIX: next + oc.docker-build-ubuntu-dependencies: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: images @@ -258,6 +277,25 @@ apt_repo_ubuntu_current: ARCHITECTURES: amd64 GNUPGHOME: $CI_PROJECT_DIR/.gnupg +apt_repo_ubuntu: + image: public.ecr.aws/lts/ubuntu:20.04_stable + stage: publishing + tags: + - gcp + needs: + - oc.build-ubuntu + dependencies: + - oc.build-ubuntu + before_script: + - . ./scripts/version.sh + - ./scripts/ci/prepare-apt-repo.sh + script: + - ./scripts/ci/create_debian_repo.sh ubuntu focal jammy + variables: + ARCHITECTURES: amd64 + GNUPGHOME: $CI_PROJECT_DIR/.gnupg + PREFIX: next + oc.lintian_debian: image: debian:bookworm stage: publishing_tests diff --git a/.gitlab/ci/pipelines/debian_repository_partial.yml b/.gitlab/ci/pipelines/debian_repository_partial.yml index 4a6afdbe627b..9f252a96969d 100644 --- a/.gitlab/ci/pipelines/debian_repository_partial.yml +++ b/.gitlab/ci/pipelines/debian_repository_partial.yml @@ -127,6 +127,25 @@ apt_repo_debian_current: ARCHITECTURES: amd64 GNUPGHOME: $CI_PROJECT_DIR/.gnupg +apt_repo_debian: + image: debian:bookworm + stage: publishing + tags: + - gcp + needs: + - oc.build-debian + dependencies: + - oc.build-debian + before_script: + - . ./scripts/version.sh + - ./scripts/ci/prepare-apt-repo.sh + script: + - ./scripts/ci/create_debian_repo.sh debian bookworm + variables: + ARCHITECTURES: amd64 + GNUPGHOME: $CI_PROJECT_DIR/.gnupg + PREFIX: next + oc.lintian_debian: image: debian:bookworm stage: publishing_tests diff --git a/.gitlab/ci/pipelines/non_release_tag.yml b/.gitlab/ci/pipelines/non_release_tag.yml index db393bcce770..13a0a4450ea4 100644 --- a/.gitlab/ci/pipelines/non_release_tag.yml +++ b/.gitlab/ci/pipelines/non_release_tag.yml @@ -366,6 +366,25 @@ apt_repo_debian_current: ARCHITECTURES: amd64 GNUPGHOME: $CI_PROJECT_DIR/.gnupg +apt_repo_debian: + image: debian:bookworm + stage: publishing + tags: + - gcp + needs: + - oc.build-debian + dependencies: + - oc.build-debian + before_script: + - . ./scripts/version.sh + - ./scripts/ci/prepare-apt-repo.sh + script: + - ./scripts/ci/create_debian_repo.sh debian bookworm + variables: + ARCHITECTURES: amd64 + GNUPGHOME: $CI_PROJECT_DIR/.gnupg + PREFIX: next + oc.docker-build-ubuntu-dependencies: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: images @@ -488,3 +507,22 @@ apt_repo_ubuntu_current: variables: ARCHITECTURES: amd64 GNUPGHOME: $CI_PROJECT_DIR/.gnupg + +apt_repo_ubuntu: + image: public.ecr.aws/lts/ubuntu:20.04_stable + stage: publishing + tags: + - gcp + needs: + - oc.build-ubuntu + dependencies: + - oc.build-ubuntu + before_script: + - . ./scripts/version.sh + - ./scripts/ci/prepare-apt-repo.sh + script: + - ./scripts/ci/create_debian_repo.sh ubuntu focal jammy + variables: + ARCHITECTURES: amd64 + GNUPGHOME: $CI_PROJECT_DIR/.gnupg + PREFIX: next diff --git a/.gitlab/ci/pipelines/non_release_tag_test.yml b/.gitlab/ci/pipelines/non_release_tag_test.yml index 0d46105c9569..76361aa0940d 100644 --- a/.gitlab/ci/pipelines/non_release_tag_test.yml +++ b/.gitlab/ci/pipelines/non_release_tag_test.yml @@ -366,6 +366,25 @@ apt_repo_debian_current: ARCHITECTURES: amd64 GNUPGHOME: $CI_PROJECT_DIR/.gnupg +apt_repo_debian: + image: debian:bookworm + stage: publishing + tags: + - gcp + needs: + - oc.build-debian + dependencies: + - oc.build-debian + before_script: + - . ./scripts/version.sh + - ./scripts/ci/prepare-apt-repo.sh + script: + - ./scripts/ci/create_debian_repo.sh debian bookworm + variables: + ARCHITECTURES: amd64 + GNUPGHOME: $CI_PROJECT_DIR/.gnupg + PREFIX: next + oc.docker-build-ubuntu-dependencies: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: images @@ -488,3 +507,22 @@ apt_repo_ubuntu_current: variables: ARCHITECTURES: amd64 GNUPGHOME: $CI_PROJECT_DIR/.gnupg + +apt_repo_ubuntu: + image: public.ecr.aws/lts/ubuntu:20.04_stable + stage: publishing + tags: + - gcp + needs: + - oc.build-ubuntu + dependencies: + - oc.build-ubuntu + before_script: + - . ./scripts/version.sh + - ./scripts/ci/prepare-apt-repo.sh + script: + - ./scripts/ci/create_debian_repo.sh ubuntu focal jammy + variables: + ARCHITECTURES: amd64 + GNUPGHOME: $CI_PROJECT_DIR/.gnupg + PREFIX: next diff --git a/.gitlab/ci/pipelines/octez_beta_release_tag.yml b/.gitlab/ci/pipelines/octez_beta_release_tag.yml index cd0a32b21415..470126ba45f9 100644 --- a/.gitlab/ci/pipelines/octez_beta_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_beta_release_tag.yml @@ -367,6 +367,25 @@ apt_repo_debian_current: ARCHITECTURES: amd64 GNUPGHOME: $CI_PROJECT_DIR/.gnupg +apt_repo_debian: + image: debian:bookworm + stage: publishing + tags: + - gcp + needs: + - oc.build-debian + dependencies: + - oc.build-debian + before_script: + - . ./scripts/version.sh + - ./scripts/ci/prepare-apt-repo.sh + script: + - ./scripts/ci/create_debian_repo.sh debian bookworm + variables: + ARCHITECTURES: amd64 + GNUPGHOME: $CI_PROJECT_DIR/.gnupg + PREFIX: next + oc.docker-build-ubuntu-dependencies: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: images @@ -489,3 +508,22 @@ apt_repo_ubuntu_current: variables: ARCHITECTURES: amd64 GNUPGHOME: $CI_PROJECT_DIR/.gnupg + +apt_repo_ubuntu: + image: public.ecr.aws/lts/ubuntu:20.04_stable + stage: publishing + tags: + - gcp + needs: + - oc.build-ubuntu + dependencies: + - oc.build-ubuntu + before_script: + - . ./scripts/version.sh + - ./scripts/ci/prepare-apt-repo.sh + script: + - ./scripts/ci/create_debian_repo.sh ubuntu focal jammy + variables: + ARCHITECTURES: amd64 + GNUPGHOME: $CI_PROJECT_DIR/.gnupg + PREFIX: next diff --git a/.gitlab/ci/pipelines/octez_release_tag.yml b/.gitlab/ci/pipelines/octez_release_tag.yml index f49c61be65fd..f4b6ec7c7d74 100644 --- a/.gitlab/ci/pipelines/octez_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_release_tag.yml @@ -368,6 +368,25 @@ apt_repo_debian_current: ARCHITECTURES: amd64 GNUPGHOME: $CI_PROJECT_DIR/.gnupg +apt_repo_debian: + image: debian:bookworm + stage: publishing + tags: + - gcp + needs: + - oc.build-debian + dependencies: + - oc.build-debian + before_script: + - . ./scripts/version.sh + - ./scripts/ci/prepare-apt-repo.sh + script: + - ./scripts/ci/create_debian_repo.sh debian bookworm + variables: + ARCHITECTURES: amd64 + GNUPGHOME: $CI_PROJECT_DIR/.gnupg + PREFIX: next + oc.docker-build-ubuntu-dependencies: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: images @@ -491,6 +510,25 @@ apt_repo_ubuntu_current: ARCHITECTURES: amd64 GNUPGHOME: $CI_PROJECT_DIR/.gnupg +apt_repo_ubuntu: + image: public.ecr.aws/lts/ubuntu:20.04_stable + stage: publishing + tags: + - gcp + needs: + - oc.build-ubuntu + dependencies: + - oc.build-ubuntu + before_script: + - . ./scripts/version.sh + - ./scripts/ci/prepare-apt-repo.sh + script: + - ./scripts/ci/create_debian_repo.sh ubuntu focal jammy + variables: + ARCHITECTURES: amd64 + GNUPGHOME: $CI_PROJECT_DIR/.gnupg + PREFIX: next + opam:release: image: ${ci_image_name}/test:${ci_image_tag} stage: publish_release diff --git a/.gitlab/ci/pipelines/octez_release_tag_test.yml b/.gitlab/ci/pipelines/octez_release_tag_test.yml index a720bbf0f90b..83dab900c895 100644 --- a/.gitlab/ci/pipelines/octez_release_tag_test.yml +++ b/.gitlab/ci/pipelines/octez_release_tag_test.yml @@ -368,6 +368,25 @@ apt_repo_debian_current: ARCHITECTURES: amd64 GNUPGHOME: $CI_PROJECT_DIR/.gnupg +apt_repo_debian: + image: debian:bookworm + stage: publishing + tags: + - gcp + needs: + - oc.build-debian + dependencies: + - oc.build-debian + before_script: + - . ./scripts/version.sh + - ./scripts/ci/prepare-apt-repo.sh + script: + - ./scripts/ci/create_debian_repo.sh debian bookworm + variables: + ARCHITECTURES: amd64 + GNUPGHOME: $CI_PROJECT_DIR/.gnupg + PREFIX: next + oc.docker-build-ubuntu-dependencies: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: images @@ -491,6 +510,25 @@ apt_repo_ubuntu_current: ARCHITECTURES: amd64 GNUPGHOME: $CI_PROJECT_DIR/.gnupg +apt_repo_ubuntu: + image: public.ecr.aws/lts/ubuntu:20.04_stable + stage: publishing + tags: + - gcp + needs: + - oc.build-ubuntu + dependencies: + - oc.build-ubuntu + before_script: + - . ./scripts/version.sh + - ./scripts/ci/prepare-apt-repo.sh + script: + - ./scripts/ci/create_debian_repo.sh ubuntu focal jammy + variables: + ARCHITECTURES: amd64 + GNUPGHOME: $CI_PROJECT_DIR/.gnupg + PREFIX: next + docker:promote_to_latest: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 stage: publish_release diff --git a/ci/bin/debian_repository.ml b/ci/bin/debian_repository.ml index 7a0c99d1b6f9..e6121dd62318 100644 --- a/ci/bin/debian_repository.ml +++ b/ci/bin/debian_repository.ml @@ -45,13 +45,15 @@ let ubuntu_package_release_matrix = function (* Push .deb artifacts to storagecloud apt repository. *) let job_apt_repo ?rules ~__POS__ ~name ?(stage = Stages.publishing) - ?dependencies ?(archs = [Amd64]) ~image script : tezos_job = + ?(prefix = false) ?dependencies ?(archs = [Amd64]) ~image script : tezos_job + = let variables = [ ( "ARCHITECTURES", String.concat " " (List.map Tezos_ci.arch_to_string_alt archs) ); ("GNUPGHOME", "$CI_PROJECT_DIR/.gnupg"); ] + @ if prefix then [("PREFIX", "next")] else [] in job ?rules @@ -193,7 +195,7 @@ let jobs pipeline_type = ~matrix:(ubuntu_package_release_matrix pipeline_type) in - (* These create the apt repository for the current packages *) + (* These jobs create the apt repository for the current packages *) let job_apt_repo_debian_current = job_apt_repo ~__POS__ @@ -220,7 +222,25 @@ let jobs pipeline_type = ~image:Images.ubuntu_focal ["./scripts/ci/create_debian_repo.sh ubuntu focal jammy"] in - + (* These jobs create the apt repository for the next packages *) + let job_apt_repo_debian = + job_apt_repo + ~__POS__ + ~name:"apt_repo_debian" + ~prefix:true + ~dependencies:(Dependent [Artifacts job_build_debian_package]) + ~image:Images.debian_bookworm + ["./scripts/ci/create_debian_repo.sh debian bookworm"] + in + let job_apt_repo_ubuntu = + job_apt_repo + ~__POS__ + ~name:"apt_repo_ubuntu" + ~prefix:true + ~dependencies:(Dependent [Artifacts job_build_ubuntu_package]) + ~image:Images.ubuntu_focal + ["./scripts/ci/create_debian_repo.sh ubuntu focal jammy"] + in (* These test the installability of the current packages *) let job_install_bin ~__POS__ ~name ~dependencies ~image ?allow_failure script = @@ -298,6 +318,7 @@ let jobs pipeline_type = job_build_debian_package_current_a; job_build_debian_package_current_b; job_apt_repo_debian_current; + job_apt_repo_debian; ] in let ubuntu_jobs = @@ -307,6 +328,7 @@ let jobs pipeline_type = job_build_ubuntu_package_current_a; job_build_ubuntu_package_current_b; job_apt_repo_ubuntu_current; + job_apt_repo_ubuntu; ] in match pipeline_type with diff --git a/scripts/ci/create_debian_repo.sh b/scripts/ci/create_debian_repo.sh index 4ab6de6fe17f..2531d7582492 100755 --- a/scripts/ci/create_debian_repo.sh +++ b/scripts/ci/create_debian_repo.sh @@ -34,6 +34,9 @@ fi ARCHITECTURES=${ARCHITECTURES:-"amd64"} +#The prefix used for these packages in the repository. E.g. 'next' +PREFIX=${PREFIX:-""} + # The linux distribution for which we are creating the apt repository # E.g. 'ubuntu' or 'debian' DISTRIBUTION=${1} @@ -64,10 +67,10 @@ if [ -n "${gitlab_release_no_v:-}" ]; then # It a release tag, this can be either a real or test release if [ -n "${gitlab_release_rc_version}" ]; then # Release candidate - TARGETDIR="public/RC/$DISTRIBUTION" + TARGETDIR="public/$PREFIX/RC/$DISTRIBUTION" else # Release - TARGETDIR="public/$DISTRIBUTION" + TARGETDIR="public/$PREFIX/$DISTRIBUTION" fi else # Not a release tag. This is strictly for testing @@ -81,13 +84,13 @@ else exit 1 else # Branch is not protected, this is for testing ordinary MRs - TARGETDIR="public/$CI_COMMIT_REF_NAME/$DISTRIBUTION" + TARGETDIR="public/$PREFIX/$CI_COMMIT_REF_NAME/$DISTRIBUTION" fi else # For protected branches that are not release, we allow # a repository only for master. if [ "$CI_COMMIT_REF_NAME" = "master" ]; then - TARGETDIR="public/master/$DISTRIBUTION" + TARGETDIR="public/$PREFIX/master/$DISTRIBUTION" else echo "Cannot create a repository for a protected branch that \ is not associated with a release tag or it's master" @@ -103,6 +106,7 @@ mkdir -p "$TARGETDIR/dists" for architecture in $ARCHITECTURES; do # amd64, arm64 ... for release in $RELEASES; do # unstable, jammy, focal ... echo "Setting up APT repository for $DISTRIBUTION / $release / $architecture" + echo "targetdir: $TARGETDIR" # create the apt repository root directory and copy the public key cp scripts/packaging/package-signing-key.asc "$TARGETDIR/octez.asc" @@ -157,7 +161,3 @@ export GOOGLE_OAUTH_ACCESS_TOKEN echo "Push to $BUCKET" gsutil -m cp -r public/* gs://"${BUCKET}" - -echo "Check the content of the bucket gs://${BUCKET}" - -gcloud storage ls -R gs://"${BUCKET}/*" -- GitLab