diff --git a/.gitlab/ci/jobs/build/bin_packages_deb_dependencies.sh b/.gitlab/ci/jobs/build/bin_packages_deb_dependencies.sh new file mode 100755 index 0000000000000000000000000000000000000000..ec1a5328da2d0c467ef918907777f25592bd99df --- /dev/null +++ b/.gitlab/ci/jobs/build/bin_packages_deb_dependencies.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +export DEBIAN_FRONTEND=noninteractive + +apt update +apt-get install -y rsync git m4 build-essential patch unzip wget \ + opam jq bc autoconf cmake libev-dev libffi-dev libgmp-dev \ + libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev \ + protobuf-compiler libsqlite3-dev jq devscripts diff --git a/.gitlab/ci/jobs/build/bin_packages_rpm.sh b/.gitlab/ci/jobs/build/bin_packages_rpm_dependencies.sh similarity index 100% rename from .gitlab/ci/jobs/build/bin_packages_rpm.sh rename to .gitlab/ci/jobs/build/bin_packages_rpm_dependencies.sh diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index f1a88fda6e0b7d8a61bd9dd3c15246c676598feb..26d1ccfda2d885e3f11173eb61648b7f74cdad87 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -7435,12 +7435,11 @@ oc.install_bin_ubuntu_focal: needs: - trigger dependencies: [] - allow_failure: true script: - - ./docs/introduction/install-bin-ubuntu.sh + - ./docs/introduction/install-bin-deb.sh ubuntu focal -oc.install_bin_ubuntu_jammy: - image: public.ecr.aws/lts/ubuntu:22.04_stable +oc.install_bin_debian_bookworm: + image: debian:bookworm stage: test tags: - gcp @@ -7454,11 +7453,10 @@ oc.install_bin_ubuntu_jammy: needs: - trigger dependencies: [] - allow_failure: true script: - - ./docs/introduction/install-bin-ubuntu.sh + - ./docs/introduction/install-bin-deb.sh debian bookworm -oc.install_bin_rc_ubuntu_focal: +oc.install_bin_ubuntu_focal_rc: image: public.ecr.aws/lts/ubuntu:20.04_stable stage: test tags: @@ -7473,12 +7471,11 @@ oc.install_bin_rc_ubuntu_focal: needs: - trigger dependencies: [] - allow_failure: true script: - - ./docs/introduction/install-bin-ubuntu.sh rc + - ./docs/introduction/install-bin-deb.sh ubuntu focal rc -oc.install_bin_rc_ubuntu_jammy: - image: public.ecr.aws/lts/ubuntu:22.04_stable +oc.install_bin_debian_bookworm_rc: + image: debian:bookworm stage: test tags: - gcp @@ -7492,9 +7489,8 @@ oc.install_bin_rc_ubuntu_jammy: needs: - trigger dependencies: [] - allow_failure: true script: - - ./docs/introduction/install-bin-ubuntu.sh rc + - ./docs/introduction/install-bin-deb.sh debian bookworm rc oc.install_opam_focal: image: ocaml/opam:ubuntu-20.04 @@ -8395,11 +8391,7 @@ oc.build:dpkg:amd64: dependencies: [] before_script: - . ./scripts/version.sh - - export DEBIAN_FRONTEND=noninteractive - - apt update - - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf - cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev - protobuf-compiler libsqlite3-dev jq + - .gitlab/ci/jobs/build/bin_packages_deb_dependencies.sh script: - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh @@ -8445,7 +8437,7 @@ oc.build:rpm:amd64: dependencies: [] before_script: - . ./scripts/version.sh - - .gitlab/ci/jobs/build/bin_packages_rpm.sh + - .gitlab/ci/jobs/build/bin_packages_rpm_dependencies.sh script: - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh diff --git a/.gitlab/ci/pipelines/non_release_tag.yml b/.gitlab/ci/pipelines/non_release_tag.yml index 98e6a9b0ba47501c952867601ae49450a96dce9f..5f5546332e7604b96846b64efb15f9ef50be0d21 100644 --- a/.gitlab/ci/pipelines/non_release_tag.yml +++ b/.gitlab/ci/pipelines/non_release_tag.yml @@ -84,6 +84,25 @@ oc.docker:arm64: EXECUTABLE_FILES: script-inputs/released-executables rust_toolchain_image_tag: is-never-pulled +docker:merge_manifests: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: prepare_release + tags: + - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "true" + oc.build:dpkg:amd64: image: $DISTRIBUTION stage: build @@ -93,11 +112,7 @@ oc.build:dpkg:amd64: dependencies: [] before_script: - . ./scripts/version.sh - - export DEBIAN_FRONTEND=noninteractive - - apt update - - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf - cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev - protobuf-compiler libsqlite3-dev jq + - .gitlab/ci/jobs/build/bin_packages_deb_dependencies.sh script: - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh @@ -140,7 +155,7 @@ oc.build:rpm:amd64: dependencies: [] before_script: - . ./scripts/version.sh - - .gitlab/ci/jobs/build/bin_packages_rpm.sh + - .gitlab/ci/jobs/build/bin_packages_rpm_dependencies.sh script: - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh @@ -173,25 +188,6 @@ oc.build:rpm:amd64: - fedora:39 - rockylinux:9.3 -docker:merge_manifests: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 - stage: prepare_release - tags: - - gcp - needs: - - oc.docker:amd64 - - oc.docker:arm64 - dependencies: [] - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - ./scripts/ci/docker_merge_manifests.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - CI_DOCKER_HUB: "true" - gitlab:publish: image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.6.0 stage: publish_package_gitlab diff --git a/.gitlab/ci/pipelines/non_release_tag_test.yml b/.gitlab/ci/pipelines/non_release_tag_test.yml index 7b13790209a61f537f28b437902f0e6e2de226f8..2ba4ef7ff8f9492287544e9c32858697d922348e 100644 --- a/.gitlab/ci/pipelines/non_release_tag_test.yml +++ b/.gitlab/ci/pipelines/non_release_tag_test.yml @@ -84,6 +84,25 @@ oc.docker:arm64: EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables rust_toolchain_image_tag: is-never-pulled +docker:merge_manifests: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: prepare_release + tags: + - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + oc.build:dpkg:amd64: image: $DISTRIBUTION stage: build @@ -93,11 +112,7 @@ oc.build:dpkg:amd64: dependencies: [] before_script: - . ./scripts/version.sh - - export DEBIAN_FRONTEND=noninteractive - - apt update - - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf - cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev - protobuf-compiler libsqlite3-dev jq + - .gitlab/ci/jobs/build/bin_packages_deb_dependencies.sh script: - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh @@ -140,7 +155,7 @@ oc.build:rpm:amd64: dependencies: [] before_script: - . ./scripts/version.sh - - .gitlab/ci/jobs/build/bin_packages_rpm.sh + - .gitlab/ci/jobs/build/bin_packages_rpm_dependencies.sh script: - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh @@ -173,25 +188,6 @@ oc.build:rpm:amd64: - fedora:39 - rockylinux:9.3 -docker:merge_manifests: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 - stage: prepare_release - tags: - - gcp - needs: - - oc.docker:amd64 - - oc.docker:arm64 - dependencies: [] - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - ./scripts/ci/docker_merge_manifests.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - CI_DOCKER_HUB: "false" - gitlab:publish: image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.6.0 stage: publish_package_gitlab diff --git a/.gitlab/ci/pipelines/octez_beta_release_tag.yml b/.gitlab/ci/pipelines/octez_beta_release_tag.yml index 70b7c19cc5f0bc1bd740023e3e8ff8ee35b80876..6985d7f69e5bb2a450403c203526b249a1a090fb 100644 --- a/.gitlab/ci/pipelines/octez_beta_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_beta_release_tag.yml @@ -84,6 +84,25 @@ oc.docker:arm64: EXECUTABLE_FILES: script-inputs/released-executables rust_toolchain_image_tag: is-never-pulled +docker:merge_manifests: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: prepare_release + tags: + - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "true" + oc.build:dpkg:amd64: image: $DISTRIBUTION stage: build @@ -93,11 +112,7 @@ oc.build:dpkg:amd64: dependencies: [] before_script: - . ./scripts/version.sh - - export DEBIAN_FRONTEND=noninteractive - - apt update - - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf - cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev - protobuf-compiler libsqlite3-dev jq + - .gitlab/ci/jobs/build/bin_packages_deb_dependencies.sh script: - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh @@ -140,7 +155,7 @@ oc.build:rpm:amd64: dependencies: [] before_script: - . ./scripts/version.sh - - .gitlab/ci/jobs/build/bin_packages_rpm.sh + - .gitlab/ci/jobs/build/bin_packages_rpm_dependencies.sh script: - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh @@ -173,25 +188,6 @@ oc.build:rpm:amd64: - fedora:39 - rockylinux:9.3 -docker:merge_manifests: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 - stage: prepare_release - tags: - - gcp - needs: - - oc.docker:amd64 - - oc.docker:arm64 - dependencies: [] - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - ./scripts/ci/docker_merge_manifests.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - CI_DOCKER_HUB: "true" - gitlab:release: image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.6.0 stage: publish_release_gitlab diff --git a/.gitlab/ci/pipelines/octez_release_tag.yml b/.gitlab/ci/pipelines/octez_release_tag.yml index 8b14ac603d7daba26bc24562a491c1fe17d7f7de..87ac48e06a805bce40eb5b532133bb9e0754f603 100644 --- a/.gitlab/ci/pipelines/octez_release_tag.yml +++ b/.gitlab/ci/pipelines/octez_release_tag.yml @@ -85,6 +85,25 @@ oc.docker:arm64: EXECUTABLE_FILES: script-inputs/released-executables rust_toolchain_image_tag: is-never-pulled +docker:merge_manifests: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: prepare_release + tags: + - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "true" + oc.build:dpkg:amd64: image: $DISTRIBUTION stage: build @@ -94,11 +113,7 @@ oc.build:dpkg:amd64: dependencies: [] before_script: - . ./scripts/version.sh - - export DEBIAN_FRONTEND=noninteractive - - apt update - - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf - cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev - protobuf-compiler libsqlite3-dev jq + - .gitlab/ci/jobs/build/bin_packages_deb_dependencies.sh script: - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh @@ -141,7 +156,7 @@ oc.build:rpm:amd64: dependencies: [] before_script: - . ./scripts/version.sh - - .gitlab/ci/jobs/build/bin_packages_rpm.sh + - .gitlab/ci/jobs/build/bin_packages_rpm_dependencies.sh script: - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh @@ -174,25 +189,6 @@ oc.build:rpm:amd64: - fedora:39 - rockylinux:9.3 -docker:merge_manifests: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 - stage: prepare_release - tags: - - gcp - needs: - - oc.docker:amd64 - - oc.docker:arm64 - dependencies: [] - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - ./scripts/ci/docker_merge_manifests.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - CI_DOCKER_HUB: "true" - gitlab:release: image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.6.0 stage: publish_release_gitlab diff --git a/.gitlab/ci/pipelines/octez_release_tag_test.yml b/.gitlab/ci/pipelines/octez_release_tag_test.yml index bcf83daba098923248e77ad6521a237ee89c4b0e..c59d38827f00b9cede6b767d3df0edc0747ec129 100644 --- a/.gitlab/ci/pipelines/octez_release_tag_test.yml +++ b/.gitlab/ci/pipelines/octez_release_tag_test.yml @@ -84,6 +84,25 @@ oc.docker:arm64: EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables rust_toolchain_image_tag: is-never-pulled +docker:merge_manifests: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: prepare_release + tags: + - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + oc.build:dpkg:amd64: image: $DISTRIBUTION stage: build @@ -93,11 +112,7 @@ oc.build:dpkg:amd64: dependencies: [] before_script: - . ./scripts/version.sh - - export DEBIAN_FRONTEND=noninteractive - - apt update - - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf - cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev - protobuf-compiler libsqlite3-dev jq + - .gitlab/ci/jobs/build/bin_packages_deb_dependencies.sh script: - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh @@ -140,7 +155,7 @@ oc.build:rpm:amd64: dependencies: [] before_script: - . ./scripts/version.sh - - .gitlab/ci/jobs/build/bin_packages_rpm.sh + - .gitlab/ci/jobs/build/bin_packages_rpm_dependencies.sh script: - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh @@ -173,25 +188,6 @@ oc.build:rpm:amd64: - fedora:39 - rockylinux:9.3 -docker:merge_manifests: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 - stage: prepare_release - tags: - - gcp - needs: - - oc.docker:amd64 - - oc.docker:arm64 - dependencies: [] - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - ./scripts/ci/docker_merge_manifests.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - CI_DOCKER_HUB: "false" - gitlab:release: image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.6.0 stage: publish_release_gitlab @@ -211,3 +207,39 @@ gitlab:release: script: - ./scripts/ci/restrict_export_to_octez_source.sh - ./scripts/ci/gitlab-release.sh + +apt_repo_debian_bookworm: + image: debian:bookworm + stage: build + tags: + - gcp + needs: + - oc.build:dpkg:amd64 + dependencies: + - oc.build:dpkg:amd64 + before_script: + - . ./scripts/version.sh + - ./scripts/ci/apt-repo-before.sh + script: + - ./scripts/ci/create_debian_repo.sh debian bookworm + variables: + ARCHITECTURES: amd64 + GNUPGHOME: $CI_PROJECT_DIR/.gnupg + +apt_repo_ubuntu_focal: + image: public.ecr.aws/lts/ubuntu:20.04_stable + stage: build + tags: + - gcp + needs: + - oc.build:dpkg:amd64 + dependencies: + - oc.build:dpkg:amd64 + before_script: + - . ./scripts/version.sh + - ./scripts/ci/apt-repo-before.sh + script: + - ./scripts/ci/create_debian_repo.sh ubuntu focal + variables: + ARCHITECTURES: amd64 + GNUPGHOME: $CI_PROJECT_DIR/.gnupg diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index a29ed3abc5211ff533dabd4b931d5a511fe09f03..1745cc0171e1b21b89afa4b7ee686b0a1fb2d1e3 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -325,11 +325,7 @@ oc.build:dpkg:amd64: dependencies: [] before_script: - . ./scripts/version.sh - - export DEBIAN_FRONTEND=noninteractive - - apt update - - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf - cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev - protobuf-compiler libsqlite3-dev jq + - .gitlab/ci/jobs/build/bin_packages_deb_dependencies.sh script: - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh @@ -372,7 +368,7 @@ oc.build:rpm:amd64: dependencies: [] before_script: - . ./scripts/version.sh - - .gitlab/ci/jobs/build/bin_packages_rpm.sh + - .gitlab/ci/jobs/build/bin_packages_rpm_dependencies.sh script: - wget https://sh.rustup.rs/rustup-init.sh - chmod +x rustup-init.sh @@ -7144,23 +7140,21 @@ oc.install_bin_ubuntu_focal: rules: - when: always dependencies: [] - allow_failure: true script: - - ./docs/introduction/install-bin-ubuntu.sh + - ./docs/introduction/install-bin-deb.sh ubuntu focal -oc.install_bin_ubuntu_jammy: - image: public.ecr.aws/lts/ubuntu:22.04_stable +oc.install_bin_debian_bookworm: + image: debian:bookworm stage: test tags: - gcp rules: - when: always dependencies: [] - allow_failure: true script: - - ./docs/introduction/install-bin-ubuntu.sh + - ./docs/introduction/install-bin-deb.sh debian bookworm -oc.install_bin_rc_ubuntu_focal: +oc.install_bin_ubuntu_focal_rc: image: public.ecr.aws/lts/ubuntu:20.04_stable stage: test tags: @@ -7168,21 +7162,19 @@ oc.install_bin_rc_ubuntu_focal: rules: - when: always dependencies: [] - allow_failure: true script: - - ./docs/introduction/install-bin-ubuntu.sh rc + - ./docs/introduction/install-bin-deb.sh ubuntu focal rc -oc.install_bin_rc_ubuntu_jammy: - image: public.ecr.aws/lts/ubuntu:22.04_stable +oc.install_bin_debian_bookworm_rc: + image: debian:bookworm stage: test tags: - gcp rules: - when: always dependencies: [] - allow_failure: true script: - - ./docs/introduction/install-bin-ubuntu.sh rc + - ./docs/introduction/install-bin-deb.sh debian bookworm rc oc.install_opam_focal: image: ocaml/opam:ubuntu-20.04 diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index f5d800d85f257483903d591a4df1a82ef5c18c5e..4164867659525aa27e274a29008b6b3f1f218c53 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -150,17 +150,19 @@ let read_opam_packages = Some {name; group; batch_index} | _ -> fail () -(* These are the set of Linux distributions and their release for - which we test installation of the deprecated Serokell PPA binary - packages. *) -type install_octez_distribution = Ubuntu_focal | Ubuntu_jammy | Fedora_37 +(* These are the type of distribution tested for the pre-v19.1 packages. + There is a second set of debian packages not released that are going to + deprecate and replace these packages. + See: https://gitlab.com/tezos/tezos/-/milestones/322#tab-issues +*) +type install_octez_distribution = Ubuntu_focal | Debian_bookworm | Fedora_37 let image_of_distribution = function | Ubuntu_focal -> Images.ubuntu_focal - | Ubuntu_jammy -> Images.ubuntu_jammy + | Debian_bookworm -> Images.debian_bookworm | Fedora_37 -> Images.fedora_37 -let job_tezt ~__POS__ ?rules ?parallel ?(tags = ["gcp_tezt"]) ~name +let job_tezt ~__POS__ ?rules ?parallel ?(tag = Gcp_tezt) ~name ~(tezt_tests : Tezt_core.TSL_AST.t) ?(retry = 2) ?(tezt_retry = 1) ?(tezt_parallel = 1) ?(tezt_variant = "") ?(before_script = before_script ~source_version:true ~eval_opam:true []) @@ -217,7 +219,7 @@ let job_tezt ~__POS__ ?rules ?parallel ?(tags = ["gcp_tezt"]) ~name ~image:Images.runtime_e2etest_dependencies ~name ?parallel - ~tags + ~tag ~stage:Stages.test ?rules ~artifacts @@ -1184,17 +1186,20 @@ let jobs pipeline_type = let install_octez_rules = make_rules ~changes:changeset_install_jobs ~manual:Yes () in - (* Test installation of the deprecated Serokell PPA binary packages. *) + (* Test installation of the pre-v19.1 deb and rpm binary packages. *) let job_install_bin ~__POS__ ~name ?allow_failure ?(rc = false) distribution = - let distribution_string = - match distribution with - | Ubuntu_focal | Ubuntu_jammy -> "ubuntu" - | Fedora_37 -> "fedora" - in let script = - sf "./docs/introduction/install-bin-%s.sh" distribution_string - ^ if rc then " rc" else "" + match distribution with + | Ubuntu_focal -> + sf "./docs/introduction/install-bin-deb.sh ubuntu focal" + ^ if rc then " rc" else "" + | Debian_bookworm -> + sf "./docs/introduction/install-bin-deb.sh debian bookworm" + ^ if rc then " rc" else "" + | Fedora_37 -> + sf "./docs/introduction/install-bin-fedora.sh" + ^ if rc then " rc" else "" in job ?allow_failure @@ -1239,32 +1244,24 @@ let jobs pipeline_type = ~name:"oc.install_bin_rc_fedora_37" ~rc:true Fedora_37; - (* The Ubuntu jobs currently fail because the last rc packages can't be installed anymore. - See https://gitlab.com/tezos/tezos/-/issues/6902. - TODO: https://gitlab.com/tezos/tezos/-/issues/6915 - This should be removed after the next release candidate. *) job_install_bin ~__POS__ ~name:"oc.install_bin_ubuntu_focal" - ~allow_failure:Yes Ubuntu_focal; job_install_bin ~__POS__ - ~name:"oc.install_bin_ubuntu_jammy" - ~allow_failure:Yes - Ubuntu_jammy; + ~name:"oc.install_bin_debian_bookworm" + Debian_bookworm; job_install_bin ~__POS__ - ~name:"oc.install_bin_rc_ubuntu_focal" - ~allow_failure:Yes + ~name:"oc.install_bin_ubuntu_focal_rc" ~rc:true Ubuntu_focal; job_install_bin ~__POS__ - ~name:"oc.install_bin_rc_ubuntu_jammy" - ~allow_failure:Yes + ~name:"oc.install_bin_debian_bookworm_rc" ~rc:true - Ubuntu_jammy; + Debian_bookworm; (* Test installing through opam *) job_install_opam_focal; (* Test compiling the [latest-release] branch on Bullseye *) @@ -1432,7 +1429,7 @@ let jobs pipeline_type = let tezt_static_binaries : tezos_job = job_tezt ~__POS__ - ~tags:["gcp"] + ~tag:Gcp ~name:"tezt:static-binaries" ~tezt_tests:(tezt_tests [Has_tag "cli"; Not (Has_tag "flaky")]) ~tezt_parallel:3 diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 15afe94ca1dc1e67704bc92830894dc24d7bbb4e..c94510c02cedc720b1f931e9ffcafafeaac0dce9 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -499,7 +499,7 @@ let changeset_test_evm_compatibility = authenticate with Docker Hub provided the environment variable [CI_DOCKER_AUTH] contains the appropriate credentials. *) let job_docker_authenticated ?(skip_docker_initialization = false) - ?ci_docker_hub ?artifacts ?(variables = []) ?rules ?dependencies ?arch ?tags + ?ci_docker_hub ?artifacts ?(variables = []) ?rules ?dependencies ?arch ?tag ?allow_failure ?parallel ~__POS__ ~stage ~name script : tezos_job = let docker_version = "24.0.6" in job @@ -507,7 +507,7 @@ let job_docker_authenticated ?(skip_docker_initialization = false) ?dependencies ?artifacts ?arch - ?tags + ?tag ?allow_failure ?parallel ~__POS__ @@ -763,16 +763,8 @@ let job_build_bin_package ?dependencies ?rules ~__POS__ ~name before_script ~source_version:true (match target with - | Dpkg -> - [ - "export DEBIAN_FRONTEND=noninteractive"; - "apt update"; - "apt-get install -y rsync git m4 build-essential patch unzip wget \ - opam jq bc autoconf cmake libev-dev libffi-dev libgmp-dev \ - libhidapi-dev pkg-config zlib1g-dev libprotobuf-dev \ - protobuf-compiler libsqlite3-dev jq"; - ] - | Rpm -> [".gitlab/ci/jobs/build/bin_packages_rpm.sh"]) + | Dpkg -> [".gitlab/ci/jobs/build/bin_packages_deb_dependencies.sh"] + | Rpm -> [".gitlab/ci/jobs/build/bin_packages_rpm_dependencies.sh"]) in job ?rules diff --git a/ci/bin/debian_repository.ml b/ci/bin/debian_repository.ml index 5d6e8022b6aeb28f2be83a06baa9b1d7e686964d..01dafee4db6909a44c8437b612bf22de0759df73 100644 --- a/ci/bin/debian_repository.ml +++ b/ci/bin/debian_repository.ml @@ -46,7 +46,7 @@ let jobs = ~stage:Stages.build ~variables:(variables [("DISTRIBUTION", distribution)]) ~parallel:(Matrix matrix) - ~tags:["$TAGS"] + ~tag:Dynamic [".gitlab/ci/jobs/packaging/build-debian-packages-dependencies.sh"] in let job_docker_build_debian_dependencies : tezos_job = @@ -71,7 +71,7 @@ let jobs = ~stage:Stages.packaging ~variables:(variables [("DISTRIBUTION", distribution)]) ~parallel:(Matrix matrix) - ~tags:["$TAGS"] + ~tag:Dynamic ~artifacts:(artifacts ["packages/$DISTRIBUTION/$RELEASE"]) [".gitlab/ci/jobs/packaging/build-debian-packages.sh"] in diff --git a/ci/bin/release_tag.ml b/ci/bin/release_tag.ml index a8df04b50d91f6f33fa2f5e1f4a045d529095154..8292cd9a64c57edf4c521de9440c795e6d3ba007 100644 --- a/ci/bin/release_tag.ml +++ b/ci/bin/release_tag.ml @@ -40,6 +40,28 @@ type release_tag_pipeline_type = | Beta_release_tag | Non_release_tag +(* push debian package artifacts to storagecloud apt repository *) +let job_apt_repo ?rules ~__POS__ ~name ?(stage = Stages.build) ?dependencies + ?(archs = [Amd64]) ~image script : tezos_job = + let variables = + [ + ( "ARCHITECTURES", + String.concat " " (List.map Tezos_ci.arch_to_string_alt archs) ); + ("GNUPGHOME", "$CI_PROJECT_DIR/.gnupg"); + ] + in + job + ?rules + ?dependencies + ~__POS__ + ~stage + ~name + ~image + ~before_script: + (before_script ~source_version:true ["./scripts/ci/apt-repo-before.sh"]) + ~variables + script + (** Create an Octez release tag pipeline of type {!release_tag_pipeline_type}. If [test] is true (default is [false]), then the Docker images are @@ -90,6 +112,22 @@ let octez_jobs ?(test = false) release_tag_pipeline_type = in let job_build_dpkg_amd64 = job_build_dpkg_amd64 () in let job_build_rpm_amd64 = job_build_rpm_amd64 () in + let job_apt_repo_ubuntu_focal = + job_apt_repo + ~__POS__ + ~name:"apt_repo_ubuntu_focal" + ~dependencies:(Dependent [Artifacts job_build_dpkg_amd64]) + ~image:Images.ubuntu_focal + ["./scripts/ci/create_debian_repo.sh ubuntu focal"] + in + let job_apt_repo_debian_bookworm = + job_apt_repo + ~__POS__ + ~name:"apt_repo_debian_bookworm" + ~dependencies:(Dependent [Artifacts job_build_dpkg_amd64]) + ~image:Images.debian_bookworm + ["./scripts/ci/create_debian_repo.sh debian bookworm"] + in let job_gitlab_release_or_publish = let dependencies = Dependent @@ -118,14 +156,16 @@ let octez_jobs ?(test = false) release_tag_pipeline_type = job_static_arm64_release; job_docker_amd64; job_docker_arm64; + job_docker_merge; job_build_dpkg_amd64; job_build_rpm_amd64; - job_docker_merge; job_gitlab_release_or_publish; ] @ match (test, release_tag_pipeline_type) with | false, Release_tag -> [job_opam_release] + | true, Release_tag -> + [job_apt_repo_debian_bookworm; job_apt_repo_ubuntu_focal] | _ -> [] (** Create an etherlink release tag pipeline of type {!release_tag_pipeline_type}. *) diff --git a/ci/bin/tezos_ci.ml b/ci/bin/tezos_ci.ml index 92cecdbd789d4e35880d38eb6d0fd0e5e62a1e0c..2ac1219ae6eb42fe8918b1ce99fbef460ec00642 100644 --- a/ci/bin/tezos_ci.ml +++ b/ci/bin/tezos_ci.ml @@ -480,6 +480,34 @@ let arch_to_string = function Amd64 -> "x86_64" | Arm64 -> "arm64" let arch_to_string_alt = function Amd64 -> "amd64" | Arm64 -> "arm64" +let dynamic_tag_var = Gitlab_ci.Var.make "TAGS" + +type tag = + | Gcp + | Gcp_arm64 + | Gcp_dev + | Gcp_dev_arm64 + | Gcp_tezt + | Gcp_tezt_dev + | Aws_specific + | Dynamic + +let tag_to_string = function + | Gcp -> "gcp" + | Gcp_arm64 -> "gcp_arm64" + | Gcp_dev -> "gcp_dev" + | Gcp_dev_arm64 -> "gcp_dev_arm64" + | Gcp_tezt -> "gcp_tezt" + | Gcp_tezt_dev -> "gcp_tezt_dev" + | Aws_specific -> "aws_specific" + | Dynamic -> Gitlab_ci.Var.encode dynamic_tag_var + +(** The architecture of the runner associated to a tag . *) +let arch_of_tag = function + | Gcp_arm64 | Gcp_dev_arm64 -> Some Arm64 + | Gcp | Gcp_dev | Gcp_tezt | Gcp_tezt_dev | Aws_specific -> Some Amd64 + | Dynamic -> None + type dependency = | Job of tezos_job | Optional of tezos_job @@ -534,23 +562,23 @@ let enc_git_strategy = function let job ?arch ?after_script ?allow_failure ?artifacts ?before_script ?cache ?interruptible ?(dependencies = Staged []) ?services ?variables ?rules - ?timeout ?tags ?git_strategy ?coverage ?retry ?parallel ~__POS__ ~image + ?timeout ?tag ?git_strategy ?coverage ?retry ?parallel ~__POS__ ~image ~stage ~name script : tezos_job = - let tags = - Some - (match (arch, tags) with - | Some arch, None -> - [(match arch with Amd64 -> "gcp" | Arm64 -> "gcp_arm64")] - | None, Some tags -> tags - | None, None -> - (* By default, we assume Amd64 runners as given by the [gcp] tag. *) - ["gcp"] - | Some _, Some _ -> - failwith - "[job] cannot specify both [arch] and [tags] at the same time in \ - job '%s'." - name) + (* The tezos/tezos CI uses singleton tags for its runners. *) + let tag = + match (arch, tag) with + | Some arch, None -> ( match arch with Amd64 -> Gcp | Arm64 -> Gcp_arm64) + | None, Some tag -> tag + | None, None -> + (* By default, we assume Amd64 runners as given by the [gcp] tag. *) + Gcp + | Some _, Some _ -> + failwith + "[job] cannot specify both [arch] and [tags] at the same time in job \ + '%s'." + name in + let tags = Some [tag_to_string tag] in (match (parallel : Gitlab_ci.Types.parallel option) with | Some (Vector n) when n < 2 -> failwith @@ -602,6 +630,15 @@ let job ?arch ?after_script ?allow_failure ?artifacts ?before_script ?cache retry name | _ -> ()) ; + (match + (Sys.getenv_opt Gitlab_ci.Predefined_vars.(show gitlab_user_login), tag) + with + | Some "nomadic-margebot", (Gcp_dev | Gcp_dev_arm64) -> + failwith + "[job] Attempting to merge a CI configuration using development \ + runners (job: %s)" + name + | _ -> ()) ; let job : Gitlab_ci.Types.job = { name; diff --git a/ci/bin/tezos_ci.mli b/ci/bin/tezos_ci.mli index 11f700837937e17a93e699b489a82b95c558e042..3b92291aa96b9f233761e5e6e39e26dd2e300f95 100644 --- a/ci/bin/tezos_ci.mli +++ b/ci/bin/tezos_ci.mli @@ -194,6 +194,33 @@ val arch_to_string : arch -> string (** Alternative string representation of architectures ([Amd64] is ["amd64"]) *) val arch_to_string_alt : arch -> string +(** The list of available runner tags. *) +type tag = + | Gcp (** GCP prod AMD64 runner, general purpose. *) + | Gcp_arm64 (** GCP prod ARM64 runner, general purpose. *) + | Gcp_dev (** GCP dev AMD64 runner, general purpose. *) + | Gcp_dev_arm64 (** GCP dev ARM64 runner, general purpose. *) + | Gcp_tezt + (** GCP prod AMD64 runner, suitable for tezt jobs (more RAM and CPU) *) + | Gcp_tezt_dev + (** GCP dev AMD64 runner, suitable for tezt jobs (more RAM and CPU) *) + | Aws_specific + (** AWS runners, in cases where a CI is legacy or not suitable for GCP. *) + | Dynamic + (** The runner is dynamically set through the CI variable {!dynamic_tag_var}. *) + +(** The variable to set enabling dynamic runner selection. + + To dynamically set the runner of a job through a CI/CD variable, + assign to this variable using [variables:] or [parallel:matrix:]. *) +val dynamic_tag_var : Gitlab_ci.Var.t + +(** The architecture of the runner associated to a tag if statically known. *) +val arch_of_tag : tag -> arch option + +(** The string representation of a tag. *) +val tag_to_string : tag -> string + (** A job dependency. - A job that depends on [Job j] will not start until [j] finishes. @@ -251,15 +278,15 @@ val enc_git_strategy : git_strategy -> string - Translates each {!dependency} to [needs:] and [dependencies:] keywords as detailed in the documentation of {!dependency}. - - Adds [tags:] based on [arch] and [tags]: - - - If only [arch] is set to [Amd64] (resp. [Arm64]) then the tag - ["gcp"] (resp ["gcp_arm64"]) is set. - - If only [tags] is set, then it is passed as is to the job's [tags:] - field. - - Setting both [arch] and [tags] throws an error. - - Omitting both [arch] and [tags] is equivalent to setting - [~arch:Amd64] and omitting [tags]. *) + - Adds [tag:] based on [arch] and [tag]: + + - If only [tag] is set, then it is passed as is to the job's [tags:] + field. The runners of the tezos/tezos CI all use singleton tags, + hence we only allow one tag per job. + - Setting both [arch] and [tag] throws an error. + - Omitting both [arch] and [tag] is equivalent to setting + [~tag:Gcp] or, equivalently, omitting tag and setting + [~arch:Amd64].*) val job : ?arch:arch -> ?after_script:string list -> @@ -273,7 +300,7 @@ val job : ?variables:Gitlab_ci.Types.variables -> ?rules:Gitlab_ci.Types.job_rule list -> ?timeout:Gitlab_ci.Types.time_interval -> - ?tags:string list -> + ?tag:tag -> ?git_strategy:git_strategy -> ?coverage:string -> ?retry:int -> diff --git a/ci/lib_gitlab_ci/var.ml b/ci/lib_gitlab_ci/var.ml index 1a696ab470f67e7b6a0f8ae161823fcb8b59c7b3..c940fb9c7a448578a5ca746150a238ceb711799d 100644 --- a/ci/lib_gitlab_ci/var.ml +++ b/ci/lib_gitlab_ci/var.ml @@ -9,8 +9,6 @@ open Base type t = string -let encode = Fun.id - let make variable_name = (* See {{:https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb#L9}string.rb}} @@ -23,4 +21,8 @@ let make variable_name = then raise (Invalid_argument (sf "[Var.t] invalid variable name '%s'" variable_name)) ; - "$" ^ variable_name + variable_name + +let name = Fun.id + +let encode variabble = "$" ^ variabble diff --git a/ci/lib_gitlab_ci/var.mli b/ci/lib_gitlab_ci/var.mli index b63c1383b7c6604a4d0ad1a7d93f4dd6debda6b3..2e7ff1742f5c2dcb927ddf6ca615a12416c9509d 100644 --- a/ci/lib_gitlab_ci/var.mli +++ b/ci/lib_gitlab_ci/var.mli @@ -27,3 +27,8 @@ val make : string -> t [encode @@ make "foo"] is ["$foo"]. *) val encode : t -> string + +(** The name of a variable. + + [encode @@ make "foo"] is ["foo"]. *) +val name : t -> string diff --git a/docs/Makefile b/docs/Makefile index 1086bb834246ec85b4f12b55ee62fa37b393419c..a7d4cb4464cb16cbd209996568daac6fecf4484a 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -127,11 +127,10 @@ xrefscheck: installcheck: scripts/test_install_scripts.sh install-bin-focal - scripts/test_install_scripts.sh install-bin-jammy + scripts/test_install_scripts.sh install-bin-rc-focal + scripts/test_install_scripts.sh install-bin-bookworm scripts/test_install_scripts.sh install-bin-fedora37 scripts/test_install_scripts.sh install-bin-fedora38 - scripts/test_install_scripts.sh install-bin-rc-focal - scripts/test_install_scripts.sh install-bin-rc-jammy scripts/test_install_scripts.sh install-bin-rc-fedora37 scripts/test_install_scripts.sh install-bin-rc-fedora38 scripts/test_install_scripts.sh install-opam-scratch diff --git a/docs/introduction/howtoget.rst b/docs/introduction/howtoget.rst index 18d2ea3c046553ea3dbfc59bdcfb0574a364bc51..d7d7602a45fa44e95396c70ed44269739d0e3c7f 100644 --- a/docs/introduction/howtoget.rst +++ b/docs/introduction/howtoget.rst @@ -84,6 +84,43 @@ There are several packages: - ``octez-smartrollup``: the Octez Smart Rollup daemons - ``octez-signer``: the remote signer, to hold keys on (and sign from) a different machine from the baker or client + +Ubuntu and Debian Octez packages +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you're using Ubuntu or Debian, you can also install packages with Octez binaries +using ``apt`` directly from an APT repository, instead of going to the Octez +release page as explained above. + +We support the following distribution/releases: +- ``debian/bookwom`` +- ``ubuntu/focal`` +- ``ubuntu/jammy`` + +both on ``amd64`` and ``arm64`` architectures. + +In order to add the Tezos package repository to your machine, do: + +- ``export distribution=debian`` +- ``export release=bookworm`` + +and run : + +.. literalinclude:: install-bin-deb.sh + :language: shell + :start-after: [add repository] + :end-before: [install tezos] + +We also maintain a separate repository for release candidates. To install +the last release candidate simply prepend ``RC`` to the distribution name +as in ``export distribution=RC/debian`` + +Then, to install the binaries, run the following commands: + +.. literalinclude:: install-bin-deb.sh + :language: shell + :start-after: [install tezos] + :end-before: [test executables] Also there are some experimental packages: - ``octez-experimental`` - binaries that are considered experimental including @@ -98,24 +135,11 @@ possible to configure the software to use a different user (even root). The documentation for these packages, originally developed by Chris Pinnock, can be found here: https://chrispinnock.com/tezos/packages/ -Ubuntu and Debian Octez packages -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If you're using Ubuntu or Debian, you can install packages with Octez binaries -using ``dpkg`` or ``apt``. Currently it supports the two latest LTS releases -for Ubuntu and for Debian, the stable and testing release. - -Upgrading to a newer release requires downloading again all the ``deb`` -packages and repeat the installation. - -For example using dpkg:: - - dpkg -i octez-client_19.1-1_arm64.deb - Fedora Octez packages ~~~~~~~~~~~~~~~~~~~~~ If you're using Fedora, you can install packages with Octez binaries +from the Octez release page indicated above using ``rpm`` or ``dnf``. Currently it supports the latest LTS release for Fedora and for RockyLinux. diff --git a/docs/introduction/install-bin-deb.sh b/docs/introduction/install-bin-deb.sh new file mode 100755 index 0000000000000000000000000000000000000000..731cc0d771b4e39f0dd3cf973b5967b743e96060 --- /dev/null +++ b/docs/introduction/install-bin-deb.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +distribution=$1 +release=$2 +rc=$3 + +if [ "$rc" = "rc" ]; then + distribution="RC/$distribution" +fi + +set -e +set -x + +# [install prerequisites] +sudo apt-get update +sudo apt-get install -y sudo gpg curl +# [add repository] +REPO="deb https://tezos-linux-repo-dev.storage.googleapis.com/$distribution $release main" +curl "https://tezos-linux-repo-dev.storage.googleapis.com/$distribution/octez.asc" | sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/octez.gpg +echo "$REPO" | sudo tee /etc/apt/sources.list.d/octez.list +sudo apt-get update + +# [install tezos] +sudo apt-get install -y octez-client +sudo apt-get install -y octez-node +sudo apt-get install -y octez-baker + +# [test executables] +octez-client --version +octez-node --version +octez-baker-Proxford --version +octez-accuser-Proxford --version diff --git a/docs/introduction/install-bin-ubuntu.sh b/docs/introduction/install-bin-ubuntu.sh deleted file mode 100755 index 510127c6d3a02a12f842d6c6786280736d859b94..0000000000000000000000000000000000000000 --- a/docs/introduction/install-bin-ubuntu.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/sh - -usage() { - cat >&2 << !EOF -usage: - $0 [rc] -!EOF -} - -if [ $# -eq 1 ] && [ "$1" = "rc" ]; then - # [setup rc repository] - REPO="ppa:serokell/tezos-rc" - # [end] -elif [ $# -eq 0 ]; then - # [setup stable repository] - REPO="ppa:serokell/tezos" - # [end] -else - usage - exit 1 -fi - -# TODO: https://gitlab.com/tezos/tezos/-/issues/6552 -# Tezos ubuntu packages cannot be installed in a k8s pod. -if [ -n "${FF_KUBERNETES_HONOR_ENTRYPOINT}" ]; then - echo "Container orchestrated by Kubernetes detected, need to create the file /.dockerenv." - echo "See https://github.com/serokell/tezos-packaging/issues/734 for further explanation." - echo "Note: Remove this workaround when the issue above is fixed." - touch /.dockerenv -fi -# TODO tezos/tezos#2170: search shifted protocol name/number & adapt -set -e -set -x -# [install prerequisites] -apt-get update -apt-get install sudo -apt-get install -y software-properties-common < /dev/null -# [install tezos] -sudo add-apt-repository -y $REPO && sudo apt-get update -sudo apt-get install -y tezos-client -sudo apt-get install -y tezos-node -sudo apt-get install -y tezos-baker-proxford -sudo apt-get install -y tezos-accuser-proxford -# [test executables] -octez-client --version -octez-node --version -octez-baker-Proxford --version -octez-accuser-Proxford --version diff --git a/docs/scripts/test_install_scripts.sh b/docs/scripts/test_install_scripts.sh index bac3c57c3c610238357ed1fe76f27f1323ce5b4d..3a951d990a9f8d7d777fbe71b20f306b44517d43 100755 --- a/docs/scripts/test_install_scripts.sh +++ b/docs/scripts/test_install_scripts.sh @@ -14,8 +14,8 @@ # Ubuntu Focal Fossa 20.04 LTS: UBUNTU_FOCAL=public.ecr.aws/lts/ubuntu:20.04_stable -# Ubuntu Ubuntu 22.04 LTS (Jammy Jellyfish): -UBUNTU_JAMMY=public.ecr.aws/lts/ubuntu:22.04_stable +# Debian stable +DEBIAN_BOOKWORM=debian/bookworm SCRIPT_DIR="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")" DOCS_DIR="$(dirname "$SCRIPT_DIR")" @@ -26,11 +26,10 @@ usage: $0 where can be: * install-bin-focal -* install-bin-jammy +* install-bin-rc-focal +* install-bin-bookworm * install-bin-fedora37 * install-bin-fedora38 -* install-bin-rc-focal -* install-bin-rc-jammy * install-bin-rc-fedora37 * install-bin-rc-fedora38 * install-opam-scratch @@ -54,10 +53,13 @@ fi for test_case in "$@"; do case "$test_case" in "install-bin-focal") - docker run --rm -i -v "$DOCS_DIR/introduction":/Scripts "$UBUNTU_FOCAL" /Scripts/install-bin-ubuntu.sh + docker run --rm -i -v "$DOCS_DIR/introduction":/Scripts "$UBUNTU_FOCAL" /Scripts/install-bin-deb.sh ubuntu focal + ;; + "install-bin-rc-focal") + docker run --rm -i -v "$DOCS_DIR/introduction":/Scripts "$UBUNTU_FOCAL" /Scripts/install-bin-deb.sh ubuntu focal rc ;; - "install-bin-jammy") - docker run --rm -i -v "$DOCS_DIR/introduction":/Scripts "$UBUNTU_JAMMY" /Scripts/install-bin-ubuntu.sh + "install-bin-bookworm") + docker run --rm -i -v "$DOCS_DIR/introduction":/Scripts "$DEBIAN_BOOKWORM" /Scripts/install-bin-deb.sh debian bookworm ;; "install-bin-fedora37") docker run --rm -i -v "$DOCS_DIR/introduction":/Scripts fedora:37 /Scripts/install-bin-fedora.sh @@ -65,12 +67,6 @@ for test_case in "$@"; do "install-bin-fedora38") docker run --rm -i -v "$DOCS_DIR/introduction":/Scripts fedora:38 /Scripts/install-bin-fedora.sh ;; - "install-bin-rc-focal") - docker run --rm -i -v "$DOCS_DIR/introduction":/Scripts "$UBUNTU_FOCAL" /Scripts/install-bin-ubuntu.sh rc - ;; - "install-bin-rc-jammy") - docker run --rm -i -v "$DOCS_DIR/introduction":/Scripts "$UBUNTU_JAMMY" /Scripts/install-bin-ubuntu.sh rc - ;; "install-bin-rc-fedora37") docker run --rm -i -v "$DOCS_DIR/introduction":/Scripts fedora:37 /Scripts/install-bin-fedora.sh rc ;; diff --git a/scripts/ci/apt-repo-before.sh b/scripts/ci/apt-repo-before.sh new file mode 100755 index 0000000000000000000000000000000000000000..699483e05f9a4760d9251b03654d1a78b9058f56 --- /dev/null +++ b/scripts/ci/apt-repo-before.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +# Install depedendencies for the apt_repo job + +export DEBIAN_FRONTEND=noninteractive + +apt-get update +apt install -y apt-utils debsigs gnupg curl + +# Install google-cloud-cli so we can upload packages to the Google Cloud Storage bucket. +gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg \ + scripts/packaging/apt-key-gcloud.gpg + +echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + +apt-get update +apt-get -y install google-cloud-cli diff --git a/scripts/ci/create_debian_repo.sh b/scripts/ci/create_debian_repo.sh new file mode 100755 index 0000000000000000000000000000000000000000..68112db66ef54cf8beb893cf6faf4a94778974eb --- /dev/null +++ b/scripts/ci/create_debian_repo.sh @@ -0,0 +1,112 @@ +#!/bin/sh + +set -eu + +# Create the APT repository for debian packages and sign it using +# the private key available as ENV variable. + +# uses : +# - scripts/packaging/Release.conf for release metadata +# - scripts/packaging/key.asc as the repository pub key + +# expected env vars +# - ARCHITECTURES +# - GNUPGHOME + +if [ $# -lt 2 ]; then + echo "$0 " +fi + +# shellcheck source=./scripts/ci/octez-release.sh +. ./scripts/ci/octez-release.sh + +ARCHITECTURES=${ARCHITECTURES:-"amd64"} +BRANCH=$CI_COMMIT_REF_NAME + +# E.g. 'ubuntu' or 'debian' +distribution=${1} +shift +RELEASES=$* + +# make available the private key for signing the release file +echo "$GPG_PRIVATE_KEY" | base64 --decode | gpg --batch --import -- + +BUCKET="$GCP_LINUX_PACKAGES_BUCKET" + +oldPWD=$PWD + +# if it's a release tag, then omit $BRANCH +if [ -n "${gitlab_release_no_v}" ]; then + # It a release tag, this can be either a real or fake release + if [ -n "${gitlab_release_rc_version}" ]; then + # Release candidate + TARGETDIR="public/RC/$distribution" + else + # Release + TARGETDIR="public/$distribution" + fi +else + # Not a release tag. This can be either a normal branch or + # a protected branch. + TARGETDIR="public/$BRANCH/$distribution" +fi + +mkdir -p "$TARGETDIR/dists" + +for architecture in $ARCHITECTURES; do # amd64, arm64 ... + for release in $RELEASES; do # unstable, jammy, focal ... + echo "Setting up APT repository for $distribution / $release / $architecture" + + # create the apt repository root directory and copy the public key + cp scripts/packaging/package-signing-key.asc "$TARGETDIR/octez.asc" + + target="dists/${release}/main/binary-${architecture}" + + mkdir -p "$TARGETDIR/${target}/" + cp -r packages/"${distribution}/${release}"/* "$TARGETDIR/${target}" + + for file in packages/"${distribution}/${release}"/*.deb; do + cp "$file" "$TARGETDIR/${target}/" + echo "Adding package $file to $TARGETDIR/${target}/" + done + + cd "$TARGETDIR" + echo "Create the Packages file" + apt-ftparchive packages "dists/${release}" > "${target}/Packages" + gzip -k -f "${target}/Packages" + + echo "Create the Release files using a static configuration file" + apt-ftparchive \ + -o APT::FTPArchive::Release::Codename="$release" \ + -o APT::FTPArchive::Release::Architectures="$architecture" \ + -c "$oldPWD/scripts/packaging/Release.conf" release \ + "dists/${release}/" > "dists/${release}/Release" + + # sign the release file using GPG. Since gpg is run in a script we need to set + # some variables and extra options to make it work. The InRelease file contains + # both the gpg signature and the content of the Release file. + echo "Sign the release file" + echo "$GPG_PASSPHRASE" | + gpg --batch --passphrase-fd 0 --pinentry-mode loopback \ + -u "$GPG_KEY_ID" --clearsign \ + -o "dists/${release}/InRelease" "dists/${release}/Release" + done + # back to base + cd "$oldPWD" +done + +if [ "$CI_COMMIT_REF_PROTECTED" = "true" ]; then + echo "### Logging into protected APT repository..." + echo "${GCP_PROTECTED_SERVICE_ACCOUNT}" | base64 -d > protected_sa.json + gcloud auth activate-service-account --key-file=protected_sa.json +else + echo "### Logging into standard APT repository..." + # Nothing to do +fi + +GOOGLE_OAUTH_ACCESS_TOKEN=$(gcloud auth print-access-token) +export GOOGLE_OAUTH_ACCESS_TOKEN + +echo "Push to $BUCKET" + +gsutil -m cp -r public/* gs://"${BUCKET}" diff --git a/scripts/install_build_deps.rust.sh b/scripts/install_build_deps.rust.sh index b14c225791c293fc9db6501404dd6db9f974dab0..25832a94c1dc9c47dd90a19e8d573a33c0953956 100755 --- a/scripts/install_build_deps.rust.sh +++ b/scripts/install_build_deps.rust.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -e +set -ex script_dir="$(cd "$(dirname "$0")" && pwd -P)" diff --git a/scripts/packaging/Readme.md b/scripts/packaging/Readme.md index 5a2d8b5dbbcdd264cf3a6c89997f0d37727b6039..3e90315d0f95acb6203ef776e0e66b799258c6c6 100644 --- a/scripts/packaging/Readme.md +++ b/scripts/packaging/Readme.md @@ -1,4 +1,3 @@ - # Debian packaging The `octez` directory contains the Debian package specification for Octez. @@ -46,3 +45,25 @@ same environment used in the CI. The script builds the Debian packages. The pre-requisite to this script is to run `make build-deps` and ensure all runtime dependencies as well as the build dependencies needed to build the Debian packages are correctly installed. + +# Apt Repository Management + +The file [package-signing-key.asc] contains the public key associated to +the gpg signing key stored in the CI. The CI / Release Manager will +sign the repository using this key, while the user can download +the public key to verify that the signature is indeed valid. + +This user must download the key and make it known to apt. + +Ex. + + curl "/octez.asc" \ + | sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/octez.gpg + + +### gutils + +We use the gutils suite to upload the apt repository to the +google cloud storage. The key used to install those packages +is committed in the repository and can be refreshed from this url +`https://packages.cloud.google.com/apt/doc/apt-key.gpg` diff --git a/scripts/packaging/Release.conf b/scripts/packaging/Release.conf new file mode 100644 index 0000000000000000000000000000000000000000..77ec79eace540e12ea575131410870d0c80a3066 --- /dev/null +++ b/scripts/packaging/Release.conf @@ -0,0 +1,5 @@ +APT::FTPArchive::Release::Origin "Octez"; +APT::FTPArchive::Release::Label "Octez Debian Repository"; +APT::FTPArchive::Release::Architectures "noarch amd64"; +APT::FTPArchive::Release::Components "main"; +APT::FTPArchive::Release::Description "Debian Repository for Octez"; diff --git a/scripts/packaging/apt-key-gcloud.gpg b/scripts/packaging/apt-key-gcloud.gpg new file mode 100644 index 0000000000000000000000000000000000000000..e6e856f0c806a82eea17bc2c563cf1c45b3e9db5 --- /dev/null +++ b/scripts/packaging/apt-key-gcloud.gpg @@ -0,0 +1,44 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQENBGKItdQBCADWmKTNZEYWgXy73FvKFY5fRro4tGNa4Be4TZW3wZpct9Cj8Ejy +kU7S9EPoJ3EdKpxFltHRu7QbDi6LWSNA4XxwnudQrYGxnxx6Ru1KBHFxHhLfWsvF +cGMwit/znpxtIt9UzqCm2YTEW5NUnzQ4rXYqVQK2FLG4weYJ5bKwkY+ZsnRJpzxd +HGJ0pBiqwkMT8bfQdJymUBown+SeuQ2HEqfjVMsIRe0dweD2PHWeWo9fTXsz1Q5a +biGckyOVyoN9//DgSvLUocUcZsrWvYPaN+o8lXTO3GYFGNVsx069rxarkeCjOpiQ +OWrQmywXISQudcusSgmmgfsRZYW7FDBy5MQrABEBAAG0UVJhcHR1cmUgQXV0b21h +dGljIFNpZ25pbmcgS2V5IChjbG91ZC1yYXB0dXJlLXNpZ25pbmcta2V5LTIwMjIt +MDMtMDctMDhfMDFfMDEucHViKYkBIgQTAQgAFgUCYoi11AkQtT3IDRPt7wUCGwMC +GQEAAMGoB/98QBNIIN3Q2D3aahrfkb6axd55zOwR0tnriuJRoPHoNuorOpCv9aWM +MvQACNWkxsvJxEF8OUbzhSYjAR534RDigjTetjK2i2wKLz/kJjZbuF4ZXMynCm40 +eVm1XZqU63U9XR2RxmXppyNpMqQO9LrzGEnNJuh23icaZY6no12axymxcle/+SCm +da8oDAfa0iyA2iyg/eU05buZv54MC6RB13QtS+8vOrKDGr7RYp/VYvQzYWm+ck6D +vlaVX6VB51BkLl23SQknyZIJBVPm8ttU65EyrrgG1jLLHFXDUqJ/RpNKq+PCzWiy +t4uy3AfXK89RczLu3uxiD0CQI0T31u/IuQENBGKItdQBCADIMMJdRcg0Phv7+CrZ +z3xRE8Fbz8AN+YCLigQeH0B9lijxkjAFr+thB0IrOu7ruwNY+mvdP6dAewUur+pJ +aIjEe+4s8JBEFb4BxJfBBPuEbGSxbi4OPEJuwT53TMJMEs7+gIxCCmwioTggTBp6 +JzDsT/cdBeyWCusCQwDWpqoYCoUWJLrUQ6dOlI7s6p+iIUNIamtyBCwb4izs27Hd +EpX8gvO9rEdtcb7399HyO3oD4gHgcuFiuZTpvWHdn9WYwPGM6npJNG7crtLnctTR +0cP9KutSPNzpySeAniHx8L9ebdD9tNPCWC+OtOcGRrcBeEznkYh1C4kzdP1ORm5u +pnknABEBAAGJAR8EGAEIABMFAmKItdQJELU9yA0T7e8FAhsMAABJmAgAhRPk/dFj +71bU/UTXrkEkZZzE9JzUgan/ttyRrV6QbFZABByf4pYjBj+yLKw3280//JWurKox +2uzEq1hdXPedRHICRuh1Fjd00otaQ+wGF3kY74zlWivB6Wp6tnL9STQ1oVYBUv7H +hSHoJ5shELyedxxHxurUgFAD+pbFXIiK8cnAHfXTJMcrmPpC+YWEC/DeqIyEcNPk +zRhtRSuERXcq1n+KJvMUAKMD/tezwvujzBaaSWapmdnGmtRjjL7IxUeGamVWOwLQ +bUr+34MwzdeJdcL8fav5LA8Uk0ulyeXdwiAK8FKQsixI+xZvz7HUs8ln4pZwGw/T +pvO9cMkHogtgzZkBDQRgkbezAQgA5GCRx0EKC+rSq1vy25n0fZY8+4m9mlp6OCTt +1SkLy8I8lDD6av0l1zDp8fI18IFos6T8UGA0SdEkF0vVCydYV0S/zoDJ2QGL2A3l +dowZyrACBHYhv3tapvD+FvaqViXPoTauxTk9d0cxlkcee0nS1kl6NCnmN/K/Zb44 +zpk/3LjnJo8JQ0/V2H/0UjvsifwLMjHQK/mWw3kFHfR2CYj3SNOJRmhjNNjIwzJ8 +fpqJ3PsueLfmfq8tVrUHc6ELfXR5SD5VdbUfsVeQxx7HowmcbvU1s80pS+cHwQXh +M+0fziM4rxiaVkHSc3ftkA10kYPatl2Fj+WVbUoI1VSYzZW+mQARAQABtFRBcnRp +ZmFjdCBSZWdpc3RyeSBSZXBvc2l0b3J5IFNpZ25lciA8YXJ0aWZhY3QtcmVnaXN0 +cnktcmVwb3NpdG9yeS1zaWduZXJAZ29vZ2xlLmNvbT6JAU4EEwEKADgWIQQ1uqCz +Pp6zlvWcqDjAulzm3GMVowUCYJG3swIbAwULCQgHAgYVCgkICwIEFgIDAQIeAQIX +gAAKCRDAulzm3GMVo/ooCADBYeg6wGDHqvbG2dWRuqADK4p1IXhkGxKnu+pyA0Db +GZ4Q8GdsFqoFQuw4DjKpYUJjps5uzOjc5qtnbz8Kt8QtjniPX0Ms40+9nXgU8yz+ +zyaJPTyRTjHS3yC0rFJ5jLIXkLeA1DtI2AF9ilLljiF1yWmd9fUMqETQT2Guas+6 +l0u8ByzmPPSA6nx7egLnfBEec4cjsocrXGDHmhgtYNSClpoHsJ4RKtNhWp7TCRpZ +phYtngNBDw9Nhgt++NkBqkcS8I1rJuf06crlNuBGCkRgkZu0HVSKN7oBUnrSq59G +8jsVhgb7buHx/F1r2ZEU/rvssx9bOchWAanNiU66yb0V +=UL8X +-----END PGP PUBLIC KEY BLOCK----- diff --git a/scripts/packaging/package-signing-key.asc b/scripts/packaging/package-signing-key.asc new file mode 100644 index 0000000000000000000000000000000000000000..1d8bbf97dbe96dc9cae002b9760a1217df9fc158 --- /dev/null +++ b/scripts/packaging/package-signing-key.asc @@ -0,0 +1,41 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQGNBGWnyQoBDAC6YOwv480lfE0TmRr6pFkEnjEF8MgSNOY7GX6OmqJZ4n3bCq4K +AXgjfNHsYrf9yMDCMzbzD65W4nl+NhU1lZKLITYm2eeYj76M+DHc1LxWDo95kTPh +QlPXEZ2KKHmjObV2L1KY1te8TTjtAwEF3Ycuf+Rs7gmvaKa53Kq23BBwroX432Gp +rt3fOPSlJ2XsCBkJZ0LaoIay74LJCnvUIAxY0xJusYVtmljSQouuU15d41Fjzz1W +ZVFZ1Wbnamoau/TgCVLUWLIAfZufPHMjSCOnWVoeOWV06vLOHU1kC0gSxwBdrei3 +zbkQ1ip/gmZHNnQGuZIsqg1epliIlrXdZ4fBKdbw5qaeVa/WVucjge4YeO7vtGf2 +3IMui5CXM99hd4yJli1/xu9BJmOqR2RZSNQBpFPCAs/QJijt4eVwwbHA9OfEqA5s +yHMCsmcKtQHM38v5bK3F5QD7av0+zZ8HX28bAPNTMDMcHhgeM96TZjG4ZTLSLPgq +nWeT9JKQABVJricAEQEAAbQkTm9tYWRpYyBMYWJzIDxpbmZvQG5vbWFkaWMtbGFi +cy5jb20+iQHUBBMBCgA+FiEEs365ctXSdaJDWU7Kz8SC880I020FAmWnyQoCGwMF +CQHhM4AFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQz8SC880I022LiQwAsUFb +Dn1qJNCskzT0qmo4x6xwqHBp3ztWTCUZUFn1inW2ycrkBu4JiyVZDtAGmZ+uwXHT +/26Pa9B2NLUGkqtH3wV2HZ7zCIULj9Qe5cjIpRoANCRdHte04g82RXFfqesAVdOP +ZnWGTF8iXqQDJPiC7KLiYoaNqqxxyN/lOSElvBWU31eZQyuAYcLYCsXuVE4oHrOG +RDkXMxapr9yL3i+qnfqSV11KS3lvuz+KSZOyKcPGyMBXsw+z1qoQYL8JYODTDga8 +GG0942MnkA9Y6v49GPcj7zxCISzLpdBkJrEbJSiB3mKHJrZJ03GVv9jACUtKrgNW +SlK5ndrqOxdyrqny0qnIeBwV8eLAyNfB6V1wysC3YAUimgeNh+B0XL/TC0Fi0Jw4 +J4oqCFPGfyyLce52sI2bvBQwEADslY9p3i7VpMgYL2t/Vh21SdJgjUwKE/cD6MnU +hdoYsDGIEKG4AjVM+njasKAyix016Md8OQAx8G4FW1UEuD+cWNw0oH7OjjwwuQGN +BGWnyQoBDADBj8DQkw4ZmIR8Yeq62/RQ+kLXzG2mHcfh+8gswwA8sANviwphW9MP +VmblZ6QXBZ4G9ERrAznf3bDLqGWw7JngjJkriMKvzLfj3UjltmdVfVlwYrrflmTc +GAa4Nb6Bj+IIh6erYXqy0taeElQ+n/cNQVPHekcoaQs8DwZE2hGhiEYETxpL63Rf +5GuCfBph+u7sWWmbJSW1knvx3JxvvlkFRSbZVIh/bL+Hj5MlaWLttGFX6k/WLKOe +pdSYv24qQ2AL7KgwDQ+Y6Z2NSyMUHCzPOv8cwllkMwRrWUgqyiXXaBWHhXfC/YAk +ZBSerL12baqiHuAiMB41td5WRZqQY+3JrqoobkKmBBVLm6VqgTA4rBQNYhVLcLie +OScKGG0xZrxSmRRWtP59YP96vzRrDZr0yBRmxTYnVMxPBfluzybZQyluWmMdAOM4 +MIbaQSlu1dA3GoJS7jqS0R08N2JBwSxjJfVnSx3LyY/yDb16+oXDjlNCgv/hWok/ +1fQGhZREdxsAEQEAAYkBvAQYAQoAJhYhBLN+uXLV0nWiQ1lOys/EgvPNCNNtBQJl +p8kKAhsMBQkB4TOAAAoJEM/EgvPNCNNtxBUL/2qXwILQvGleiV0h6SvaTCKmGKfK +gSQS2dJcwpS/4oRfEAc49sVz4S9a7ObfU/jx1FtpTXtDoNgnMvPTa58wsQoWIj3i +BOCiO6DKmSJRaB7KgEfRMyZh2NG+Qe8F5+vaY6VSZ88A2Ah0A6RII2G1Q7eSo/at +0w1dmB+bNsYJIxunrZwotNo+POBzSeW3Lnw/tb00qJazGG+zSuAqeWr/247+Haj9 +uUDN6b7XxSKM9dTC7KFaSVzMaI4TXWumZ+RIQgjtFFGx0/iMryrERqMwrcn7ErVd +nt08zBKBgCwWgtF8GKtzcCMEv5ZaBEVkbAS/6kL0GkYYU/khqyP0ht+F4kXHq2hi +KHmW/KfncC31zt8tmYOyVQxugmvruQYFQT4lBwmzesv8PfPUg2buweYBIQrif6Cx +M9Z0tKJQavlXh4zkv6XWmw0/XZPpe4s/9xzm7TrHths+6zm0+jgoSEhrOoj2UZx/ +1Sl1UvXGKRzuFddDleCGIx+4UnJc2FOm0iM3lQ== +=9Ckp +-----END PGP PUBLIC KEY BLOCK-----