From 7bcc18c8221dc90c9b5ea0550d54ca0aa5bb35ba Mon Sep 17 00:00:00 2001 From: Killian Delarue Date: Mon, 6 Nov 2023 15:43:23 +0100 Subject: [PATCH 1/3] CI: Create bin_package job --- .gitlab/ci/jobs/build/bin_packages.yml | 55 +++++++++++++++++++++++ .gitlab/ci/pipelines/release_tag.yml | 1 + .gitlab/ci/pipelines/release_tag_test.yml | 1 + 3 files changed, 57 insertions(+) create mode 100644 .gitlab/ci/jobs/build/bin_packages.yml diff --git a/.gitlab/ci/jobs/build/bin_packages.yml b/.gitlab/ci/jobs/build/bin_packages.yml new file mode 100644 index 000000000000..3d817d72a9d4 --- /dev/null +++ b/.gitlab/ci/jobs/build/bin_packages.yml @@ -0,0 +1,55 @@ +build:dpkg:amd64: + image: debian:bookworm + stage: build + needs: [] + script: + - apt update + - apt-get install -y rsync git m4 build-essential patch unzip wget opam jq bc autoconf cmake libev-dev libffi-dev libgmp-dev libhidapi-dev pkg-config zlib1g-dev + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain 1.64.0 -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make dpkg + variables: + OCTEZ_PKGMAINTAINER: "dev@nomadic-labs.com" + BLST_PORTABLE: "yes" + tags: + - amd64 + artifacts: + name: "dpkg-$ARCH-$CI_COMMIT_REF_SLUG" + paths: + - octez-*.deb + expire_in: 1 day + when: on_success + +build:rpm:amd64: + image: fedora + stage: build + needs: [] + script: + - dnf update -y + - dnf install -y libev-devel gmp-devel hidapi-devel libffi-devel zlib-devel libpq-devel m4 perl git pkg-config rpmdevtools python3-devel python3-setuptools wget opam rsync which cargo autoconf mock systemd systemd-rpm-macros cmake python3-wheel python3-tox-current-env + - wget https://sh.rustup.rs/rustup-init.sh + - chmod +x rustup-init.sh + - ./rustup-init.sh --profile minimal --default-toolchain 1.64.0 -y + - . $HOME/.cargo/env + - export OPAMYES="true" + - opam init --bare --disable-sandboxing + - make build-deps + - eval $(opam env) + - make rpm + variables: + OCTEZ_PKGMAINTAINER: "dev@nomadic-labs.com" + BLST_PORTABLE: "yes" + tags: + - amd64 + artifacts: + name: "rpm-$ARCH-$CI_COMMIT_REF_SLUG" + paths: + - octez-*.rpm + expire_in: 1 day + when: on_success diff --git a/.gitlab/ci/pipelines/release_tag.yml b/.gitlab/ci/pipelines/release_tag.yml index 57b6999931c5..d097b67f87f0 100644 --- a/.gitlab/ci/pipelines/release_tag.yml +++ b/.gitlab/ci/pipelines/release_tag.yml @@ -4,6 +4,7 @@ include: - .gitlab/ci/jobs/build/static_arm64_release.yml - .gitlab/ci/jobs/build/docker_amd64_release.yml - .gitlab/ci/jobs/build/docker_arm64_release.yml + - .gitlab/ci/jobs/build/bin_packages.yml # Stage: prepare_release - .gitlab/ci/jobs/prepare_release/docker_merge_manifests_release.yml diff --git a/.gitlab/ci/pipelines/release_tag_test.yml b/.gitlab/ci/pipelines/release_tag_test.yml index 51a9f9049b5b..56d82116a627 100644 --- a/.gitlab/ci/pipelines/release_tag_test.yml +++ b/.gitlab/ci/pipelines/release_tag_test.yml @@ -4,6 +4,7 @@ include: - .gitlab/ci/jobs/build/static_arm64_release.yml - .gitlab/ci/jobs/build/docker_amd64_test.yml - .gitlab/ci/jobs/build/docker_arm64_test.yml + - .gitlab/ci/jobs/build/bin_packages.yml # Stage: prepare_release - .gitlab/ci/jobs/prepare_release/docker_merge_manifests_test.yml -- GitLab From 0d1629b7916494b47705dff8a52daf97d7697760 Mon Sep 17 00:00:00 2001 From: Killian Delarue Date: Mon, 6 Nov 2023 15:45:40 +0100 Subject: [PATCH 2/3] CI: Link debian packages in gitlab release --- .gitlab/ci/jobs/publish/gitlab:release.yml | 1 + scripts/ci/create_gitlab_package.sh | 12 +++++++++++- scripts/ci/create_gitlab_release.sh | 18 ++++++++++++++++-- scripts/ci/release.sh | 2 ++ 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/.gitlab/ci/jobs/publish/gitlab:release.yml b/.gitlab/ci/jobs/publish/gitlab:release.yml index 5175fe073fbd..cb00d06e429a 100644 --- a/.gitlab/ci/jobs/publish/gitlab:release.yml +++ b/.gitlab/ci/jobs/publish/gitlab:release.yml @@ -9,6 +9,7 @@ gitlab:release: dependencies: - build:static-x86_64-linux-binaries - build:static-arm64-linux-binaries + - build:dpkg:amd64 - docker:merge_manifests script: - ./scripts/ci/gitlab-release.sh diff --git a/scripts/ci/create_gitlab_package.sh b/scripts/ci/create_gitlab_package.sh index bfd7c19cc9e7..e0ed050360d4 100755 --- a/scripts/ci/create_gitlab_package.sh +++ b/scripts/ci/create_gitlab_package.sh @@ -14,10 +14,12 @@ set -eu # https://docs.gitlab.com/ee/user/packages/generic_packages/index.html#download-package-file # :gitlab_api_url/projects/:id/packages/generic/:package_name/:package_version/:file_name gitlab_package_url="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${gitlab_package_name}/${gitlab_package_version}" +gitlab_deb_package_url="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${gitlab_deb_package_name}/${gitlab_package_version}" gitlab_upload() { local_path="${1}" remote_file="${2}" + url="${3-${gitlab_package_url}}" echo "Upload to ${gitlab_package_url}/${remote_file}" i=0 @@ -30,7 +32,7 @@ gitlab_upload() { http_code=$(curl -fsSL -o /dev/null -w "%{http_code}" \ -H "JOB-TOKEN: ${CI_JOB_TOKEN}" \ -T "${local_path}" \ - "${gitlab_package_url}/${remote_file}") + "${url}/${remote_file}") # Success [ "${http_code}" = '201' ] && return @@ -58,6 +60,14 @@ do gitlab_upload "octez-binaries/${architecture}/${binary}" "${architecture}-${binary}" done + echo "Upload debian packages (${architecture})" + + # Loop over debian packages + for package in ${deb_packages} + do + gitlab_upload "${package}" "${package}" "${gitlab_deb_package_url}" + done + echo "Upload tarball with all binaries (${architecture})" mkdir -pv "octez-binaries/octez-${architecture}" diff --git a/scripts/ci/create_gitlab_release.sh b/scripts/ci/create_gitlab_release.sh index cc27229057b7..ce3308cee120 100755 --- a/scripts/ci/create_gitlab_release.sh +++ b/scripts/ci/create_gitlab_release.sh @@ -19,12 +19,25 @@ web_path=$(curl -fsSL -X GET \ "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages?sort=desc&package_name=${gitlab_package_name}" \ | jq -r ".[] | select(.version==\"${gitlab_package_version}\") | ._links.web_path") +deb_web_path=$(curl -fsSL -X GET \ + -H "JOB-TOKEN: ${CI_JOB_TOKEN}" \ + "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages?sort=desc&package_name=${gitlab_deb_package_name}" \ + | jq -r ".[] | select(.version==\"${gitlab_package_version}\") | ._links.web_path") + if [ -z "${web_path}" ] then echo "Error: could not find package matching version ${gitlab_package_version}" exit 1 else - gitlab_package_url="https://${CI_SERVER_HOST}${web_path}" + gitlab_binaries_url="https://${CI_SERVER_HOST}${web_path}" +fi + +if [ -z "${deb_web_path}" ] +then + echo "Error: could not find debian package matching version ${gitlab_package_version}" + exit 1 +else + gitlab_deb_packages_url="https://${CI_SERVER_HOST}${deb_web_path}" fi if [ "${CI_PROJECT_NAMESPACE}" = "tezos" ] @@ -65,4 +78,5 @@ release-cli create \ --assets-link="{\"name\":\"Changelog\",\"url\":\"https://tezos.gitlab.io/CHANGES.html#version-${gitlab_release_no_dot}\",\"link_type\":\"other\"}" \ --assets-link="{\"name\":\"Announcement\",\"url\":\"https://tezos.gitlab.io/releases/version-${gitlab_release_major_version}.html\",\"link_type\":\"other\"}" \ --assets-link="{\"name\":\"Docker image\",\"url\":\"${docker_image_url}\",\"link_type\":\"image\"}" \ - --assets-link="{\"name\":\"Static binaries\",\"url\":\"${gitlab_package_url}\",\"link_type\":\"package\"}" + --assets-link="{\"name\":\"Static binaries\",\"url\":\"${gitlab_binaries_url}\",\"link_type\":\"package\"}" \ + --assets-link="{\"name\":\"Debian packages\",\"url\":\"${gitlab_deb_packages_url}\",\"link_type\":\"package\"}" diff --git a/scripts/ci/release.sh b/scripts/ci/release.sh index ed049a8101be..bbff0d3008c3 100755 --- a/scripts/ci/release.sh +++ b/scripts/ci/release.sh @@ -11,6 +11,7 @@ src_dir=$(dirname "$scripts_dir") script_inputs_dir="$src_dir/script-inputs" binaries="$(cat "$script_inputs_dir/released-executables")" +deb_packages="$(ls ./octez-*.deb)" ### Compute GitLab release names @@ -47,6 +48,7 @@ fi ### Compute GitLab generic package names gitlab_package_name="${CI_PROJECT_NAME}-${gitlab_release_no_v}" +gitlab_deb_package_name="${CI_PROJECT_NAME}-debian-${gitlab_release_no_v}" # X.Y or X.Y-rcZ gitlab_package_version="${gitlab_release_no_v}" -- GitLab From 57a657c1faf062de3661650d85d589e3562be21a Mon Sep 17 00:00:00 2001 From: Killian Delarue Date: Mon, 6 Nov 2023 16:23:56 +0100 Subject: [PATCH 3/3] CI: Link Redhat packages in gitlab release --- .gitlab/ci/jobs/publish/gitlab:release.yml | 1 + scripts/ci/create_gitlab_package.sh | 9 +++++++++ scripts/ci/create_gitlab_release.sh | 16 +++++++++++++++- scripts/ci/release.sh | 2 ++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/.gitlab/ci/jobs/publish/gitlab:release.yml b/.gitlab/ci/jobs/publish/gitlab:release.yml index cb00d06e429a..80b3685f3e4a 100644 --- a/.gitlab/ci/jobs/publish/gitlab:release.yml +++ b/.gitlab/ci/jobs/publish/gitlab:release.yml @@ -10,6 +10,7 @@ gitlab:release: - build:static-x86_64-linux-binaries - build:static-arm64-linux-binaries - build:dpkg:amd64 + - build:rpm:amd64 - docker:merge_manifests script: - ./scripts/ci/gitlab-release.sh diff --git a/scripts/ci/create_gitlab_package.sh b/scripts/ci/create_gitlab_package.sh index e0ed050360d4..39803bdcb36f 100755 --- a/scripts/ci/create_gitlab_package.sh +++ b/scripts/ci/create_gitlab_package.sh @@ -15,6 +15,7 @@ set -eu # :gitlab_api_url/projects/:id/packages/generic/:package_name/:package_version/:file_name gitlab_package_url="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${gitlab_package_name}/${gitlab_package_version}" gitlab_deb_package_url="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${gitlab_deb_package_name}/${gitlab_package_version}" +gitlab_rpm_package_url="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${gitlab_rpm_package_name}/${gitlab_package_version}" gitlab_upload() { local_path="${1}" @@ -68,6 +69,14 @@ do gitlab_upload "${package}" "${package}" "${gitlab_deb_package_url}" done + echo "Upload rpm packages (${architecture})" + + # Loop over rpm packages + for package in ${rpm_packages} + do + gitlab_upload "./${package}" "${package}" "${gitlab_rpm_package_url}" + done + echo "Upload tarball with all binaries (${architecture})" mkdir -pv "octez-binaries/octez-${architecture}" diff --git a/scripts/ci/create_gitlab_release.sh b/scripts/ci/create_gitlab_release.sh index ce3308cee120..cb89f17bacee 100755 --- a/scripts/ci/create_gitlab_release.sh +++ b/scripts/ci/create_gitlab_release.sh @@ -24,6 +24,11 @@ deb_web_path=$(curl -fsSL -X GET \ "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages?sort=desc&package_name=${gitlab_deb_package_name}" \ | jq -r ".[] | select(.version==\"${gitlab_package_version}\") | ._links.web_path") +rpm_web_path=$(curl -fsSL -X GET \ + -H "JOB-TOKEN: ${CI_JOB_TOKEN}" \ + "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages?sort=desc&package_name=${gitlab_rpm_package_name}" \ + | jq -r ".[] | select(.version==\"${gitlab_package_version}\") | ._links.web_path") + if [ -z "${web_path}" ] then echo "Error: could not find package matching version ${gitlab_package_version}" @@ -40,6 +45,14 @@ else gitlab_deb_packages_url="https://${CI_SERVER_HOST}${deb_web_path}" fi +if [ -z "${rpm_web_path}" ] +then + echo "Error: could not find rpm package matching version ${gitlab_package_version}" + exit 1 +else + gitlab_rpm_packages_url="https://${CI_SERVER_HOST}${rpm_web_path}" +fi + if [ "${CI_PROJECT_NAMESPACE}" = "tezos" ] then ## Production => Docker Hub @@ -79,4 +92,5 @@ release-cli create \ --assets-link="{\"name\":\"Announcement\",\"url\":\"https://tezos.gitlab.io/releases/version-${gitlab_release_major_version}.html\",\"link_type\":\"other\"}" \ --assets-link="{\"name\":\"Docker image\",\"url\":\"${docker_image_url}\",\"link_type\":\"image\"}" \ --assets-link="{\"name\":\"Static binaries\",\"url\":\"${gitlab_binaries_url}\",\"link_type\":\"package\"}" \ - --assets-link="{\"name\":\"Debian packages\",\"url\":\"${gitlab_deb_packages_url}\",\"link_type\":\"package\"}" + --assets-link="{\"name\":\"Debian packages\",\"url\":\"${gitlab_deb_packages_url}\",\"link_type\":\"package\"}" \ + --assets-link="{\"name\":\"Red Hat packages\",\"url\":\"${gitlab_rpm_packages_url}\",\"link_type\":\"package\"}" diff --git a/scripts/ci/release.sh b/scripts/ci/release.sh index bbff0d3008c3..3ff4a5e8bccd 100755 --- a/scripts/ci/release.sh +++ b/scripts/ci/release.sh @@ -12,6 +12,7 @@ script_inputs_dir="$src_dir/script-inputs" binaries="$(cat "$script_inputs_dir/released-executables")" deb_packages="$(ls ./octez-*.deb)" +rpm_packages="$(ls ./octez-*.rpm)" ### Compute GitLab release names @@ -49,6 +50,7 @@ fi gitlab_package_name="${CI_PROJECT_NAME}-${gitlab_release_no_v}" gitlab_deb_package_name="${CI_PROJECT_NAME}-debian-${gitlab_release_no_v}" +gitlab_rpm_package_name="${CI_PROJECT_NAME}-redhat-${gitlab_release_no_v}" # X.Y or X.Y-rcZ gitlab_package_version="${gitlab_release_no_v}" -- GitLab