From 48717278a211642c4ac0b50cc750b8d53d53c950 Mon Sep 17 00:00:00 2001 From: Killian Delarue Date: Wed, 17 Apr 2024 17:49:24 +0200 Subject: [PATCH 1/4] CI, Scripts: Introduce and use create_octez_tarball.sh --- scripts/ci/create_gitlab_package.sh | 7 ++----- scripts/ci/create_octez_tarball.sh | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) create mode 100755 scripts/ci/create_octez_tarball.sh diff --git a/scripts/ci/create_gitlab_package.sh b/scripts/ci/create_gitlab_package.sh index 5f5a1bfd026e..c5afb363b2f4 100755 --- a/scripts/ci/create_gitlab_package.sh +++ b/scripts/ci/create_gitlab_package.sh @@ -119,12 +119,9 @@ source_tarball="${gitlab_octez_source_package_name}.tar.bz2" git --version # Verify the placeholder %(describe:tags) is available git describe --tags -# Pass '--worktree-attributes' to ensure that ignores written by restrict_export_to_octez_source.sh -# are respected. -git archive "${CI_COMMIT_TAG}" --format=tar --worktree-attributes --prefix "${gitlab_octez_source_package_name}/" | bzip2 > "${source_tarball}" -# Check tarball is valid -tar -tjf "${source_tarball}" > /dev/null +# shellcheck source=./scripts/ci/create_octez_tarball.sh +. ./scripts/ci/create_octez_tarball.sh # Verify git expanded placeholders in archive tar -Oxf "${source_tarball}" "${gitlab_octez_source_package_name}/src/lib_version/exe/get_git_info.ml" | grep "let raw_current_version = \"${CI_COMMIT_TAG}\"" diff --git a/scripts/ci/create_octez_tarball.sh b/scripts/ci/create_octez_tarball.sh new file mode 100755 index 000000000000..bb44aa6da01e --- /dev/null +++ b/scripts/ci/create_octez_tarball.sh @@ -0,0 +1,24 @@ +#!/bin/sh +set -eu + +if [ $# -eq 0 ]; then + # No arguments are not provided. Assume we're running in a tag + # pipeline and set package_name accordingly. + + # shellcheck source=./scripts/ci/octez-release.sh + . ./scripts/ci/octez-release.sh + package_name="${gitlab_octez_source_package_name}" + commit="${CI_COMMIT_TAG}" +else + package_name="$1" + commit="HEAD" +fi + +source_tarball="${package_name}.tar.bz2" + +# Pass '--worktree-attributes' to ensure that ignores written by restrict_export_to_octez_source.sh +# are respected. +git archive "${commit}" --format=tar --worktree-attributes --prefix "${package_name}/" | bzip2 > "${source_tarball}" + +# Check tarball is valid +tar -tjf "${source_tarball}" > /dev/null -- GitLab From 99fe66ce68dbd23f73ea43f4697e9c23e9d468a5 Mon Sep 17 00:00:00 2001 From: Killian Delarue Date: Wed, 17 Apr 2024 17:35:00 +0200 Subject: [PATCH 2/4] CI: Add build_octez_source to schedule_extended_test --- .../ci/pipelines/schedule_extended_test.yml | 20 ++++++++++++ ci/bin/code_verification.ml | 32 ++++++++++++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index b2439a9a1e1b..a974fd85ef9f 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -399,6 +399,26 @@ oc.build:rpm:amd64: - fedora:39 - rockylinux:9.3 +build_octez_source: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/ci/restrict_export_to_octez_source.sh + - ./scripts/ci/create_octez_tarball.sh octez + - mv octez.tar.bz2 ../ + - cd ../ + - tar xf octez.tar.bz2 + - cd octez/ + - eval $(opam env) + - make octez + opam:prepare: image: ${build_deps_image_name}:runtime-prebuild-dependencies--${build_deps_image_version} stage: packaging diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index bad08109de83..530ea52ab469 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -697,6 +697,36 @@ let jobs pipeline_type = []) ["dune build @check"] in + let build_octez_source = + (* We check compilation of the octez tarball on scheduled pipelines because + it's not worth testing it for every merge request pipeline. *) + match pipeline_type with + | Schedule_extended_test -> + [ + job + ~__POS__ + ~stage:Stages.build + ~image:Images.runtime_build_dependencies + ~before_script: + (before_script + ~take_ownership:true + ~source_version:true + ~eval_opam:true + []) + ~name:"build_octez_source" + [ + "./scripts/ci/restrict_export_to_octez_source.sh"; + "./scripts/ci/create_octez_tarball.sh octez"; + "mv octez.tar.bz2 ../"; + "cd ../"; + "tar xf octez.tar.bz2"; + "cd octez/"; + "eval $(opam env)"; + "make octez"; + ]; + ] + | Before_merging -> [] + in [ job_docker_rust_toolchain; job_docker_client_libs_dependencies; @@ -710,7 +740,7 @@ let jobs pipeline_type = job_tezt_fetch_records; job_select_tezts; ] - @ bin_packages_jobs + @ bin_packages_jobs @ build_octez_source in let packaging = let job_opam_prepare : tezos_job = -- GitLab From a1ecef4dd952529972de72ec2d6003ac5e1c976f Mon Sep 17 00:00:00 2001 From: Killian Delarue Date: Mon, 22 Apr 2024 14:15:53 +0200 Subject: [PATCH 3/4] Scripts: Prevent octez-release failure when finding packages --- scripts/ci/octez-release.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/ci/octez-release.sh b/scripts/ci/octez-release.sh index e9074b8f74e4..83d0177462ad 100755 --- a/scripts/ci/octez-release.sh +++ b/scripts/ci/octez-release.sh @@ -13,11 +13,11 @@ script_inputs_dir="$src_dir/script-inputs" binaries="$(cat "$script_inputs_dir/released-executables")" # these variables are used in the script scripts/ci/create_gitlab_package.sh -debian_bookworm_packages="$(find packages/debian/bookworm/ -maxdepth 1 -name octez-\*.deb)" -ubuntu_focal_packages="$(find packages/ubuntu/focal/ -maxdepth 1 -name octez-\*.deb)" -ubuntu_jammy_packages="$(find packages/ubuntu/jammy/ -maxdepth 1 -name octez-\*.deb)" -fedora_packages="$(find packages/fedora/39/ -maxdepth 1 -name octez-\*.rpm)" -rockylinux_packages="$(find packages/rockylinux/9.3/ -maxdepth 1 -name octez-\*.rpm)" +debian_bookworm_packages="$(find packages/debian/bookworm/ -maxdepth 1 -name octez-\*.deb 2> /dev/null || printf '')" +ubuntu_focal_packages="$(find packages/ubuntu/focal/ -maxdepth 1 -name octez-\*.deb 2> /dev/null || printf '')" +ubuntu_jammy_packages="$(find packages/ubuntu/jammy/ -maxdepth 1 -name octez-\*.deb 2> /dev/null || printf '')" +fedora_packages="$(find packages/fedora/39/ -maxdepth 1 -name octez-\*.rpm 2> /dev/null || printf '')" +rockylinux_packages="$(find packages/rockylinux/9.3/ -maxdepth 1 -name octez-\*.rpm 2> /dev/null || printf '')" octez_source_content="$script_inputs_dir/octez-source-content" -- GitLab From 62182948844517c338e1d51e017fecf64af43688 Mon Sep 17 00:00:00 2001 From: Killian Delarue Date: Wed, 24 Apr 2024 16:12:38 +0200 Subject: [PATCH 4/4] Script-inputs: Add brassaia to source content --- script-inputs/octez-source-content | 1 + 1 file changed, 1 insertion(+) diff --git a/script-inputs/octez-source-content b/script-inputs/octez-source-content index cfa5c45661c4..972d181c37f6 100644 --- a/script-inputs/octez-source-content +++ b/script-inputs/octez-source-content @@ -10,3 +10,4 @@ opam script-inputs scripts irmin +brassaia -- GitLab