From 27e12ccb8208d4df28dde336d5d664e351ee6a81 Mon Sep 17 00:00:00 2001 From: Thomas Letan Date: Wed, 21 Aug 2024 21:34:06 +0200 Subject: [PATCH] CI: Add `octez-evm-node` release pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hats off to everyone involved in the CIAO project. The result of your hard work is true gem. In a nutshell, we rewrite the Etherlink release logic to make use of the newly introduced `octez-evm-node` product. Our goal is to introduce a specific pipeline for releases of the Octez EVM Node (`octez-evm-node`), with two kinds of artifacts:  1. Docker images only containing the `octez-evm-node` binary 2. `octez-evm-node` static binaries for x86-64 and Amd64 In order to implement that, I have ~~stolen~~ drawn inspiration from the Octez release pipeline. Some scripts had to be generalized a little, because they were assuming `octez-node` nad `octez-client` would always be part of Docker images built by the CI. Some general-purpose functions were duplicated, like the one to upload an artifact to a Gitlab package. This is to keep the “official” release pipeline as simple as possible, and avoid indirections when possible. You can review the release page generated by Gitlab for a fake tage [here][release-page]. Can you guess how many trials I had to do before getting something working as I wanted? [release-page]: https://gitlab.com/nomadic-labs/tezos/-/releases/octez-evm-node-v0.11 --- .gitlab-ci.yml | 16 +- .../ci/pipelines/etherlink_release_tag.yml | 43 ---- .../pipelines/octez_evm_node_release_tag.yml | 202 ++++++++++++++++++ ci/bin/common.ml | 27 ++- ci/bin/main.ml | 12 +- ci/bin/release_tag.ml | 71 ++++-- script-inputs/octez-evm-node-executable | 1 + scripts/ci/build_static_binaries.sh | 9 +- scripts/ci/create_gitlab_etherlink_release.sh | 14 -- .../create_gitlab_octez_evm_node_release.sh | 105 +++++++++ .../ci/docker_prepare_etherlink_release.sh | 17 -- scripts/ci/docker_release.sh | 2 +- scripts/ci/docker_smoke_test.sh | 4 +- scripts/ci/etherlink-release.sh | 23 -- scripts/ci/octez-evm-node-release.sh | 50 +++++ 15 files changed, 458 insertions(+), 138 deletions(-) delete mode 100644 .gitlab/ci/pipelines/etherlink_release_tag.yml create mode 100644 .gitlab/ci/pipelines/octez_evm_node_release_tag.yml create mode 100644 script-inputs/octez-evm-node-executable delete mode 100755 scripts/ci/create_gitlab_etherlink_release.sh create mode 100755 scripts/ci/create_gitlab_octez_evm_node_release.sh delete mode 100755 scripts/ci/docker_prepare_etherlink_release.sh delete mode 100644 scripts/ci/etherlink-release.sh create mode 100644 scripts/ci/octez-evm-node-release.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6da553e4d375..a64065dbea52 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,19 +46,19 @@ workflow: variables: PIPELINE_TYPE: octez_release_tag_test when: always - - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG =~ /^etherlink-v\d+\.\d+(?:\-rc\d+)?$/ + - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG =~ /^octez-evm-node-v\d+\.\d+(?:\-rc\d+)?$/ variables: - PIPELINE_TYPE: etherlink_release_tag + PIPELINE_TYPE: octez_evm_node_release_tag when: always - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG != null && $CI_COMMIT_TAG !~ /^octez-v\d+\.\d+(?:\-rc\d+)?$/ && $CI_COMMIT_TAG - !~ /^octez-v\d+\.\d+\-beta\d*$/ && $CI_COMMIT_TAG !~ /^etherlink-v\d+\.\d+(?:\-rc\d+)?$/ + !~ /^octez-v\d+\.\d+\-beta\d*$/ && $CI_COMMIT_TAG !~ /^octez-evm-node-v\d+\.\d+(?:\-rc\d+)?$/ variables: PIPELINE_TYPE: non_release_tag when: always - if: $CI_PROJECT_NAMESPACE != "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG != null && $CI_COMMIT_TAG !~ /^octez-v\d+\.\d+(?:\-rc\d+)?$/ && $CI_COMMIT_TAG - !~ /^octez-v\d+\.\d+\-beta\d*$/ && $CI_COMMIT_TAG !~ /^etherlink-v\d+\.\d+(?:\-rc\d+)?$/ + !~ /^octez-v\d+\.\d+\-beta\d*$/ && $CI_COMMIT_TAG !~ /^octez-evm-node-v\d+\.\d+(?:\-rc\d+)?$/ variables: PIPELINE_TYPE: non_release_tag_test when: always @@ -141,21 +141,21 @@ include: - if: $CI_PROJECT_NAMESPACE != "tezos" && $CI_PIPELINE_SOURCE == "push" && ($CI_COMMIT_TAG =~ /^octez-v\d+\.\d+(?:\-rc\d+)?$/ || $CI_COMMIT_TAG =~ /^octez-v\d+\.\d+\-beta\d*$/) when: always -- local: .gitlab/ci/pipelines/etherlink_release_tag.yml +- local: .gitlab/ci/pipelines/octez_evm_node_release_tag.yml rules: - - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG =~ /^etherlink-v\d+\.\d+(?:\-rc\d+)?$/ + - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG =~ /^octez-evm-node-v\d+\.\d+(?:\-rc\d+)?$/ when: always - local: .gitlab/ci/pipelines/non_release_tag.yml rules: - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG != null && $CI_COMMIT_TAG !~ /^octez-v\d+\.\d+(?:\-rc\d+)?$/ && $CI_COMMIT_TAG - !~ /^octez-v\d+\.\d+\-beta\d*$/ && $CI_COMMIT_TAG !~ /^etherlink-v\d+\.\d+(?:\-rc\d+)?$/ + !~ /^octez-v\d+\.\d+\-beta\d*$/ && $CI_COMMIT_TAG !~ /^octez-evm-node-v\d+\.\d+(?:\-rc\d+)?$/ when: always - local: .gitlab/ci/pipelines/non_release_tag_test.yml rules: - if: $CI_PROJECT_NAMESPACE != "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG != null && $CI_COMMIT_TAG !~ /^octez-v\d+\.\d+(?:\-rc\d+)?$/ && $CI_COMMIT_TAG - !~ /^octez-v\d+\.\d+\-beta\d*$/ && $CI_COMMIT_TAG !~ /^etherlink-v\d+\.\d+(?:\-rc\d+)?$/ + !~ /^octez-v\d+\.\d+\-beta\d*$/ && $CI_COMMIT_TAG !~ /^octez-evm-node-v\d+\.\d+(?:\-rc\d+)?$/ when: always - local: .gitlab/ci/pipelines/schedule_extended_test.yml rules: diff --git a/.gitlab/ci/pipelines/etherlink_release_tag.yml b/.gitlab/ci/pipelines/etherlink_release_tag.yml deleted file mode 100644 index 1d679ac6bcb1..000000000000 --- a/.gitlab/ci/pipelines/etherlink_release_tag.yml +++ /dev/null @@ -1,43 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -stages: -- prepare_release -- publish_package_gitlab - -docker:prepare-etherlink-release: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 - stage: prepare_release - tags: - - gcp - dependencies: [] - timeout: 60 minutes - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - ./scripts/ci/docker_prepare_etherlink_release.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.7 - artifacts: - expire_in: 1 hour - paths: - - kernels.tar.gz - -gitlab:etherlink-release: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.6.0 - stage: publish_package_gitlab - tags: - - gcp - needs: - - docker:prepare-etherlink-release - dependencies: [] - timeout: 60 minutes - interruptible: false - script: - - ./scripts/ci/create_gitlab_etherlink_release.sh - artifacts: - expire_in: never - paths: - - kernels.tar.gz diff --git a/.gitlab/ci/pipelines/octez_evm_node_release_tag.yml b/.gitlab/ci/pipelines/octez_evm_node_release_tag.yml new file mode 100644 index 000000000000..5baa8841da20 --- /dev/null +++ b/.gitlab/ci/pipelines/octez_evm_node_release_tag.yml @@ -0,0 +1,202 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +stages: +- images +- build +- prepare_release +- publish_release_gitlab + +oc.docker:ci:amd64: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: images + tags: + - gcp + dependencies: [] + timeout: 60 minutes + before_script: [] + script: + - ./images/ci_create_ci_images.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + CI_DOCKER_HUB: "false" + ARCH: amd64 + artifacts: + reports: + dotenv: ci_image_tag.env + +oc.docker:ci:arm64: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: images + tags: + - gcp_arm64 + dependencies: [] + timeout: 60 minutes + before_script: [] + script: + - ./images/ci_create_ci_images.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + CI_DOCKER_HUB: "false" + ARCH: arm64 + artifacts: + reports: + dotenv: ci_image_tag.env + retry: + max: 1 + when: + - runner_system_failure + +oc.build:static-arm64-linux-binaries: + image: ${ci_image_name}/build:${ci_image_tag} + stage: build + tags: + - gcp_arm64 + dependencies: + - oc.docker:ci:arm64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + - . ./scripts/ci/sccache-start.sh + script: + - ./scripts/ci/build_static_binaries.sh + after_script: + - sccache --stop-server || true + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/octez-evm-node-executable + VERSION_EXECUTABLE: octez-evm-node + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + artifacts: + expire_in: 90 days + paths: + - octez-binaries/$ARCH/* + +oc.build:static-x86_64-linux-binaries: + image: ${ci_image_name}/build:${ci_image_tag} + stage: build + tags: + - gcp + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + - . ./scripts/ci/sccache-start.sh + script: + - ./scripts/ci/build_static_binaries.sh + after_script: + - sccache --stop-server || true + variables: + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/octez-evm-node-executable + VERSION_EXECUTABLE: octez-evm-node + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + artifacts: + expire_in: 90 days + paths: + - octez-binaries/$ARCH/* + +oc.docker:amd64: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: build + tags: + - gcp + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + CI_DOCKER_HUB: "true" + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: amd64_ + EXECUTABLE_FILES: script-inputs/octez-evm-node-executable + +oc.docker:arm64: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: build + tags: + - gcp_arm64 + dependencies: + - oc.docker:ci:arm64 + timeout: 60 minutes + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + CI_DOCKER_HUB: "true" + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: arm64_ + EXECUTABLE_FILES: script-inputs/octez-evm-node-executable + +docker:merge_manifests: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.12.0 + stage: prepare_release + tags: + - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 + dependencies: [] + timeout: 60 minutes + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + CI_DOCKER_HUB: "true" + +gitlab:octez-evm-node-release: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.6.0 + stage: publish_release_gitlab + tags: + - gcp + needs: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + dependencies: + - oc.build:static-x86_64-linux-binaries + - oc.build:static-arm64-linux-binaries + timeout: 60 minutes + interruptible: false + script: + - ./scripts/ci/create_gitlab_octez_evm_node_release.sh diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 88bb2bf2ad3c..150bd5aa2336 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -742,8 +742,9 @@ end (no need to test that we pass the -static flag twice) - released variants exist, that are used in release tag pipelines (they do not build experimental executables) *) -let job_build_static_binaries ~__POS__ ~arch ?(release = false) ?rules - ?dependencies () : tezos_job = +let job_build_static_binaries ~__POS__ ~arch + ?(executable_files = "script-inputs/released-executables") + ?version_executable ?(release = false) ?rules ?dependencies () : tezos_job = let arch_string = arch_to_string arch in let name = "oc.build:static-" ^ arch_string ^ "-linux-binaries" in let artifacts = @@ -752,9 +753,14 @@ let job_build_static_binaries ~__POS__ ~arch ?(release = false) ?rules artifacts ?expire_in ["octez-binaries/$ARCH/*"] in let executable_files = - "script-inputs/released-executables" + executable_files ^ if not release then " script-inputs/experimental-executables" else "" in + let version_executable = + match version_executable with + | Some exe -> [("VERSION_EXECUTABLE", exe)] + | None -> [] + in job ?rules ?dependencies @@ -764,7 +770,9 @@ let job_build_static_binaries ~__POS__ ~arch ?(release = false) ?rules ~name ~image:Images.CI.build ~before_script:(before_script ~take_ownership:true ~eval_opam:true []) - ~variables:[("ARCH", arch_string); ("EXECUTABLE_FILES", executable_files)] + ~variables: + ([("ARCH", arch_string); ("EXECUTABLE_FILES", executable_files)] + @ version_executable) ~artifacts ["./scripts/ci/build_static_binaries.sh"] |> enable_cargo_cache |> enable_sccache @@ -782,6 +790,7 @@ let job_build_static_binaries ~__POS__ ~arch ?(release = false) ?rules - [Release] Docker builds include only released executables whereas other types also includes experimental ones. + - [Octez_evm_node_release] Docker build include only `octez-evm-node` - [Test_manual] and [Experimental] Docker builds include the EVM kernels in amd64 builds. - [Release] and [Experimental] Docker builds are pushed to Docker hub, @@ -789,7 +798,12 @@ let job_build_static_binaries ~__POS__ ~arch ?(release = false) ?rules - [Test_manual] Docker builds are started manually, put in the stage [manual] and their failure is allowed. The other types are in the build stage, run [on_success] and are not allowed to fail. *) -type docker_build_type = Experimental | Release | Test | Test_manual +type docker_build_type = + | Experimental + | Release + | Octez_evm_node_release + | Test + | Test_manual (** Creates a Docker build job of the given [arch] and [docker_build_type]. *) let job_docker_build ?rules ?dependencies ~__POS__ ~arch docker_build_type : @@ -797,7 +811,7 @@ let job_docker_build ?rules ?dependencies ~__POS__ ~arch docker_build_type : let arch_string = arch_to_string_alt arch in let ci_docker_hub = match docker_build_type with - | Release | Experimental -> true + | Release | Octez_evm_node_release | Experimental -> true | Test | Test_manual -> false in (* Whether to include evm artifacts. @@ -828,6 +842,7 @@ let job_docker_build ?rules ?dependencies ~__POS__ ~arch docker_build_type : ( "EXECUTABLE_FILES", match docker_build_type with | Release -> "script-inputs/released-executables" + | Octez_evm_node_release -> "script-inputs/octez-evm-node-executable" | Test | Test_manual | Experimental -> "script-inputs/released-executables \ script-inputs/experimental-executables" ); diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 8dc1587c0215..96d0ee384d59 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -72,7 +72,9 @@ let () = (* Matches Octez beta release tags, e.g. [octez-v1.2.3-beta5]. *) let octez_beta_release_tag_re = "/^octez-v\\d+\\.\\d+\\-beta\\d*$/" in (* Matches Etherlink release tags, e.g. [etherlink-v1.2.3] or [etherlink-v1.2.3-rc4]. *) - let etherlink_release_tag_re = "/^etherlink-v\\d+\\.\\d+(?:\\-rc\\d+)?$/" in + let octez_evm_node_release_tag_re = + "/^octez-evm-node-v\\d+\\.\\d+(?:\\-rc\\d+)?$/" + in let open Rules in let open Pipeline in (* Matches either Octez release tags or Octez beta release tags, @@ -86,7 +88,7 @@ let () = If.( Predefined_vars.ci_commit_tag != null && (not has_any_octez_release_tag) - && not (has_tag_match etherlink_release_tag_re)) + && not (has_tag_match octez_evm_node_release_tag_re)) in register "before_merging" @@ -126,9 +128,9 @@ let () = (* To test this type of release, push a tag to a fork of [tezos/tezos] e.g. [nomadic-labs/tezos] project. *) register - "etherlink_release_tag" - If.(push && has_tag_match etherlink_release_tag_re) - ~jobs:(Release_tag.etherlink_jobs ()) ; + "octez_evm_node_release_tag" + If.(push && has_tag_match octez_evm_node_release_tag_re) + ~jobs:(Release_tag.octez_evm_node_jobs ()) ; register "non_release_tag" If.(on_tezos_namespace && push && has_non_release_tag) diff --git a/ci/bin/release_tag.ml b/ci/bin/release_tag.ml index 584d5fd7acd5..939c9e3072b5 100644 --- a/ci/bin/release_tag.ml +++ b/ci/bin/release_tag.ml @@ -156,27 +156,66 @@ let octez_jobs ?(test = false) release_tag_pipeline_type = | _ -> [] (** Create an etherlink release tag pipeline of type {!release_tag_pipeline_type}. *) -let etherlink_jobs () = - let job_produce_docker_artifacts : Tezos_ci.tezos_job = - job_docker_authenticated +let octez_evm_node_jobs ?(test = false) () = + let job_docker_amd64 = + job_docker_build + ~__POS__ + ~arch:Amd64 + (if test then Test else Octez_evm_node_release) + in + let job_docker_arm64 = + job_docker_build + ~__POS__ + ~arch:Arm64 + (if test then Test else Octez_evm_node_release) + in + let job_docker_merge = + job_docker_merge_manifests + ~__POS__ + ~ci_docker_hub:(not test) + ~job_docker_amd64 + ~job_docker_arm64 + in + let job_static_x86_64_release = + job_build_static_binaries ~__POS__ - ~stage:Stages.prepare_release - ~name:"docker:prepare-etherlink-release" - ~artifacts: - (Gitlab_ci.Util.artifacts - ~expire_in:(Duration (Hours 1)) - ["kernels.tar.gz"]) - ["./scripts/ci/docker_prepare_etherlink_release.sh"] + ~arch:Amd64 + ~executable_files:"script-inputs/octez-evm-node-executable" + ~release:true + ~version_executable:"octez-evm-node" + () in + let job_static_arm64_release = + job_build_static_binaries + ~__POS__ + ~arch:Arm64 + ~executable_files:"script-inputs/octez-evm-node-executable" + ~release:true + ~version_executable:"octez-evm-node" + () + in + let job_gitlab_release : Tezos_ci.tezos_job = + let dependencies = + Dependent + [ + Artifacts job_static_x86_64_release; Artifacts job_static_arm64_release; + ] + in job ~__POS__ ~image:Images.ci_release - ~stage:Stages.publish_package_gitlab + ~stage:Stages.publish_release_gitlab ~interruptible:false - ~dependencies:(Dependent [Job job_produce_docker_artifacts]) - ~artifacts:(Gitlab_ci.Util.artifacts ~expire_in:Never ["kernels.tar.gz"]) - ~name:"gitlab:etherlink-release" - ["./scripts/ci/create_gitlab_etherlink_release.sh"] + ~dependencies + ~name:"gitlab:octez-evm-node-release" + ["./scripts/ci/create_gitlab_octez_evm_node_release.sh"] in - [job_produce_docker_artifacts; job_gitlab_release] + [ + job_static_arm64_release; + job_static_x86_64_release; + job_docker_amd64; + job_docker_arm64; + job_docker_merge; + job_gitlab_release; + ] diff --git a/script-inputs/octez-evm-node-executable b/script-inputs/octez-evm-node-executable new file mode 100644 index 000000000000..58f972c91b5f --- /dev/null +++ b/script-inputs/octez-evm-node-executable @@ -0,0 +1 @@ +octez-evm-node diff --git a/scripts/ci/build_static_binaries.sh b/scripts/ci/build_static_binaries.sh index e03c6e3774b2..e39453579cfd 100755 --- a/scripts/ci/build_static_binaries.sh +++ b/scripts/ci/build_static_binaries.sh @@ -50,14 +50,15 @@ cat $EXECUTABLE_FILES | chmod +w "octez-binaries/$ARCH/$executable" done -echo 'Check octez-client --version' +VERSION_EXECUTABLE="${VERSION_EXECUTABLE:-octez-client}" +echo "Check $VERSION_EXECUTABLE --version" SHA=$(git rev-parse --short=8 HEAD) -client_version=$("octez-binaries/$ARCH/octez-client" --version | cut -f 1 -d ' ') +client_version=$("octez-binaries/$ARCH/$VERSION_EXECUTABLE" --version | cut -f 1 -d ' ') if [ "$SHA" != "$client_version" ]; then - echo "Unexpected version for octez-client (expected $SHA, found $client_version)" + echo "Unexpected version for $VERSION_EXECUTABLE (expected $SHA, found $client_version)" exit 1 fi -echo "octez-client --version returned the expected commit hash: $SHA" +echo "$VERSION_EXECUTABLE --version returned the expected commit hash: $SHA" echo "Strip debug symbols and compress binaries (parallelized)" # shellcheck disable=SC2046,SC2038 diff --git a/scripts/ci/create_gitlab_etherlink_release.sh b/scripts/ci/create_gitlab_etherlink_release.sh deleted file mode 100755 index 89ec92453a56..000000000000 --- a/scripts/ci/create_gitlab_etherlink_release.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh -set -eu - -### Create a GitLab Etherlink release with links to all the related resources - -# shellcheck source=./scripts/ci/etherlink-release.sh -. ./scripts/ci/etherlink-release.sh - -kernels_artifact_url="$CI_JOB_URL/artifacts/raw/kernels.tar.gz" - -release-cli create \ - --name="${gitlab_release_name}" \ - --tag-name="${CI_COMMIT_TAG}" \ - --assets-link="{\"name\":\"Kernels\",\"url\":\"${kernels_artifact_url}\"}" diff --git a/scripts/ci/create_gitlab_octez_evm_node_release.sh b/scripts/ci/create_gitlab_octez_evm_node_release.sh new file mode 100755 index 000000000000..c7569773ab76 --- /dev/null +++ b/scripts/ci/create_gitlab_octez_evm_node_release.sh @@ -0,0 +1,105 @@ +#!/bin/sh +set -eu + +### Create a GitLab Octez EVM Node release with links to static binaries + +# In case of failures with this pipeline, contact the maintainer. +# Maintainer: Thomas Letan + +# shellcheck source=./scripts/ci/octez-evm-node-release.sh +. ./scripts/ci/octez-evm-node-release.sh +gitlab_octez_package_url="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${gitlab_octez_binaries_package_name}/${gitlab_package_version}" + +# Copied from `create_gitlab_release.sh`. +# If the pipeline is broken because of uploads, a safe bet is to update this +# function with its latest version in the original file. +package_web_path() { + f_package_name="$1" + ret=$(curl -fsSL -X GET \ + -H "JOB-TOKEN: ${CI_JOB_TOKEN}" \ + "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages?sort=desc&package_name=${f_package_name}" | + jq -r ".[] | select(.version==\"${gitlab_package_version}\") | ._links.web_path") + + if [ -z "${ret}" ]; then + echo "Error: ${f_package_name} could not find package matching version ${gitlab_package_version}" + exit 1 + else + echo "https://${CI_SERVER_HOST}${ret}" + fi +} + +# Copied from `create_gitlab_package.sh`. +# If the pipeline is broken because of uploads, a safe bet is to update this +# function with its latest version in the original file. +gitlab_upload() { + local_path="${1}" + remote_file="${2}" + url="${3-${gitlab_octez_package_url}}" + echo "Upload to ${url}/${remote_file}" + + i=0 + max_attempts=10 + + # Retry because gitlab.com is flaky sometimes, curl upload fails with http status code 524 (timeout) + while [ "${i}" != "${max_attempts}" ]; do + i=$((i + 1)) + http_code=$(curl -fsSL -o /dev/null -w "%{http_code}" \ + -H "JOB-TOKEN: ${CI_JOB_TOKEN}" \ + -T "${local_path}" \ + "${url}/${remote_file}") + + # Success + [ "${http_code}" = '201' ] && return + # Failure + echo "Error: HTTP response code ${http_code}, expected 201" + # Do not backoff after last attempt + [ "${i}" = "${max_attempts}" ] && break + # Backoff + echo "Retry (${i}) in one minute..." + sleep 60s + done + + echo "Error: maximum attempts exhausted (${max_attempts})" + exit 1 +} + +echo "Create GitLab package" + +# We only support one OS for now +os="linux" + +# Loop over architectures +for architecture in ${architectures}; do + echo "Upload raw binariy (${architecture})" + + gitlab_upload \ + "octez-binaries/${architecture}/octez-evm-node" \ + "$os-${architecture}-octez-evm-node" + + echo "Upload tarball (${architecture})" + + mkdir -pv "octez-binaries/octez-evm-node-$os-${architecture}" + cp -a \ + octez-binaries/"${architecture}"/* \ + "octez-binaries/octez-evm-node-$os-${architecture}/" + + cd octez-binaries/ + tar -czf \ + "octez-evm-node-$os-${architecture}.tar.gz" \ + "octez-evm-node-$os-${architecture}/" + gitlab_upload \ + "octez-evm-node-$os-${architecture}.tar.gz" \ + "${gitlab_octez_binaries_package_name}-$os-${architecture}.tar.gz" + cd .. +done + +echo "Query GitLab to get generic package URL" + +gitlab_binaries_url=$(package_web_path "${gitlab_octez_binaries_package_name}") + +echo "Release" + +release-cli create \ + --name="${gitlab_release_name}" \ + --tag-name="${CI_COMMIT_TAG}" \ + --assets-link="{\"name\":\"Static binaries\",\"url\":\"${gitlab_binaries_url}\",\"link_type\":\"package\"}" diff --git a/scripts/ci/docker_prepare_etherlink_release.sh b/scripts/ci/docker_prepare_etherlink_release.sh deleted file mode 100755 index c7faa7763307..000000000000 --- a/scripts/ci/docker_prepare_etherlink_release.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh -set -eu - -### Prepare etherlink release by building wasm and preimages with docker - -current_dir=$(cd "$(dirname "${0}")" && pwd) - -. scripts/ci/docker.env - -# shellcheck source=./scripts/ci/docker.sh -. "${current_dir}/docker.sh" - -# produce evm wasms and preimages -etherlink/scripts/build-wasm.sh etherlink/config/dev.yaml kernels - -# compress the directory -tar -czf kernels.tar.gz kernels diff --git a/scripts/ci/docker_release.sh b/scripts/ci/docker_release.sh index aeab7f733160..490b2d1e9458 100755 --- a/scripts/ci/docker_release.sh +++ b/scripts/ci/docker_release.sh @@ -42,7 +42,7 @@ OCTEZ_EXECUTABLES="$(cat $EXECUTABLE_FILES)" # auth gitlab or dockerhub registry # notice the different namespace for gitlab and that we remove the `-` # Test bare image -./scripts/ci/docker_smoke_test.sh "${DOCKER_IMAGE_NAME}bare:${DOCKER_IMAGE_TAG}" "${CI_COMMIT_SHORT_SHA}" version +REQUIRED_EXECUTABLES="$OCTEZ_EXECUTABLES" ./scripts/ci/docker_smoke_test.sh "${DOCKER_IMAGE_NAME}bare:${DOCKER_IMAGE_TAG}" "${CI_COMMIT_SHORT_SHA}" version # Push minimal, bare and debug images ./scripts/ci/docker_push_all.sh diff --git a/scripts/ci/docker_smoke_test.sh b/scripts/ci/docker_smoke_test.sh index c5ca388e5a51..74df8aec55e4 100755 --- a/scripts/ci/docker_smoke_test.sh +++ b/scripts/ci/docker_smoke_test.sh @@ -23,6 +23,8 @@ else fi TESTS=${*:-version} +REQUIRED_EXECUTABLES="${REQUIRED_EXECUTABLES:-"octez-client octez-node"}" + test_version() { echo "Testing version of binaries in Docker image" @@ -31,7 +33,7 @@ test_version() { echo "Found the following executables in the Docker image:" echo "$binaries" - for bin in octez-client octez-node; do + for bin in $REQUIRED_EXECUTABLES; do if ! echo "$binaries" | grep -q "^${bin}$"; then echo "Could not find binary $bin in Docker image." exit 1 diff --git a/scripts/ci/etherlink-release.sh b/scripts/ci/etherlink-release.sh deleted file mode 100644 index 1de4490b5e00..000000000000 --- a/scripts/ci/etherlink-release.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -### Compute GitLab release names from git tags - -# Git tags for Etherlink releases are on the form `etherlink-vX.Y`, `etherlink-vX.Y-rcZ` or `etherlink-vX.Y-betaZ`. - -# X -gitlab_release_major_version=$(echo "${CI_COMMIT_TAG}" | sed -nE 's/^etherlink-v([0-9]+)\.([0-9]+)(-rc[0-9]+)?$/\1/p') -# Y -gitlab_release_minor_version=$(echo "${CI_COMMIT_TAG}" | sed -nE 's/^etherlink-v([0-9]+)\.([0-9]+)(-rc[0-9]+)?$/\2/p') -# Z -gitlab_release_rc_version=$(echo "${CI_COMMIT_TAG}" | sed -nE 's/^etherlink-v([0-9]+)\.([0-9]+)(-rc)?([0-9]+)?$/\4/p') - -# Is this a release candidate? -if [ -n "${gitlab_release_rc_version}" ]; then - # Yes, release name: Etherlink Release Candidate X.Y~rcZ - # shellcheck disable=SC2034 - gitlab_release_name="Etherlink Release Candidate ${gitlab_release_major_version}.${gitlab_release_minor_version}~rc${gitlab_release_rc_version}" -else - # No, release name: Etherlink Release X.Y - # shellcheck disable=SC2034 - gitlab_release_name="Etherlink Release ${gitlab_release_major_version}.${gitlab_release_minor_version}" -fi diff --git a/scripts/ci/octez-evm-node-release.sh b/scripts/ci/octez-evm-node-release.sh new file mode 100644 index 000000000000..bfcc116a1d36 --- /dev/null +++ b/scripts/ci/octez-evm-node-release.sh @@ -0,0 +1,50 @@ +#!/bin/sh + +## Sourceable file with common variables for other scripts related to release + +# shellcheck disable=SC2034 +architectures='x86_64 arm64' + +# Full octez release tag +# octez-evm-node-vX.Y, octez-evm-node-vX.Y-rcZ or octez-evm-node-vX.Y-betaZ +gitlab_release=$(echo "${CI_COMMIT_TAG}" | grep -oE '^octez-evm-node-v([0-9]+)\.([0-9]+)$' || :) + +# Strips the leading 'octez-evm-node-v' +# X.Y, X.Y-rcZ or X.Y-betaZ +gitlab_release_no_v=$(echo "${gitlab_release}" | sed -e 's/^octez-evm-node-v//g') + +# Replace '.' with '-' +# X-Y or X-Y-rcZ +# shellcheck disable=SC2034 +gitlab_release_no_dot=$(echo "${gitlab_release_no_v}" | sed -e 's/\./-/g') + +# X +gitlab_release_major_version=$(echo "${CI_COMMIT_TAG}" | sed -nE 's/^octez-evm-node-v([0-9]+)\.([0-9]+)((-rc[0-9]+)?|(-beta[0-9]+)?)$/\1/p') +# Y +gitlab_release_minor_version=$(echo "${CI_COMMIT_TAG}" | sed -nE 's/^octez-evm-node-v([0-9]+)\.([0-9]+)((-rc[0-9]+)?|(-beta[0-9]+)?)$/\2/p') +# Z +gitlab_release_rc_version=$(echo "${CI_COMMIT_TAG}" | sed -nE 's/^octez-evm-node-v([0-9]+)\.([0-9]+)(-rc)?([0-9]+)?$/\4/p') +# Beta +gitlab_release_beta_version=$(echo "${CI_COMMIT_TAG}" | sed -nE 's/^octez-evm-node-v([0-9]+)\.([0-9]+)(-beta)?([0-9]+)?$/\4/p') + +# Is this a release candidate? +if [ -n "${gitlab_release_rc_version}" ]; then + # Yes, release name: X.Y~rcZ + # shellcheck disable=SC2034 + gitlab_release_name="Octez EVM Node Release Candidate ${gitlab_release_major_version}.${gitlab_release_minor_version}~rc${gitlab_release_rc_version}" +# Is this a beta ? +elif [ -n "${gitlab_release_beta_version}" ]; then + gitlab_release_name="Octez EVM Node Beta ${gitlab_release_major_version}.${gitlab_release_minor_version}~beta${gitlab_release_beta_version}" +else + # No, release name: Octez EVM Node Release X.Y + # shellcheck disable=SC2034 + gitlab_release_name="Octez EVM Node Release ${gitlab_release_major_version}.${gitlab_release_minor_version}" +fi + +### Compute GitLab generic package names + +suffix="${gitlab_release_no_v}" +gitlab_octez_binaries_package_name="octez-evm-node-${suffix}" + +# X.Y or X.Y-rcZ +gitlab_package_version="${suffix}" -- GitLab