From 709cce6ed41e8359c0a8785003661f1bf595191f Mon Sep 17 00:00:00 2001 From: Maxime Levillain Date: Wed, 3 Apr 2024 16:46:57 +0200 Subject: [PATCH 1/3] CI/lib: expiration type with `Never` case --- ci/bin/code_verification.ml | 16 ++++++++++------ ci/bin/common.ml | 10 +++++----- ci/bin/tezos_ci.ml | 30 +++++++++++++++++------------- ci/bin/tezos_ci.mli | 2 +- ci/lib_gitlab_ci/to_yaml.ml | 6 +++++- ci/lib_gitlab_ci/types.ml | 4 +++- ci/lib_gitlab_ci/util.mli | 2 +- 7 files changed, 42 insertions(+), 28 deletions(-) diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 1e75cf177203..1e885b045d6b 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -137,7 +137,8 @@ let job_opam_package ?dependencies {name; group; batch_index} : tezos_job = "eval $(opam env)"; "OPAM_LOGS=opam_logs ./scripts/ci/opam_handle_output.sh"; ] - ~artifacts:(artifacts ~expire_in:(Weeks 1) ~when_:Always ["opam_logs/"]) + ~artifacts: + (artifacts ~expire_in:(Duration (Weeks 1)) ~when_:Always ["opam_logs/"]) ~cache:[{key = "opam-sccache"; paths = ["_build/_sccache"]}] |> (* We store caches in [_build] for two reasons: (1) the [_build] folder is excluded from opam's rsync. (2) gitlab ci cache @@ -237,7 +238,7 @@ let jobs_unit_tests ~job_build_x86_64_release ~job_build_x86_64_exp_dev_extra ~name:"$CI_JOB_NAME-$CI_COMMIT_SHA-${ARCH}" ["test_results"] ~reports:(reports ~junit:"test_results/*.xml" ()) - ~expire_in:(Days 1) + ~expire_in:(Duration (Days 1)) ~when_:Always) ~before_script:(before_script ~source_version:true ~eval_opam:true []) script @@ -590,7 +591,7 @@ let jobs pipeline_type = ~artifacts: (artifacts ~name:"build-kernels-$CI_COMMIT_REF_SLUG" - ~expire_in:(Days 1) + ~expire_in:(Duration (Days 1)) ~when_:On_success [ "evm_kernel.wasm"; @@ -637,7 +638,7 @@ let jobs pipeline_type = ~allow_failure:Yes ~artifacts: (artifacts - ~expire_in:(Hours 4) + ~expire_in:(Duration (Hours 4)) ~when_:Always [ "tezt-fetch-records.log"; @@ -665,7 +666,10 @@ let jobs pipeline_type = ["scripts/ci/select_tezts.sh || exit $?"] ~allow_failure:(With_exit_codes [17]) ~artifacts: - (artifacts ~expire_in:(Days 3) ~when_:Always ["selected_tezts.tsl"]) + (artifacts + ~expire_in:(Duration (Days 3)) + ~when_:Always + ["selected_tezts.tsl"]) |> job_external_once in [ @@ -730,7 +734,7 @@ let jobs pipeline_type = ] ~artifacts: (artifacts - ~expire_in:(Hours 1) + ~expire_in:(Duration (Hours 1)) ~when_:On_success ["_build/default/client-libs/bin_codec_kaitai/codec.exe"]) |> job_external_split diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 2e66043289aa..23a55bc27e56 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -215,7 +215,7 @@ let enable_coverage_location : tezos_job -> tezos_job = artifact. This function should be applied to test jobs that produce coverage. *) -let enable_coverage_output_artifact ?(expire_in = Days 1) : +let enable_coverage_output_artifact ?(expire_in = Duration (Days 1)) : tezos_job -> tezos_job = fun job -> job |> enable_coverage_location @@ -237,7 +237,7 @@ let enable_coverage_report job : tezos_job = path = "_coverage_report/cobertura.xml"; } ()) - ~expire_in:(Days 15) + ~expire_in:(Duration (Days 15)) ~when_:Always ["_coverage_report/"; "$BISECT_FILE"] |> Tezos_ci.append_variables [("SLACK_COVERAGE_CHANNEL", "C02PHBE7W73")] @@ -437,7 +437,7 @@ let job_build_static_binaries ~__POS__ ~arch ?(release = false) ?rules let name = "oc.build:static-" ^ arch_string ^ "-linux-binaries" in let artifacts = (* Extend the lifespan to prevent failure for external tools using artifacts. *) - let expire_in = if release then Some (Days 90) else None in + let expire_in = if release then Some (Duration (Days 90)) else None in artifacts ?expire_in ["octez-binaries/$ARCH/*"] in let executable_files = @@ -638,7 +638,7 @@ let job_build_bin_package ?dependencies ?rules ~__POS__ ~name // ("octez-*." ^ match target with Dpkg -> "deb" | Rpm -> "rpm") in artifacts - ~expire_in:(Days 1) + ~expire_in:(Duration (Days 1)) ~when_:On_success ~name:"${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG" [artifact_path] @@ -748,7 +748,7 @@ let job_build_dynamic_binaries ?rules ~__POS__ ~arch ?(release = false) artifacts ~name:"build-$ARCH-$CI_COMMIT_REF_SLUG" ~when_:On_success - ~expire_in:(Days 1) + ~expire_in:(Duration (Days 1)) (* TODO: [paths] can be refined based on [release] *) [ "octez-*"; diff --git a/ci/bin/tezos_ci.ml b/ci/bin/tezos_ci.ml index a6e9f940a8b9..ad13fc1b77c5 100644 --- a/ci/bin/tezos_ci.ml +++ b/ci/bin/tezos_ci.ml @@ -482,21 +482,25 @@ let add_artifacts ?name ?expose_as ?reports ?expire_in ?when_ paths let name = opt_merge_right artifacts.name name in let expose_as = opt_merge_right artifacts.expose_as expose_as in let expire_in = - opt_combine artifacts.expire_in expire_in @@ fun interval1 interval2 -> + opt_combine artifacts.expire_in expire_in + @@ fun expiration1 expiration2 -> (* This function gives a measure of the size of a duration in seconds. It is only used to compare durations. *) - let interval_to_seconds = function - | Gitlab_ci.Types.Seconds n -> n - | Minutes n -> n * 60 - | Hours n -> n * 3600 - | Days n -> n * 24 * 3600 - | Weeks n -> n * 7 * 24 * 3600 - | Months n -> n * 31 * 7 * 24 * 3600 - | Years n -> n * 365 * 7 * 24 * 3600 - in - if interval_to_seconds interval1 > interval_to_seconds interval2 then - interval1 - else interval2 + match (expiration1, expiration2) with + | Never, _ | _, Never -> Never + | Duration interval1, Duration interval2 -> + let interval_to_seconds = function + | Gitlab_ci.Types.Seconds n -> n + | Minutes n -> n * 60 + | Hours n -> n * 3600 + | Days n -> n * 24 * 3600 + | Weeks n -> n * 7 * 24 * 3600 + | Months n -> n * 31 * 7 * 24 * 3600 + | Years n -> n * 365 * 7 * 24 * 3600 + in + if interval_to_seconds interval1 > interval_to_seconds interval2 + then Duration interval1 + else Duration interval2 in let when_ = opt_combine artifacts.when_ when_ @@ fun when1 when2 -> diff --git a/ci/bin/tezos_ci.mli b/ci/bin/tezos_ci.mli index f982ce15aa85..d83bc804d638 100644 --- a/ci/bin/tezos_ci.mli +++ b/ci/bin/tezos_ci.mli @@ -287,7 +287,7 @@ val add_artifacts : ?name:string -> ?expose_as:string -> ?reports:Gitlab_ci.Types.reports -> - ?expire_in:Gitlab_ci.Types.time_interval -> + ?expire_in:Gitlab_ci.Types.expiration -> ?when_:Gitlab_ci.Types.when_artifact -> string list -> tezos_job -> diff --git a/ci/lib_gitlab_ci/to_yaml.ml b/ci/lib_gitlab_ci/to_yaml.ml index 66d265f761ae..41a4fdb547f4 100644 --- a/ci/lib_gitlab_ci/to_yaml.ml +++ b/ci/lib_gitlab_ci/to_yaml.ml @@ -90,6 +90,10 @@ let enc_time_interval interval = | Years 1 -> "1 year" | Years x -> string_of_int x ^ " years") +let enc_expiration = function + | Duration interval -> enc_time_interval interval + | Never -> `String "never" + let enc_job_rule : job_rule -> value = fun {changes; if_; variables; when_; allow_failure} -> let start_in = @@ -157,7 +161,7 @@ let enc_artifacts : artifacts -> value = obj_flatten [ opt "name" string name; - opt "expire_in" enc_time_interval expire_in; + opt "expire_in" enc_expiration expire_in; opt "paths" strings paths; opt "reports" enc_report reports; opt "when" enc_when_artifact when_; diff --git a/ci/lib_gitlab_ci/types.ml b/ci/lib_gitlab_ci/types.ml index cb36c9822b70..d725a6feaf40 100644 --- a/ci/lib_gitlab_ci/types.ml +++ b/ci/lib_gitlab_ci/types.ml @@ -20,6 +20,8 @@ type time_interval = | Months of int | Years of int +type expiration = Duration of time_interval | Never + (** Represents values of the [when:] field in job rules. *) type when_ = Always | Never | On_success | Manual | Delayed of time_interval @@ -77,7 +79,7 @@ type image = Image of string type when_artifact = Always | On_success | On_failure type artifacts = { - expire_in : time_interval option; + expire_in : expiration option; paths : string list option; reports : reports option; when_ : when_artifact option; diff --git a/ci/lib_gitlab_ci/util.mli b/ci/lib_gitlab_ci/util.mli index 03f2e3a59c96..68694de8831e 100644 --- a/ci/lib_gitlab_ci/util.mli +++ b/ci/lib_gitlab_ci/util.mli @@ -90,7 +90,7 @@ val job : At least one of [paths] or [reports] must be non-empty. *) val artifacts : - ?expire_in:time_interval -> + ?expire_in:expiration -> ?reports:reports -> ?when_:when_artifact -> ?expose_as:string -> -- GitLab From 2cb947ec8cf58a8cae0839253eacb609deb01b3c Mon Sep 17 00:00:00 2001 From: Maxime Levillain Date: Wed, 3 Apr 2024 16:51:20 +0200 Subject: [PATCH 2/3] EVM/Makefile: display root hash option in Makefile --- etherlink.mk | 17 +++++++++++++---- .../evm_kernel_builder.Dockerfile | 4 ++-- kernels.mk | 1 + 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/etherlink.mk b/etherlink.mk index da0984d9cacc..ae8556085f53 100644 --- a/etherlink.mk +++ b/etherlink.mk @@ -35,22 +35,30 @@ kernel_sdk: evm_installer.wasm:: kernel_sdk evm_kernel.wasm ifdef EVM_CONFIG $(eval CONFIG := --setup-file ${EVM_CONFIG}) +endif +ifeq (${DISPLAY_ROOT_HASH}, true) + $(eval DISPLAY_CONFIG := --display-root-hash) endif @./smart-rollup-installer get-reveal-installer \ --upgrade-to evm_kernel.wasm \ --preimages-dir ${EVM_KERNEL_PREIMAGES} \ --output $@ \ - ${CONFIG} + ${CONFIG} \ + ${DISPLAY_CONFIG} evm_unstripped_installer.wasm:: kernel_sdk evm_kernel_unstripped.wasm ifdef EVM_CONFIG $(eval CONFIG := --setup-file ${EVM_CONFIG}) +endif +ifeq (${DISPLAY_ROOT_HASH}, true) + $(eval DISPLAY_CONFIG := --display-root-hash) endif @./smart-rollup-installer get-reveal-installer \ --upgrade-to evm_kernel_unstripped.wasm \ --preimages-dir ${EVM_UNSTRIPPED_KERNEL_PREIMAGES} \ --output $@ \ - ${CONFIG} + ${CONFIG} \ + ${DISPLAY_CONFIG} evm_benchmark_kernel.wasm:: @${MAKE} -f etherlink.mk \ @@ -89,9 +97,10 @@ check: build-dev-deps .PHONY: clean clean: - @rm -f ${KERNELS} + @$(MAKE) -f kernels.mk clean + @rm -f evm_kernel_unstripped.wasm evm_kernel.wasm evm_installer.wasm evm_unstripped_installer.wasm evm_installer.wasm evm_installer_dev.wasm evm_benchmark_kernel.wasm sequencer.wasm @$(MAKE) -C ${EVM_DIR} clean - @rm -rf ${EVM_KERNEL_PREIMAGES} + @rm -rf ${EVM_KERNEL_PREIMAGES} ${EVM_UNSTRIPPED_KERNEL_PREIMAGES} sequencer.wasm:: @${MAKE} -f etherlink.mk EVM_CONFIG=etherlink/config/sequencer.yaml evm_installer.wasm diff --git a/etherlink/scripts/docker-compose/evm_kernel_builder.Dockerfile b/etherlink/scripts/docker-compose/evm_kernel_builder.Dockerfile index 6b52ad8485ca..b44ea0c1ec53 100644 --- a/etherlink/scripts/docker-compose/evm_kernel_builder.Dockerfile +++ b/etherlink/scripts/docker-compose/evm_kernel_builder.Dockerfile @@ -14,8 +14,8 @@ COPY kernels.mk etherlink.mk /build/ COPY src/kernel_sdk /build/src/kernel_sdk COPY etherlink /build/etherlink RUN make -f etherlink.mk build-deps -RUN make -f etherlink.mk EVM_CONFIG=${EVM_CONFIG} CI_COMMIT_SHA=${CI_COMMIT_SHA} evm_installer.wasm +RUN make --no-print-directory -f etherlink.mk EVM_CONFIG=${EVM_CONFIG} CI_COMMIT_SHA=${CI_COMMIT_SHA} DISPLAY_ROOT_HASH=true evm_installer.wasm > root_hash FROM ${BASE_IMAGE} -COPY --from=kernel_build /build/*.wasm /kernel/ +COPY --from=kernel_build /build/*.wasm /build/root_hash /kernel/ COPY --from=kernel_build /build/_evm_installer_preimages /kernel/_evm_installer_preimages diff --git a/kernels.mk b/kernels.mk index 131f66e93733..aab344ef3664 100644 --- a/kernels.mk +++ b/kernels.mk @@ -87,3 +87,4 @@ clean: @rm -f ${KERNELS} @make -C ${SDK_DIR} clean @make -C ${DEMO_DIR} clean + @rm -f smart-rollup-installer tx-demo-collector -- GitLab From 8d7f6de8acc6a8e5622efeeba20ce77deb58913e Mon Sep 17 00:00:00 2001 From: Maxime Levillain Date: Wed, 3 Apr 2024 16:52:07 +0200 Subject: [PATCH 3/3] EVM/CI: publich evm kernel and preimages --- .../ci/pipelines/etherlink_release_tag.yml | 28 +++++++++++++++++-- ci/bin/release_tag.ml | 18 ++++++++++-- etherlink/scripts/build-wasm.sh | 12 ++++---- scripts/ci/create_gitlab_etherlink_release.sh | 7 +++-- .../ci/docker_prepare_etherlink_release.sh | 17 +++++++++++ 5 files changed, 69 insertions(+), 13 deletions(-) create mode 100755 scripts/ci/docker_prepare_etherlink_release.sh diff --git a/.gitlab/ci/pipelines/etherlink_release_tag.yml b/.gitlab/ci/pipelines/etherlink_release_tag.yml index d2803c2a1c7b..bb9334466c12 100644 --- a/.gitlab/ci/pipelines/etherlink_release_tag.yml +++ b/.gitlab/ci/pipelines/etherlink_release_tag.yml @@ -1,13 +1,37 @@ # This file was automatically generated, do not edit. # Edit file ci/bin/main.ml instead. -gitlab:publish: +docker:prepare-etherlink-release: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + stage: prepare_release + tags: + - gcp + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_prepare_etherlink_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + artifacts: + expire_in: 1 hour + paths: + - kernels.tar.gz + +gitlab:etherlink-release: image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.4.0 stage: publish_package_gitlab tags: - gcp - needs: [] + needs: + - docker:prepare-etherlink-release dependencies: [] interruptible: false script: - ./scripts/ci/create_gitlab_etherlink_release.sh + artifacts: + expire_in: never + paths: + - kernels.tar.gz diff --git a/ci/bin/release_tag.ml b/ci/bin/release_tag.ml index 4a2c44c589ca..a8df04b50d91 100644 --- a/ci/bin/release_tag.ml +++ b/ci/bin/release_tag.ml @@ -130,14 +130,26 @@ let octez_jobs ?(test = false) release_tag_pipeline_type = (** Create an etherlink release tag pipeline of type {!release_tag_pipeline_type}. *) let etherlink_jobs () = + let job_produce_docker_artifacts : Tezos_ci.tezos_job = + job_docker_authenticated + ~__POS__ + ~stage:Stages.prepare_release + ~name:"docker:prepare-etherlink-release" + ~artifacts: + (Gitlab_ci.Util.artifacts + ~expire_in:(Duration (Hours 1)) + ["kernels.tar.gz"]) + ["./scripts/ci/docker_prepare_etherlink_release.sh"] + in let job_gitlab_release : Tezos_ci.tezos_job = job ~__POS__ ~image:Images.ci_release ~stage:Stages.publish_package_gitlab ~interruptible:false - ~dependencies:(Dependent []) - ~name:"gitlab:publish" + ~dependencies:(Dependent [Job job_produce_docker_artifacts]) + ~artifacts:(Gitlab_ci.Util.artifacts ~expire_in:Never ["kernels.tar.gz"]) + ~name:"gitlab:etherlink-release" ["./scripts/ci/create_gitlab_etherlink_release.sh"] in - [job_gitlab_release] + [job_produce_docker_artifacts; job_gitlab_release] diff --git a/etherlink/scripts/build-wasm.sh b/etherlink/scripts/build-wasm.sh index 9f7dbc59feb4..4043bdd58af3 100755 --- a/etherlink/scripts/build-wasm.sh +++ b/etherlink/scripts/build-wasm.sh @@ -12,8 +12,8 @@ rust_image_tag=${rust_image_tag:-"master"} platform=${platform:-"linux/amd64"} # register information about the rust-toolchain image -rust_toolchain_info() { - docker pull -q "${rust_image}:${rust_image_tag}" +rust_toolchain_image_id() { + docker pull -q "${rust_image}:${rust_image_tag}" &> /dev/null docker inspect --format='{{index .RepoDigests 0}}' "${rust_image}:${rust_image_tag}" } @@ -33,11 +33,11 @@ build() { # copy images in output directory copy() { output_dir=$1 - rust_image_version=$2 + rust_image_id=$2 container=$(docker create etherlink_kernel:"$commit") docker cp "$container":/kernel "$output_dir" { - echo "rust-toolchain: $rust_image_version" + echo "rust-toolchain: $rust_image_id" echo "tezos: $commit" } > "$output_dir/.versions" echo "$container" @@ -71,11 +71,11 @@ EOF evm_config=${1:-"$etherlink_dir/config/dev.yaml"} output_dir=${2:-"$etherlink_dir/kernels-$commit"} echo "fetching rust-toolchain image info: $rust_image:$rust_image_tag" - rust_image_version=$(rust_toolchain_info) + rust_image_id=$(rust_toolchain_image_id) echo "building the docker image with $evm_config" build "$evm_config" echo "copying kernels and preimages in $output_dir" - container=$(copy "$output_dir" "$rust_image_version") + container=$(copy "$output_dir" "$rust_image_id") echo "cleaning up" cleanup "$container" ;; diff --git a/scripts/ci/create_gitlab_etherlink_release.sh b/scripts/ci/create_gitlab_etherlink_release.sh index 9453d92f5c85..89ec92453a56 100755 --- a/scripts/ci/create_gitlab_etherlink_release.sh +++ b/scripts/ci/create_gitlab_etherlink_release.sh @@ -3,9 +3,12 @@ set -eu ### Create a GitLab Etherlink release with links to all the related resources -# shellcheck source=./scripts/ci/octez-release.sh +# shellcheck source=./scripts/ci/etherlink-release.sh . ./scripts/ci/etherlink-release.sh +kernels_artifact_url="$CI_JOB_URL/artifacts/raw/kernels.tar.gz" + release-cli create \ --name="${gitlab_release_name}" \ - --tag-name="${CI_COMMIT_TAG}" + --tag-name="${CI_COMMIT_TAG}" \ + --assets-link="{\"name\":\"Kernels\",\"url\":\"${kernels_artifact_url}\"}" diff --git a/scripts/ci/docker_prepare_etherlink_release.sh b/scripts/ci/docker_prepare_etherlink_release.sh new file mode 100755 index 000000000000..c7faa7763307 --- /dev/null +++ b/scripts/ci/docker_prepare_etherlink_release.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -eu + +### Prepare etherlink release by building wasm and preimages with docker + +current_dir=$(cd "$(dirname "${0}")" && pwd) + +. scripts/ci/docker.env + +# shellcheck source=./scripts/ci/docker.sh +. "${current_dir}/docker.sh" + +# produce evm wasms and preimages +etherlink/scripts/build-wasm.sh etherlink/config/dev.yaml kernels + +# compress the directory +tar -czf kernels.tar.gz kernels -- GitLab