diff --git a/.gitlab/ci/pipelines/etherlink_release_tag.yml b/.gitlab/ci/pipelines/etherlink_release_tag.yml index d2803c2a1c7b6a64154eafe56dc64b487a5a07b3..bb9334466c1262fac66e42ffb3fd1325baad1558 100644 --- a/.gitlab/ci/pipelines/etherlink_release_tag.yml +++ b/.gitlab/ci/pipelines/etherlink_release_tag.yml @@ -1,13 +1,37 @@ # This file was automatically generated, do not edit. # Edit file ci/bin/main.ml instead. -gitlab:publish: +docker:prepare-etherlink-release: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.10.0 + stage: prepare_release + tags: + - gcp + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_prepare_etherlink_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + artifacts: + expire_in: 1 hour + paths: + - kernels.tar.gz + +gitlab:etherlink-release: image: ${GCP_REGISTRY}/tezos/docker-images/ci-release:v1.4.0 stage: publish_package_gitlab tags: - gcp - needs: [] + needs: + - docker:prepare-etherlink-release dependencies: [] interruptible: false script: - ./scripts/ci/create_gitlab_etherlink_release.sh + artifacts: + expire_in: never + paths: + - kernels.tar.gz diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 1e75cf177203faeff0136e5d7b0f679caa1b8b2b..1e885b045d6b80d273fedfd73287d64d194815d0 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -137,7 +137,8 @@ let job_opam_package ?dependencies {name; group; batch_index} : tezos_job = "eval $(opam env)"; "OPAM_LOGS=opam_logs ./scripts/ci/opam_handle_output.sh"; ] - ~artifacts:(artifacts ~expire_in:(Weeks 1) ~when_:Always ["opam_logs/"]) + ~artifacts: + (artifacts ~expire_in:(Duration (Weeks 1)) ~when_:Always ["opam_logs/"]) ~cache:[{key = "opam-sccache"; paths = ["_build/_sccache"]}] |> (* We store caches in [_build] for two reasons: (1) the [_build] folder is excluded from opam's rsync. (2) gitlab ci cache @@ -237,7 +238,7 @@ let jobs_unit_tests ~job_build_x86_64_release ~job_build_x86_64_exp_dev_extra ~name:"$CI_JOB_NAME-$CI_COMMIT_SHA-${ARCH}" ["test_results"] ~reports:(reports ~junit:"test_results/*.xml" ()) - ~expire_in:(Days 1) + ~expire_in:(Duration (Days 1)) ~when_:Always) ~before_script:(before_script ~source_version:true ~eval_opam:true []) script @@ -590,7 +591,7 @@ let jobs pipeline_type = ~artifacts: (artifacts ~name:"build-kernels-$CI_COMMIT_REF_SLUG" - ~expire_in:(Days 1) + ~expire_in:(Duration (Days 1)) ~when_:On_success [ "evm_kernel.wasm"; @@ -637,7 +638,7 @@ let jobs pipeline_type = ~allow_failure:Yes ~artifacts: (artifacts - ~expire_in:(Hours 4) + ~expire_in:(Duration (Hours 4)) ~when_:Always [ "tezt-fetch-records.log"; @@ -665,7 +666,10 @@ let jobs pipeline_type = ["scripts/ci/select_tezts.sh || exit $?"] ~allow_failure:(With_exit_codes [17]) ~artifacts: - (artifacts ~expire_in:(Days 3) ~when_:Always ["selected_tezts.tsl"]) + (artifacts + ~expire_in:(Duration (Days 3)) + ~when_:Always + ["selected_tezts.tsl"]) |> job_external_once in [ @@ -730,7 +734,7 @@ let jobs pipeline_type = ] ~artifacts: (artifacts - ~expire_in:(Hours 1) + ~expire_in:(Duration (Hours 1)) ~when_:On_success ["_build/default/client-libs/bin_codec_kaitai/codec.exe"]) |> job_external_split diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 2e66043289aaae9e0abb4c47ab41e670df1810af..23a55bc27e564404bb41c792ab189b8824dfc9ba 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -215,7 +215,7 @@ let enable_coverage_location : tezos_job -> tezos_job = artifact. This function should be applied to test jobs that produce coverage. *) -let enable_coverage_output_artifact ?(expire_in = Days 1) : +let enable_coverage_output_artifact ?(expire_in = Duration (Days 1)) : tezos_job -> tezos_job = fun job -> job |> enable_coverage_location @@ -237,7 +237,7 @@ let enable_coverage_report job : tezos_job = path = "_coverage_report/cobertura.xml"; } ()) - ~expire_in:(Days 15) + ~expire_in:(Duration (Days 15)) ~when_:Always ["_coverage_report/"; "$BISECT_FILE"] |> Tezos_ci.append_variables [("SLACK_COVERAGE_CHANNEL", "C02PHBE7W73")] @@ -437,7 +437,7 @@ let job_build_static_binaries ~__POS__ ~arch ?(release = false) ?rules let name = "oc.build:static-" ^ arch_string ^ "-linux-binaries" in let artifacts = (* Extend the lifespan to prevent failure for external tools using artifacts. *) - let expire_in = if release then Some (Days 90) else None in + let expire_in = if release then Some (Duration (Days 90)) else None in artifacts ?expire_in ["octez-binaries/$ARCH/*"] in let executable_files = @@ -638,7 +638,7 @@ let job_build_bin_package ?dependencies ?rules ~__POS__ ~name // ("octez-*." ^ match target with Dpkg -> "deb" | Rpm -> "rpm") in artifacts - ~expire_in:(Days 1) + ~expire_in:(Duration (Days 1)) ~when_:On_success ~name:"${TARGET}-$ARCH-$CI_COMMIT_REF_SLUG" [artifact_path] @@ -748,7 +748,7 @@ let job_build_dynamic_binaries ?rules ~__POS__ ~arch ?(release = false) artifacts ~name:"build-$ARCH-$CI_COMMIT_REF_SLUG" ~when_:On_success - ~expire_in:(Days 1) + ~expire_in:(Duration (Days 1)) (* TODO: [paths] can be refined based on [release] *) [ "octez-*"; diff --git a/ci/bin/release_tag.ml b/ci/bin/release_tag.ml index 4a2c44c589cab1cb4f3a2217dc8ca67b81e4fa14..a8df04b50d91f6f33fa2f5e1f4a045d529095154 100644 --- a/ci/bin/release_tag.ml +++ b/ci/bin/release_tag.ml @@ -130,14 +130,26 @@ let octez_jobs ?(test = false) release_tag_pipeline_type = (** Create an etherlink release tag pipeline of type {!release_tag_pipeline_type}. *) let etherlink_jobs () = + let job_produce_docker_artifacts : Tezos_ci.tezos_job = + job_docker_authenticated + ~__POS__ + ~stage:Stages.prepare_release + ~name:"docker:prepare-etherlink-release" + ~artifacts: + (Gitlab_ci.Util.artifacts + ~expire_in:(Duration (Hours 1)) + ["kernels.tar.gz"]) + ["./scripts/ci/docker_prepare_etherlink_release.sh"] + in let job_gitlab_release : Tezos_ci.tezos_job = job ~__POS__ ~image:Images.ci_release ~stage:Stages.publish_package_gitlab ~interruptible:false - ~dependencies:(Dependent []) - ~name:"gitlab:publish" + ~dependencies:(Dependent [Job job_produce_docker_artifacts]) + ~artifacts:(Gitlab_ci.Util.artifacts ~expire_in:Never ["kernels.tar.gz"]) + ~name:"gitlab:etherlink-release" ["./scripts/ci/create_gitlab_etherlink_release.sh"] in - [job_gitlab_release] + [job_produce_docker_artifacts; job_gitlab_release] diff --git a/ci/bin/tezos_ci.ml b/ci/bin/tezos_ci.ml index a6e9f940a8b9e072c2d97681231ef1b6b45b001e..ad13fc1b77c546f879824816b5d1656af55e5842 100644 --- a/ci/bin/tezos_ci.ml +++ b/ci/bin/tezos_ci.ml @@ -482,21 +482,25 @@ let add_artifacts ?name ?expose_as ?reports ?expire_in ?when_ paths let name = opt_merge_right artifacts.name name in let expose_as = opt_merge_right artifacts.expose_as expose_as in let expire_in = - opt_combine artifacts.expire_in expire_in @@ fun interval1 interval2 -> + opt_combine artifacts.expire_in expire_in + @@ fun expiration1 expiration2 -> (* This function gives a measure of the size of a duration in seconds. It is only used to compare durations. *) - let interval_to_seconds = function - | Gitlab_ci.Types.Seconds n -> n - | Minutes n -> n * 60 - | Hours n -> n * 3600 - | Days n -> n * 24 * 3600 - | Weeks n -> n * 7 * 24 * 3600 - | Months n -> n * 31 * 7 * 24 * 3600 - | Years n -> n * 365 * 7 * 24 * 3600 - in - if interval_to_seconds interval1 > interval_to_seconds interval2 then - interval1 - else interval2 + match (expiration1, expiration2) with + | Never, _ | _, Never -> Never + | Duration interval1, Duration interval2 -> + let interval_to_seconds = function + | Gitlab_ci.Types.Seconds n -> n + | Minutes n -> n * 60 + | Hours n -> n * 3600 + | Days n -> n * 24 * 3600 + | Weeks n -> n * 7 * 24 * 3600 + | Months n -> n * 31 * 7 * 24 * 3600 + | Years n -> n * 365 * 7 * 24 * 3600 + in + if interval_to_seconds interval1 > interval_to_seconds interval2 + then Duration interval1 + else Duration interval2 in let when_ = opt_combine artifacts.when_ when_ @@ fun when1 when2 -> diff --git a/ci/bin/tezos_ci.mli b/ci/bin/tezos_ci.mli index f982ce15aa853868cf2ef2159a83f385faff7588..d83bc804d638d0ffc6aae9ca740f7678b8c8ed65 100644 --- a/ci/bin/tezos_ci.mli +++ b/ci/bin/tezos_ci.mli @@ -287,7 +287,7 @@ val add_artifacts : ?name:string -> ?expose_as:string -> ?reports:Gitlab_ci.Types.reports -> - ?expire_in:Gitlab_ci.Types.time_interval -> + ?expire_in:Gitlab_ci.Types.expiration -> ?when_:Gitlab_ci.Types.when_artifact -> string list -> tezos_job -> diff --git a/ci/lib_gitlab_ci/to_yaml.ml b/ci/lib_gitlab_ci/to_yaml.ml index 66d265f761aec1332e74cf15d269f8f02b63dc1f..41a4fdb547f42fb1886625cd5de7e31fe1d225a4 100644 --- a/ci/lib_gitlab_ci/to_yaml.ml +++ b/ci/lib_gitlab_ci/to_yaml.ml @@ -90,6 +90,10 @@ let enc_time_interval interval = | Years 1 -> "1 year" | Years x -> string_of_int x ^ " years") +let enc_expiration = function + | Duration interval -> enc_time_interval interval + | Never -> `String "never" + let enc_job_rule : job_rule -> value = fun {changes; if_; variables; when_; allow_failure} -> let start_in = @@ -157,7 +161,7 @@ let enc_artifacts : artifacts -> value = obj_flatten [ opt "name" string name; - opt "expire_in" enc_time_interval expire_in; + opt "expire_in" enc_expiration expire_in; opt "paths" strings paths; opt "reports" enc_report reports; opt "when" enc_when_artifact when_; diff --git a/ci/lib_gitlab_ci/types.ml b/ci/lib_gitlab_ci/types.ml index cb36c9822b7031d3d0866c554915828b74c3deaf..d725a6feaf40791a1f51f0c41f4c05717c17e7d8 100644 --- a/ci/lib_gitlab_ci/types.ml +++ b/ci/lib_gitlab_ci/types.ml @@ -20,6 +20,8 @@ type time_interval = | Months of int | Years of int +type expiration = Duration of time_interval | Never + (** Represents values of the [when:] field in job rules. *) type when_ = Always | Never | On_success | Manual | Delayed of time_interval @@ -77,7 +79,7 @@ type image = Image of string type when_artifact = Always | On_success | On_failure type artifacts = { - expire_in : time_interval option; + expire_in : expiration option; paths : string list option; reports : reports option; when_ : when_artifact option; diff --git a/ci/lib_gitlab_ci/util.mli b/ci/lib_gitlab_ci/util.mli index 03f2e3a59c96ff1b4fcd2f15e37e7471fa8f287f..68694de8831e4854741d34df14d98cda509f6114 100644 --- a/ci/lib_gitlab_ci/util.mli +++ b/ci/lib_gitlab_ci/util.mli @@ -90,7 +90,7 @@ val job : At least one of [paths] or [reports] must be non-empty. *) val artifacts : - ?expire_in:time_interval -> + ?expire_in:expiration -> ?reports:reports -> ?when_:when_artifact -> ?expose_as:string -> diff --git a/etherlink.mk b/etherlink.mk index da0984d9cacc94897bcb0a61a69696e4b30d335f..ae8556085f535412cfc9014e855b4a76c9886d22 100644 --- a/etherlink.mk +++ b/etherlink.mk @@ -35,22 +35,30 @@ kernel_sdk: evm_installer.wasm:: kernel_sdk evm_kernel.wasm ifdef EVM_CONFIG $(eval CONFIG := --setup-file ${EVM_CONFIG}) +endif +ifeq (${DISPLAY_ROOT_HASH}, true) + $(eval DISPLAY_CONFIG := --display-root-hash) endif @./smart-rollup-installer get-reveal-installer \ --upgrade-to evm_kernel.wasm \ --preimages-dir ${EVM_KERNEL_PREIMAGES} \ --output $@ \ - ${CONFIG} + ${CONFIG} \ + ${DISPLAY_CONFIG} evm_unstripped_installer.wasm:: kernel_sdk evm_kernel_unstripped.wasm ifdef EVM_CONFIG $(eval CONFIG := --setup-file ${EVM_CONFIG}) +endif +ifeq (${DISPLAY_ROOT_HASH}, true) + $(eval DISPLAY_CONFIG := --display-root-hash) endif @./smart-rollup-installer get-reveal-installer \ --upgrade-to evm_kernel_unstripped.wasm \ --preimages-dir ${EVM_UNSTRIPPED_KERNEL_PREIMAGES} \ --output $@ \ - ${CONFIG} + ${CONFIG} \ + ${DISPLAY_CONFIG} evm_benchmark_kernel.wasm:: @${MAKE} -f etherlink.mk \ @@ -89,9 +97,10 @@ check: build-dev-deps .PHONY: clean clean: - @rm -f ${KERNELS} + @$(MAKE) -f kernels.mk clean + @rm -f evm_kernel_unstripped.wasm evm_kernel.wasm evm_installer.wasm evm_unstripped_installer.wasm evm_installer.wasm evm_installer_dev.wasm evm_benchmark_kernel.wasm sequencer.wasm @$(MAKE) -C ${EVM_DIR} clean - @rm -rf ${EVM_KERNEL_PREIMAGES} + @rm -rf ${EVM_KERNEL_PREIMAGES} ${EVM_UNSTRIPPED_KERNEL_PREIMAGES} sequencer.wasm:: @${MAKE} -f etherlink.mk EVM_CONFIG=etherlink/config/sequencer.yaml evm_installer.wasm diff --git a/etherlink/scripts/build-wasm.sh b/etherlink/scripts/build-wasm.sh index 9f7dbc59feb4a29ce2286ae467be296bb52fba66..4043bdd58af3735c7849aa6d39e704c97340b224 100755 --- a/etherlink/scripts/build-wasm.sh +++ b/etherlink/scripts/build-wasm.sh @@ -12,8 +12,8 @@ rust_image_tag=${rust_image_tag:-"master"} platform=${platform:-"linux/amd64"} # register information about the rust-toolchain image -rust_toolchain_info() { - docker pull -q "${rust_image}:${rust_image_tag}" +rust_toolchain_image_id() { + docker pull -q "${rust_image}:${rust_image_tag}" &> /dev/null docker inspect --format='{{index .RepoDigests 0}}' "${rust_image}:${rust_image_tag}" } @@ -33,11 +33,11 @@ build() { # copy images in output directory copy() { output_dir=$1 - rust_image_version=$2 + rust_image_id=$2 container=$(docker create etherlink_kernel:"$commit") docker cp "$container":/kernel "$output_dir" { - echo "rust-toolchain: $rust_image_version" + echo "rust-toolchain: $rust_image_id" echo "tezos: $commit" } > "$output_dir/.versions" echo "$container" @@ -71,11 +71,11 @@ EOF evm_config=${1:-"$etherlink_dir/config/dev.yaml"} output_dir=${2:-"$etherlink_dir/kernels-$commit"} echo "fetching rust-toolchain image info: $rust_image:$rust_image_tag" - rust_image_version=$(rust_toolchain_info) + rust_image_id=$(rust_toolchain_image_id) echo "building the docker image with $evm_config" build "$evm_config" echo "copying kernels and preimages in $output_dir" - container=$(copy "$output_dir" "$rust_image_version") + container=$(copy "$output_dir" "$rust_image_id") echo "cleaning up" cleanup "$container" ;; diff --git a/etherlink/scripts/docker-compose/evm_kernel_builder.Dockerfile b/etherlink/scripts/docker-compose/evm_kernel_builder.Dockerfile index 6b52ad8485ca90052c3e2f102b4ccd28af8f1e27..b44ea0c1ec5361793e537a43e34e1b40ed171974 100644 --- a/etherlink/scripts/docker-compose/evm_kernel_builder.Dockerfile +++ b/etherlink/scripts/docker-compose/evm_kernel_builder.Dockerfile @@ -14,8 +14,8 @@ COPY kernels.mk etherlink.mk /build/ COPY src/kernel_sdk /build/src/kernel_sdk COPY etherlink /build/etherlink RUN make -f etherlink.mk build-deps -RUN make -f etherlink.mk EVM_CONFIG=${EVM_CONFIG} CI_COMMIT_SHA=${CI_COMMIT_SHA} evm_installer.wasm +RUN make --no-print-directory -f etherlink.mk EVM_CONFIG=${EVM_CONFIG} CI_COMMIT_SHA=${CI_COMMIT_SHA} DISPLAY_ROOT_HASH=true evm_installer.wasm > root_hash FROM ${BASE_IMAGE} -COPY --from=kernel_build /build/*.wasm /kernel/ +COPY --from=kernel_build /build/*.wasm /build/root_hash /kernel/ COPY --from=kernel_build /build/_evm_installer_preimages /kernel/_evm_installer_preimages diff --git a/kernels.mk b/kernels.mk index 131f66e93733ad7f70e03a06dbebdc026a32a7d4..aab344ef36641ec2d98a3b1e28bd73644ab9ba69 100644 --- a/kernels.mk +++ b/kernels.mk @@ -87,3 +87,4 @@ clean: @rm -f ${KERNELS} @make -C ${SDK_DIR} clean @make -C ${DEMO_DIR} clean + @rm -f smart-rollup-installer tx-demo-collector diff --git a/scripts/ci/create_gitlab_etherlink_release.sh b/scripts/ci/create_gitlab_etherlink_release.sh index 9453d92f5c85102fb92d15b51fed4fc04a002e9b..89ec92453a56b47b34049498dcfb24df07c7fcfc 100755 --- a/scripts/ci/create_gitlab_etherlink_release.sh +++ b/scripts/ci/create_gitlab_etherlink_release.sh @@ -3,9 +3,12 @@ set -eu ### Create a GitLab Etherlink release with links to all the related resources -# shellcheck source=./scripts/ci/octez-release.sh +# shellcheck source=./scripts/ci/etherlink-release.sh . ./scripts/ci/etherlink-release.sh +kernels_artifact_url="$CI_JOB_URL/artifacts/raw/kernels.tar.gz" + release-cli create \ --name="${gitlab_release_name}" \ - --tag-name="${CI_COMMIT_TAG}" + --tag-name="${CI_COMMIT_TAG}" \ + --assets-link="{\"name\":\"Kernels\",\"url\":\"${kernels_artifact_url}\"}" diff --git a/scripts/ci/docker_prepare_etherlink_release.sh b/scripts/ci/docker_prepare_etherlink_release.sh new file mode 100755 index 0000000000000000000000000000000000000000..c7faa7763307f000cb35e4164b71b20778a58ca5 --- /dev/null +++ b/scripts/ci/docker_prepare_etherlink_release.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -eu + +### Prepare etherlink release by building wasm and preimages with docker + +current_dir=$(cd "$(dirname "${0}")" && pwd) + +. scripts/ci/docker.env + +# shellcheck source=./scripts/ci/docker.sh +. "${current_dir}/docker.sh" + +# produce evm wasms and preimages +etherlink/scripts/build-wasm.sh etherlink/config/dev.yaml kernels + +# compress the directory +tar -czf kernels.tar.gz kernels