From b6b8e3f4873996f321fc2597650cbe3d9810c2cf Mon Sep 17 00:00:00 2001 From: Thomas Letan Date: Wed, 24 May 2023 15:40:11 +0200 Subject: [PATCH] CI: Include the EVM kernel in the Docker images of master --- .gitlab/ci/jobs/build/common.yml | 2 ++ .../jobs/build/docker_amd64_experimental.yml | 1 + .../jobs/build/docker_amd64_test_manual.yml | 1 + Dockerfile | 2 +- build.Dockerfile | 19 +++++++++++++++++-- scripts/ci/docker_registry_auth.sh | 2 ++ scripts/ci/docker_release.sh | 4 +++- scripts/create_docker_image.sh | 10 ++++++++-- scripts/version.sh | 4 ++++ 9 files changed, 39 insertions(+), 6 deletions(-) diff --git a/.gitlab/ci/jobs/build/common.yml b/.gitlab/ci/jobs/build/common.yml index 940bdef037e1..7332e287afae 100644 --- a/.gitlab/ci/jobs/build/common.yml +++ b/.gitlab/ci/jobs/build/common.yml @@ -45,6 +45,8 @@ - .default_settings_template - .image_template__docker variables: + DOCKER_BUILD_TARGET: "without-evm-artifacts" + RUST_TOOLCHAIN_IMAGE: "${PROD_AWS_ECR}/rust-toolchain" IMAGE_ARCH_PREFIX: "" before_script: - ./scripts/ci/docker_registry_auth.sh diff --git a/.gitlab/ci/jobs/build/docker_amd64_experimental.yml b/.gitlab/ci/jobs/build/docker_amd64_experimental.yml index 0031bd2ac723..52fd82b428cd 100644 --- a/.gitlab/ci/jobs/build/docker_amd64_experimental.yml +++ b/.gitlab/ci/jobs/build/docker_amd64_experimental.yml @@ -4,6 +4,7 @@ docker:amd64: extends: - .build_docker_release_template variables: + DOCKER_BUILD_TARGET: "with-evm-artifacts" IMAGE_ARCH_PREFIX: "amd64_" CI_DOCKER_HUB: "true" EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" diff --git a/.gitlab/ci/jobs/build/docker_amd64_test_manual.yml b/.gitlab/ci/jobs/build/docker_amd64_test_manual.yml index 522b884ce439..4d844c0230db 100644 --- a/.gitlab/ci/jobs/build/docker_amd64_test_manual.yml +++ b/.gitlab/ci/jobs/build/docker_amd64_test_manual.yml @@ -7,6 +7,7 @@ docker:amd64: needs: [] when: manual variables: + DOCKER_BUILD_TARGET: "with-evm-artifacts" IMAGE_ARCH_PREFIX: "amd64_" CI_DOCKER_HUB: "false" EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" diff --git a/Dockerfile b/Dockerfile index 62eabbb95c53..c64f1952272c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION} as intermediate # Pull in built binaries COPY --chown=tezos:nogroup --from=builder /home/tezos/tezos/bin/* /home/tezos/bin/ # Add parameters for active protocols -COPY --chown=tezos:nogroup --from=builder /home/tezos/tezos/parameters /home/tezos/scripts/ +COPY --chown=tezos:nogroup --from=builder /home/tezos/evm_kernel/evm_kernel.wasm* /home/tezos/tezos/parameters /home/tezos/scripts/ # Add entrypoint scripts COPY --chown=tezos:nogroup scripts/docker/entrypoint.* /home/tezos/bin/ # Add scripts diff --git a/build.Dockerfile b/build.Dockerfile index f434c62d31b7..aa96b6fb9843 100644 --- a/build.Dockerfile +++ b/build.Dockerfile @@ -1,6 +1,9 @@ ARG BASE_IMAGE=registry.gitlab.com/tezos/opam-repository ARG BASE_IMAGE_VERSION -FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION} +ARG RUST_TOOLCHAIN_IMAGE +ARG RUST_TOOLCHAIN_IMAGE_VERSION + +FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION} as without-evm-artifacts # use alpine /bin/ash and set pipefail. # see https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run SHELL ["/bin/ash", "-o", "pipefail", "-c"] @@ -11,7 +14,7 @@ ARG GIT_SHORTREF ARG GIT_DATETIME ARG GIT_VERSION WORKDIR /home/tezos -RUN mkdir -p /home/tezos/tezos/scripts /home/tezos/tezos/script-inputs /home/tezos/tezos/parameters +RUN mkdir -p /home/tezos/tezos/scripts /home/tezos/tezos/script-inputs /home/tezos/tezos/parameters /home/tezos/evm_kernel COPY --chown=tezos:nogroup Makefile tezos COPY --chown=tezos:nogroup script-inputs/active_protocol_versions tezos/script-inputs/ COPY --chown=tezos:nogroup script-inputs/active_protocol_versions_without_number tezos/script-inputs/ @@ -35,3 +38,15 @@ RUN while read -r protocol; do \ mkdir -p tezos/parameters/"$protocol"-parameters && \ cp tezos/src/proto_"$(echo "$protocol" | tr - _)"/parameters/*.json tezos/parameters/"$protocol"-parameters; \ done < tezos/script-inputs/active_protocol_versions + +FROM ${RUST_TOOLCHAIN_IMAGE}:${RUST_TOOLCHAIN_IMAGE_VERSION} AS layer2-builder +WORKDIR /home/tezos/ +RUN mkdir -p /home/tezos/evm_kernel +COPY --chown=tezos:nogroup kernels.mk evm_kernel +COPY --chown=tezos:nogroup src evm_kernel/src +RUN make -C evm_kernel/src/kernel_evm build-deps && make -C evm_kernel -f kernels.mk evm_kernel.wasm + +# We move the EVM kernel in the final image in a dedicated stage to parallelize +# the two builder stages. +FROM without-evm-artifacts as with-evm-artifacts +COPY --from=layer2-builder --chown=tezos:nogroup /home/tezos/evm_kernel/evm_kernel.wasm evm_kernel diff --git a/scripts/ci/docker_registry_auth.sh b/scripts/ci/docker_registry_auth.sh index 54821d65b1bb..4c29ae8dfdab 100755 --- a/scripts/ci/docker_registry_auth.sh +++ b/scripts/ci/docker_registry_auth.sh @@ -27,6 +27,8 @@ echo "CI_DOCKER_HUB=${CI_DOCKER_HUB:-}" echo "CI_PROJECT_NAME=${CI_PROJECT_NAME}" echo "CI_PROJECT_NAMESPACE=${CI_PROJECT_NAMESPACE}" echo "IMAGE_ARCH_PREFIX=${IMAGE_ARCH_PREFIX:-}" +echo "DOCKER_BUILD_TARGET=${DOCKER_BUILD_TARGET}" +echo "RUST_TOOLCHAIN_IMAGE=${RUST_TOOLCHAIN_IMAGE}" # CI_DOCKER_HUB is used to switch to Docker Hub if credentials are available with CI_DOCKER_AUTH # /!\ CI_DOCKER_HUB can be unset, CI_DOCKER_AUTH is only available on protected branches diff --git a/scripts/ci/docker_release.sh b/scripts/ci/docker_release.sh index 1a801f160381..6951cb655c2e 100755 --- a/scripts/ci/docker_release.sh +++ b/scripts/ci/docker_release.sh @@ -29,7 +29,9 @@ OCTEZ_EXECUTABLES="$(cat $EXECUTABLE_FILES)" "${build_deps_image_name}" \ "${build_deps_image_version}" \ "${OCTEZ_EXECUTABLES}" \ - "${CI_COMMIT_SHORT_SHA}" + "${CI_COMMIT_SHORT_SHA}" \ + "${DOCKER_BUILD_TARGET}" \ + "${RUST_TOOLCHAIN_IMAGE}" # auth gitlab or dockerhub registry # notice the different namespace for gitlab and that we remove the `-` diff --git a/scripts/create_docker_image.sh b/scripts/create_docker_image.sh index e2fc0e5f1432..69b11e98cd9d 100755 --- a/scripts/create_docker_image.sh +++ b/scripts/create_docker_image.sh @@ -14,8 +14,11 @@ build_deps_image_name=${3:-registry.gitlab.com/tezos/opam-repository} build_deps_image_version=${4:-$opam_repository_tag} executables=${5:-$(cat script-inputs/released-executables)} commit_short_sha="${6:-$(git rev-parse --short HEAD)}" -commit_datetime="${7:-$(git show -s --pretty=format:%ci HEAD)}" -commit_tag="${8:-$(git describe --tags --always)}" +docker_target="${7:-without-evm-artifacts}" +rust_toolchain_image="$8" +rust_toolchain_image_version="${9:-$rust_toolchain_image_version}" +commit_datetime="${10:-$(git show -s --pretty=format:%ci HEAD)}" +commit_tag="${11:-$(git describe --tags --always)}" build_image_name="${image_name}build" @@ -29,6 +32,7 @@ echo "### Building tezos..." docker build \ -t "$build_image_name:$image_version" \ -f build.Dockerfile \ + --target "$docker_target" \ --cache-from "$build_image_name:$image_version" \ --build-arg "BASE_IMAGE=$build_deps_image_name" \ --build-arg "BASE_IMAGE_VERSION=runtime-build-dependencies--$build_deps_image_version" \ @@ -36,6 +40,8 @@ docker build \ --build-arg "GIT_SHORTREF=${commit_short_sha}" \ --build-arg "GIT_DATETIME=${commit_datetime}" \ --build-arg "GIT_VERSION=${commit_tag}" \ + --build-arg "RUST_TOOLCHAIN_IMAGE=$rust_toolchain_image" \ + --build-arg "RUST_TOOLCHAIN_IMAGE_VERSION=$rust_toolchain_image_version" \ "$src_dir" echo "### Successfully built docker image: $build_image_name:$image_version" diff --git a/scripts/version.sh b/scripts/version.sh index effd441948c8..011b60ae1cce 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -34,3 +34,7 @@ export COVERAGE_OUTPUT=_coverage_output export sapling_output_parameters_sha256=2f0ebbcbb9bb0bcffe95a397e7eba89c29eb4dde6191c339db88570e3f3fb0e4 export sapling_spend_parameters_sha256=8e48ffd23abb3a5fd9c5589204f32d9c31285a04b78096ba40a79b75677efc13 + +## The version of the rust-specific image which can be used to build Rust +## artifacts of this repository. +export rust_toolchain_image_version="v1.0.3" -- GitLab