From 415ab056967cc63c7fb9738f77309f3ea316a5df Mon Sep 17 00:00:00 2001 From: Romain Bardou Date: Mon, 28 Nov 2022 10:44:22 +0100 Subject: [PATCH 1/4] Makefile: add OCTEZ_BIN_DIR variable --- Makefile | 51 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 48a38c0e27cd..223d8e868edb 100644 --- a/Makefile +++ b/Makefile @@ -61,6 +61,16 @@ ifneq ($(filter ${ALL_EXECUTABLES},${OCTEZ_EXECUTABLES}),$(foreach executable,${ $(error Unexpected list of executables to build, make sure environment variable OCTEZ_EXECUTABLES is unset) endif +# Where to copy executables. +# Used when building Docker images to help with the COPY instruction. +OCTEZ_BIN_DIR?=. + +VALID_OCTEZ_BIN_DIRS=. bin + +ifeq ($(filter ${VALID_OCTEZ_BIN_DIRS},${OCTEZ_BIN_DIR}),) +$(error Unexpected value for OCTEZ_BIN_DIR (got: ${OCTEZ_BIN_DIR}, expecting one of: ${VALID_OCTEZ_BIN_DIRS})) +endif + # See first mention of TEZOS_WITHOUT_OPAM. ifdef TEZOS_WITHOUT_OPAM current_ocaml_version := $(shell ocamlc -version) @@ -173,26 +183,27 @@ endif @dune build --profile=$(PROFILE) $(COVERAGE_OPTIONS) \ $(foreach b, $(OCTEZ_EXECUTABLES), _build/install/default/bin/${b}) \ @copy-parameters - @cp -f $(foreach b, $(OCTEZ_EXECUTABLES), _build/install/default/bin/${b}) ./ - @ln -s octez-node tezos-node - @ln -s octez-client tezos-client - @ln -s octez-admin-client tezos-admin-client - @ln -s octez-signer tezos-signer - @ln -s octez-codec tezos-codec - @ln -s octez-protocol-compiler tezos-protocol-compiler - @ln -s octez-proxy-server tezos-proxy-server - @ln -s octez-baker-PtKathma tezos-baker-014-PtKathma - @ln -s octez-accuser-PtKathma tezos-accuser-014-PtKathma - @ln -s octez-tx-rollup-node-PtKathma tezos-tx-rollup-node-014-PtKathma - @ln -s octez-tx-rollup-client-PtKathma tezos-tx-rollup-client-014-PtKathma - @ln -s octez-baker-PtLimaPt tezos-baker-015-PtLimaPt - @ln -s octez-accuser-PtLimaPt tezos-accuser-015-PtLimaPt - @ln -s octez-tx-rollup-node-PtLimaPt tezos-tx-rollup-node-015-PtLimaPt - @ln -s octez-tx-rollup-client-PtLimaPt tezos-tx-rollup-client-015-PtLimaPt - @ln -s octez-baker-alpha tezos-baker-alpha - @ln -s octez-accuser-alpha tezos-accuser-alpha - @ln -s octez-tx-rollup-node-alpha tezos-tx-rollup-node-alpha - @ln -s octez-tx-rollup-client-alpha tezos-tx-rollup-client-alpha + @mkdir -p $(OCTEZ_BIN_DIR)/ + @cp -f $(foreach b, $(OCTEZ_EXECUTABLES), _build/install/default/bin/${b}) $(OCTEZ_BIN_DIR)/ + @ln -s $(OCTEZ_BIN_DIR)/octez-node tezos-node + @ln -s $(OCTEZ_BIN_DIR)/octez-client tezos-client + @ln -s $(OCTEZ_BIN_DIR)/octez-admin-client tezos-admin-client + @ln -s $(OCTEZ_BIN_DIR)/octez-signer tezos-signer + @ln -s $(OCTEZ_BIN_DIR)/octez-codec tezos-codec + @ln -s $(OCTEZ_BIN_DIR)/octez-protocol-compiler tezos-protocol-compiler + @ln -s $(OCTEZ_BIN_DIR)/octez-proxy-server tezos-proxy-server + @ln -s $(OCTEZ_BIN_DIR)/octez-baker-PtKathma tezos-baker-014-PtKathma + @ln -s $(OCTEZ_BIN_DIR)/octez-accuser-PtKathma tezos-accuser-014-PtKathma + @ln -s $(OCTEZ_BIN_DIR)/octez-tx-rollup-node-PtKathma tezos-tx-rollup-node-014-PtKathma + @ln -s $(OCTEZ_BIN_DIR)/octez-tx-rollup-client-PtKathma tezos-tx-rollup-client-014-PtKathma + @ln -s $(OCTEZ_BIN_DIR)/octez-baker-PtLimaPt tezos-baker-015-PtLimaPt + @ln -s $(OCTEZ_BIN_DIR)/octez-accuser-PtLimaPt tezos-accuser-015-PtLimaPt + @ln -s $(OCTEZ_BIN_DIR)/octez-tx-rollup-node-PtLimaPt tezos-tx-rollup-node-015-PtLimaPt + @ln -s $(OCTEZ_BIN_DIR)/octez-tx-rollup-client-PtLimaPt tezos-tx-rollup-client-015-PtLimaPt + @ln -s $(OCTEZ_BIN_DIR)/octez-baker-alpha tezos-baker-alpha + @ln -s $(OCTEZ_BIN_DIR)/octez-accuser-alpha tezos-accuser-alpha + @ln -s $(OCTEZ_BIN_DIR)/octez-tx-rollup-node-alpha tezos-tx-rollup-node-alpha + @ln -s $(OCTEZ_BIN_DIR)/octez-tx-rollup-client-alpha tezos-tx-rollup-client-alpha # List protocols, i.e. directories proto_* in src with a TEZOS_PROTOCOL file. TEZOS_PROTOCOL_FILES=$(wildcard src/proto_*/lib_protocol/TEZOS_PROTOCOL) -- GitLab From bda3c66952181e8ac7a75a0fbc82b188416700eb Mon Sep 17 00:00:00 2001 From: Romain Bardou Date: Thu, 24 Nov 2022 16:25:25 +0100 Subject: [PATCH 2/4] Docker: smoke test on octez-* instead of tezos-* --- scripts/ci/docker_smoke_test.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/ci/docker_smoke_test.sh b/scripts/ci/docker_smoke_test.sh index 21f037c5d5bc..ce571e873bff 100755 --- a/scripts/ci/docker_smoke_test.sh +++ b/scripts/ci/docker_smoke_test.sh @@ -27,6 +27,10 @@ test_version() { echo "Testing version of binaries in Docker image" binaries=$(docker run --entrypoint ls "$IMAGE" /usr/local/bin/) + + echo "Found the following executables in the Docker image:" + echo "$binaries" + for bin in octez-client octez-node; do if ! echo "$binaries" | grep -q "^${bin}$" ; then echo "Could not find binary $bin in Docker image." @@ -36,7 +40,7 @@ test_version() { fi done - for binary in $(echo "$binaries" | grep tezos-); do + for binary in $(echo "$binaries" | grep octez-); do printf "Checking version of %s: " "$binary" cmd="docker run --entrypoint $binary $IMAGE --version" -- GitLab From b19c1fbc7dae51d1c6de0440c6942caf88d030ff Mon Sep 17 00:00:00 2001 From: Romain Bardou Date: Thu, 24 Nov 2022 15:05:49 +0100 Subject: [PATCH 3/4] Scripts: merge docker build scripts --- docs/introduction/howtoget.rst | 5 +- scripts/ci/create_docker_image.build.sh | 38 ------------ scripts/ci/create_docker_image.minimal.sh | 75 ----------------------- scripts/create_docker_image.sh | 63 ++++++++++++++++--- scripts/shellcheck_skips | 2 - 5 files changed, 57 insertions(+), 126 deletions(-) delete mode 100755 scripts/ci/create_docker_image.build.sh delete mode 100755 scripts/ci/create_docker_image.minimal.sh diff --git a/docs/introduction/howtoget.rst b/docs/introduction/howtoget.rst index 9a0cf87f4d44..d2a521eb25d9 100644 --- a/docs/introduction/howtoget.rst +++ b/docs/introduction/howtoget.rst @@ -179,12 +179,11 @@ Building Docker Images Locally ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The docker image used throughout the docker-compose files is fetched from upstream, but you can also -build one locally and reference it. Run the following commands to build the image: +build one locally and reference it. Run the following command to build the image: :: - ./scripts/ci/create_docker_image.build.sh - ./scripts/ci/create_docker_image.minimal.sh + ./scripts/create_docker_image.sh And then update the docker-compose file (e.g., ``alpha.yml``) with the docker tag:: diff --git a/scripts/ci/create_docker_image.build.sh b/scripts/ci/create_docker_image.build.sh deleted file mode 100755 index 6851c7db7cc6..000000000000 --- a/scripts/ci/create_docker_image.build.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/sh - -set -eu - -ci_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")" -script_dir="$(dirname "$ci_dir")" -src_dir="$(dirname "$script_dir")" -cd "$src_dir" - -. "$script_dir"/version.sh - -image_name="${1:-tezos_build}" -image_version="${2:-latest}" -base_image="${3:-registry.gitlab.com/tezos/opam-repository}" -base_image_version="${4:-runtime-build-dependencies--${opam_repository_tag}}" -commit_short_sha="${5:-$(git rev-parse --short HEAD)}" -commit_datetime="${6:-$(git show -s --pretty=format:%ci HEAD)}" -commit_tag="${7:-$(git describe --tags --always)}" - -echo -echo "### Building tezos..." -echo - -docker build \ - -t "$image_name:$image_version" \ - -f build.Dockerfile \ - --cache-from "$image_name:$image_version" \ - --build-arg "BASE_IMAGE=$base_image" \ - --build-arg "BASE_IMAGE_VERSION=${base_image_version}" \ - --build-arg "BUILD_IMAGE_VERSION=${base_image_version}" \ - --build-arg "GIT_SHORTREF=${commit_short_sha}" \ - --build-arg "GIT_DATETIME=${commit_datetime}" \ - --build-arg "GIT_VERSION=${commit_tag}" \ - "$src_dir" - -echo -echo "### Successfully built docker image: $image_name:$image_version" -echo diff --git a/scripts/ci/create_docker_image.minimal.sh b/scripts/ci/create_docker_image.minimal.sh deleted file mode 100755 index 7268e7260a82..000000000000 --- a/scripts/ci/create_docker_image.minimal.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/sh - -set -ex - -ci_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")" -script_dir="$(dirname "$ci_dir")" -src_dir="$(dirname "$script_dir")" -cd "$src_dir" - -. "$script_dir"/version.sh - -# build_image and base_image are both built by the CI for the -# repository https://gitlab.com/tezos/opam-repository -# same prefix, but different tags. The build image is identified -# by the commit reference only, while the base image is -# identified by the prefix "minimal--" followed by the same -# commit reference of the build image. - -image_name="${1:-tezos-}" -image_version="${2:-latest}" -build_image="${3:-tezos_build}" -base_image="${4:-registry.gitlab.com/tezos/opam-repository}" -base_version="${5:-runtime-dependencies--${opam_repository_tag}}" -base_build_version="${6:-runtime-build-dependencies--${opam_repository_tag}}" -commit_short_sha="${7:-$(git rev-parse --short HEAD)}" - -echo -echo "### Building minimal docker images..." -echo - -docker build \ - -t "${image_name}debug:$image_version" \ - --build-arg "BASE_IMAGE=$base_image" \ - --build-arg "BASE_IMAGE_VERSION=$base_version" \ - --build-arg "BASE_IMAGE_VERSION_NON_MIN=$base_build_version" \ - --build-arg "BUILD_IMAGE=${build_image}" \ - --build-arg "BUILD_IMAGE_VERSION=${image_version}" \ - --build-arg "COMMIT_SHORT_SHA=${commit_short_sha}" \ - --target=debug \ - "$src_dir" - -echo -echo "### Successfully built docker image: ${image_name}debug:$image_version" -echo - -docker build \ - -t "${image_name}bare:$image_version" \ - --build-arg "BASE_IMAGE=$base_image" \ - --build-arg "BASE_IMAGE_VERSION=$base_version" \ - --build-arg "BUILD_IMAGE=${build_image}" \ - --build-arg "BUILD_IMAGE_VERSION=${image_version}" \ - --build-arg "BASE_IMAGE_VERSION_NON_MIN=$base_build_version" \ - --build-arg "COMMIT_SHORT_SHA=${commit_short_sha}" \ - --target=bare \ - "$src_dir" - - -echo -echo "### Successfully built docker image: ${image_name}bare:$image_version" -echo - -docker build \ - -t "${image_name%?}:$image_version" \ - --build-arg "BASE_IMAGE=$base_image" \ - --build-arg "BASE_IMAGE_VERSION=$base_version" \ - --build-arg "BUILD_IMAGE=${build_image}" \ - --build-arg "BUILD_IMAGE_VERSION=${image_version}" \ - --build-arg "BASE_IMAGE_VERSION_NON_MIN=$base_build_version" \ - --build-arg "COMMIT_SHORT_SHA=${commit_short_sha}" \ - --target=minimal \ - "$src_dir" - -echo -echo "### Successfully built docker image: ${image_name%?}:$image_version" -echo diff --git a/scripts/create_docker_image.sh b/scripts/create_docker_image.sh index 9d4c85068189..30491c2734cd 100755 --- a/scripts/create_docker_image.sh +++ b/scripts/create_docker_image.sh @@ -13,15 +13,62 @@ image_version="${2:-latest}" build_deps_image_name=${3:-registry.gitlab.com/tezos/opam-repository} build_deps_image_version=${4:-$opam_repository_tag} commit_short_sha="${5:-$(git rev-parse --short HEAD)}" +commit_datetime="${6:-$(git show -s --pretty=format:%ci HEAD)}" +commit_tag="${7:-$(git describe --tags --always)}" build_image_name="${image_name}build" -"$script_dir"/ci/create_docker_image.build.sh \ - "$build_image_name" "$image_version" \ - "$build_deps_image_name" "runtime-build-dependencies--$build_deps_image_version" "$commit_short_sha" +echo "### Building tezos..." -"$script_dir"/ci/create_docker_image.minimal.sh \ - "$image_name" "$image_version" \ - "$build_image_name" "$build_deps_image_name" \ - "runtime-dependencies--$build_deps_image_version" "runtime-build-dependencies--$build_deps_image_version" \ - "$commit_short_sha" +docker build \ + -t "$build_image_name:$image_version" \ + -f build.Dockerfile \ + --cache-from "$build_image_name:$image_version" \ + --build-arg "BASE_IMAGE=$build_deps_image_name" \ + --build-arg "BASE_IMAGE_VERSION=runtime-build-dependencies--$build_deps_image_version" \ + --build-arg "GIT_SHORTREF=${commit_short_sha}" \ + --build-arg "GIT_DATETIME=${commit_datetime}" \ + --build-arg "GIT_VERSION=${commit_tag}" \ + "$src_dir" + +echo "### Successfully built docker image: $build_image_name:$image_version" + +docker build \ + -t "${image_name}debug:$image_version" \ + --build-arg "BASE_IMAGE=$build_deps_image_name" \ + --build-arg "BASE_IMAGE_VERSION=runtime-dependencies--$build_deps_image_version" \ + --build-arg "BASE_IMAGE_VERSION_NON_MIN=runtime-build-dependencies--$build_deps_image_version" \ + --build-arg "BUILD_IMAGE=${build_image_name}" \ + --build-arg "BUILD_IMAGE_VERSION=${image_version}" \ + --build-arg "COMMIT_SHORT_SHA=${commit_short_sha}" \ + --target=debug \ + "$src_dir" + +echo "### Successfully built docker image: ${image_name}debug:$image_version" + +docker build \ + -t "${image_name}bare:$image_version" \ + --build-arg "BASE_IMAGE=$build_deps_image_name" \ + --build-arg "BASE_IMAGE_VERSION=runtime-dependencies--$build_deps_image_version" \ + --build-arg "BASE_IMAGE_VERSION_NON_MIN=runtime-build-dependencies--$build_deps_image_version" \ + --build-arg "BUILD_IMAGE=${build_image_name}" \ + --build-arg "BUILD_IMAGE_VERSION=${image_version}" \ + --build-arg "COMMIT_SHORT_SHA=${commit_short_sha}" \ + --target=bare \ + "$src_dir" + + +echo "### Successfully built docker image: ${image_name}bare:$image_version" + +docker build \ + -t "${image_name%?}:$image_version" \ + --build-arg "BASE_IMAGE=$build_deps_image_name" \ + --build-arg "BASE_IMAGE_VERSION=runtime-dependencies--$build_deps_image_version" \ + --build-arg "BASE_IMAGE_VERSION_NON_MIN=runtime-build-dependencies--$build_deps_image_version" \ + --build-arg "BUILD_IMAGE=${build_image_name}" \ + --build-arg "BUILD_IMAGE_VERSION=${image_version}" \ + --build-arg "COMMIT_SHORT_SHA=${commit_short_sha}" \ + --target=minimal \ + "$src_dir" + +echo "### Successfully built docker image: ${image_name%?}:$image_version" diff --git a/scripts/shellcheck_skips b/scripts/shellcheck_skips index cb4c5c850403..edb640cdd7ab 100644 --- a/scripts/shellcheck_skips +++ b/scripts/shellcheck_skips @@ -1,6 +1,4 @@ scripts/baker-version-stats.sh -scripts/ci/create_docker_image.build.sh -scripts/ci/create_docker_image.minimal.sh scripts/ci/docker_registry_delete.sh scripts/ci/docker_registry.inc.sh scripts/ci/docker_registry_read.sh -- GitLab From 8c055eee71b82397cd3df1324fb687843c88bf51 Mon Sep 17 00:00:00 2001 From: Romain Bardou Date: Mon, 28 Nov 2022 14:21:31 +0100 Subject: [PATCH 4/4] CI: EXECUTABLE_FILES for Docker --- .../ci/jobs/build/docker_amd64_experimental.yml | 10 ++++++++++ .gitlab/ci/jobs/build/docker_amd64_release.yml | 1 + .gitlab/ci/jobs/build/docker_amd64_test.yml | 1 + .../ci/jobs/build/docker_amd64_test_manual.yml | 1 + .../ci/jobs/build/docker_arm64_experimental.yml | 12 ++++++++++++ .gitlab/ci/jobs/build/docker_arm64_release.yml | 1 + .gitlab/ci/jobs/build/docker_arm64_test.yml | 1 + .../ci/jobs/build/docker_arm64_test_manual.yml | 1 + .gitlab/ci/pipelines/master_branch.yml | 4 ++-- Dockerfile | 11 +---------- build.Dockerfile | 6 ++++-- scripts/ci/docker_release.sh | 15 ++++++++++++++- scripts/create_docker_image.sh | 13 ++++++++++--- 13 files changed, 59 insertions(+), 18 deletions(-) create mode 100644 .gitlab/ci/jobs/build/docker_amd64_experimental.yml create mode 100644 .gitlab/ci/jobs/build/docker_arm64_experimental.yml diff --git a/.gitlab/ci/jobs/build/docker_amd64_experimental.yml b/.gitlab/ci/jobs/build/docker_amd64_experimental.yml new file mode 100644 index 000000000000..0031bd2ac723 --- /dev/null +++ b/.gitlab/ci/jobs/build/docker_amd64_experimental.yml @@ -0,0 +1,10 @@ +include: .gitlab/ci/jobs/build/common.yml + +docker:amd64: + extends: + - .build_docker_release_template + variables: + IMAGE_ARCH_PREFIX: "amd64_" + CI_DOCKER_HUB: "true" + EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" + stage: build diff --git a/.gitlab/ci/jobs/build/docker_amd64_release.yml b/.gitlab/ci/jobs/build/docker_amd64_release.yml index dc669c5cec3e..7f7544a5fe90 100644 --- a/.gitlab/ci/jobs/build/docker_amd64_release.yml +++ b/.gitlab/ci/jobs/build/docker_amd64_release.yml @@ -6,4 +6,5 @@ docker:amd64: variables: IMAGE_ARCH_PREFIX: "amd64_" CI_DOCKER_HUB: "true" + EXECUTABLE_FILES: "script-inputs/released-executables" stage: build diff --git a/.gitlab/ci/jobs/build/docker_amd64_test.yml b/.gitlab/ci/jobs/build/docker_amd64_test.yml index c13d9ed96032..590ee301acb4 100644 --- a/.gitlab/ci/jobs/build/docker_amd64_test.yml +++ b/.gitlab/ci/jobs/build/docker_amd64_test.yml @@ -6,4 +6,5 @@ docker:amd64: variables: IMAGE_ARCH_PREFIX: "amd64_" CI_DOCKER_HUB: "false" + EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" stage: build \ No newline at end of file diff --git a/.gitlab/ci/jobs/build/docker_amd64_test_manual.yml b/.gitlab/ci/jobs/build/docker_amd64_test_manual.yml index 659eec9e6439..522b884ce439 100644 --- a/.gitlab/ci/jobs/build/docker_amd64_test_manual.yml +++ b/.gitlab/ci/jobs/build/docker_amd64_test_manual.yml @@ -9,4 +9,5 @@ docker:amd64: variables: IMAGE_ARCH_PREFIX: "amd64_" CI_DOCKER_HUB: "false" + EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" allow_failure: true diff --git a/.gitlab/ci/jobs/build/docker_arm64_experimental.yml b/.gitlab/ci/jobs/build/docker_arm64_experimental.yml new file mode 100644 index 000000000000..952107cfe355 --- /dev/null +++ b/.gitlab/ci/jobs/build/docker_arm64_experimental.yml @@ -0,0 +1,12 @@ +include: .gitlab/ci/jobs/build/common.yml + +docker:arm64: + extends: + - .build_docker_release_template + stage: build + variables: + IMAGE_ARCH_PREFIX: "arm64_" + CI_DOCKER_HUB: "true" + EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" + tags: + - arm64 \ No newline at end of file diff --git a/.gitlab/ci/jobs/build/docker_arm64_release.yml b/.gitlab/ci/jobs/build/docker_arm64_release.yml index 7a011674930b..d154e617810b 100644 --- a/.gitlab/ci/jobs/build/docker_arm64_release.yml +++ b/.gitlab/ci/jobs/build/docker_arm64_release.yml @@ -7,5 +7,6 @@ docker:arm64: variables: IMAGE_ARCH_PREFIX: "arm64_" CI_DOCKER_HUB: "true" + EXECUTABLE_FILES: "script-inputs/released-executables" tags: - arm64 \ No newline at end of file diff --git a/.gitlab/ci/jobs/build/docker_arm64_test.yml b/.gitlab/ci/jobs/build/docker_arm64_test.yml index 6e28da20c842..0e565dcec760 100644 --- a/.gitlab/ci/jobs/build/docker_arm64_test.yml +++ b/.gitlab/ci/jobs/build/docker_arm64_test.yml @@ -7,5 +7,6 @@ docker:arm64: variables: IMAGE_ARCH_PREFIX: "arm64_" CI_DOCKER_HUB: "false" + EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" tags: - arm64 \ No newline at end of file diff --git a/.gitlab/ci/jobs/build/docker_arm64_test_manual.yml b/.gitlab/ci/jobs/build/docker_arm64_test_manual.yml index 5ebc0af0233f..08506ee99740 100644 --- a/.gitlab/ci/jobs/build/docker_arm64_test_manual.yml +++ b/.gitlab/ci/jobs/build/docker_arm64_test_manual.yml @@ -9,6 +9,7 @@ docker:arm64: variables: IMAGE_ARCH_PREFIX: "arm64_" CI_DOCKER_HUB: "false" + EXECUTABLE_FILES: "script-inputs/released-executables script-inputs/experimental-executables" allow_failure: true tags: - arm64 diff --git a/.gitlab/ci/pipelines/master_branch.yml b/.gitlab/ci/pipelines/master_branch.yml index 83cd91076419..3141eedc8536 100644 --- a/.gitlab/ci/pipelines/master_branch.yml +++ b/.gitlab/ci/pipelines/master_branch.yml @@ -3,8 +3,8 @@ include: - .gitlab/ci/jobs/build/static_x86_64_experimental.yml - .gitlab/ci/jobs/build/static_arm64_experimental.yml - .gitlab/ci/jobs/build/arm64.yml - - .gitlab/ci/jobs/build/docker_amd64_release.yml - - .gitlab/ci/jobs/build/docker_arm64_release.yml + - .gitlab/ci/jobs/build/docker_amd64_experimental.yml + - .gitlab/ci/jobs/build/docker_arm64_experimental.yml # Stage: test - .gitlab/ci/jobs/test/ometrics_code_quality_default.yml diff --git a/Dockerfile b/Dockerfile index ab2a03a2dff9..62eabbb95c53 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,16 +9,7 @@ FROM ${BUILD_IMAGE}:${BUILD_IMAGE_VERSION} as builder FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION} as intermediate # Pull in built binaries -COPY --chown=tezos:nogroup --from=builder /home/tezos/tezos/octez-baker-* /home/tezos/bin/ -COPY --chown=tezos:nogroup --from=builder /home/tezos/tezos/octez-accuser-* /home/tezos/bin/ -COPY --chown=tezos:nogroup --from=builder /home/tezos/tezos/octez-client /home/tezos/bin/ -COPY --chown=tezos:nogroup --from=builder /home/tezos/tezos/octez-admin-client /home/tezos/bin/ -COPY --chown=tezos:nogroup --from=builder /home/tezos/tezos/octez-node /home/tezos/bin/ -COPY --chown=tezos:nogroup --from=builder /home/tezos/tezos/octez-proxy-server /home/tezos/bin/ -COPY --chown=tezos:nogroup --from=builder /home/tezos/tezos/octez-signer /home/tezos/bin/ -COPY --chown=tezos:nogroup --from=builder /home/tezos/tezos/octez-tx-rollup-* /home/tezos/bin/ -COPY --chown=tezos:nogroup --from=builder /home/tezos/tezos/octez-sc-rollup-* /home/tezos/bin/ -COPY --chown=tezos:nogroup --from=builder /home/tezos/tezos/octez-codec /home/tezos/bin/ +COPY --chown=tezos:nogroup --from=builder /home/tezos/tezos/bin/* /home/tezos/bin/ # Add parameters for active protocols COPY --chown=tezos:nogroup --from=builder /home/tezos/tezos/parameters /home/tezos/scripts/ # Add entrypoint scripts diff --git a/build.Dockerfile b/build.Dockerfile index 8d975295dded..c93c3fda88c7 100644 --- a/build.Dockerfile +++ b/build.Dockerfile @@ -4,7 +4,9 @@ FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION} # use alpine /bin/ash and set pipefail. # see https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run SHELL ["/bin/ash", "-o", "pipefail", "-c"] -# do not move the ARG below above the FROM or it gets erased +# Do not move the ARG below above the FROM or it gets erased. +# More precisely: ARG above FROM can be used in the FROM itself, but nowhere else. +ARG EXECUTABLES ARG GIT_SHORTREF ARG GIT_DATETIME ARG GIT_VERSION @@ -27,7 +29,7 @@ COPY --chown=tezos:nogroup vendors tezos/vendors ENV GIT_SHORTREF=${GIT_SHORTREF} ENV GIT_DATETIME=${GIT_DATETIME} ENV GIT_VERSION=${GIT_VERSION} -RUN opam exec -- make -C tezos experimental-release +RUN opam exec -- make -C tezos release EXECUTABLES="${EXECUTABLES}" OCTEZ_BIN_DIR=bin # Gather the parameters of all active protocols in 1 place RUN while read -r protocol; do \ mkdir -p tezos/parameters/"$protocol"-parameters && \ diff --git a/scripts/ci/docker_release.sh b/scripts/ci/docker_release.sh index b8857532b992..3093d7ec6029 100755 --- a/scripts/ci/docker_release.sh +++ b/scripts/ci/docker_release.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -e + . scripts/ci/docker.env if [ -z "${build_deps_image_name}" ]; then echo "build_deps_image_name is unset" && exit 3; fi @@ -10,12 +12,23 @@ cd "${CI_PROJECT_DIR}" || exit 1 # Environment variables from before_script . ./scripts/ci/docker.env +if [ -z "$EXECUTABLE_FILES" ]; then + echo "Error: environment variable EXECUTABLE_FILES is empty." + echo "Set it to e.g. 'script-inputs/released-executables'" + echo "or to 'script-inputs/released-executables script-inputs/experimental-executables'." + exit 1 +fi + +# shellcheck disable=SC2086 +EXECUTABLES="$(cat $EXECUTABLE_FILES)" + # Build minimal, bare and debug images ./scripts/create_docker_image.sh \ "${DOCKER_IMAGE_NAME}" \ "${DOCKER_IMAGE_TAG}" \ "${build_deps_image_name}" \ "${build_deps_image_version}" \ + "${EXECUTABLES}" \ "${CI_COMMIT_SHORT_SHA}" # auth gitlab or dockerhub registry @@ -24,4 +37,4 @@ cd "${CI_PROJECT_DIR}" || exit 1 ./scripts/ci/docker_smoke_test.sh "${DOCKER_IMAGE_NAME}bare:${DOCKER_IMAGE_TAG}" "${CI_COMMIT_SHORT_SHA}" version # Push minimal, bare and debug images -./scripts/ci/docker_push_all.sh \ No newline at end of file +./scripts/ci/docker_push_all.sh diff --git a/scripts/create_docker_image.sh b/scripts/create_docker_image.sh index 30491c2734cd..ae827b9e8505 100755 --- a/scripts/create_docker_image.sh +++ b/scripts/create_docker_image.sh @@ -12,12 +12,18 @@ image_name="${1:-tezos-}" image_version="${2:-latest}" build_deps_image_name=${3:-registry.gitlab.com/tezos/opam-repository} build_deps_image_version=${4:-$opam_repository_tag} -commit_short_sha="${5:-$(git rev-parse --short HEAD)}" -commit_datetime="${6:-$(git show -s --pretty=format:%ci HEAD)}" -commit_tag="${7:-$(git describe --tags --always)}" +executables=${5:-$(cat script-inputs/released-executables)} +commit_short_sha="${6:-$(git rev-parse --short HEAD)}" +commit_datetime="${7:-$(git show -s --pretty=format:%ci HEAD)}" +commit_tag="${8:-$(git describe --tags --always)}" build_image_name="${image_name}build" +echo "Executables to include in Docker images:" +for executable in $executables; do + echo "- $executable" +done + echo "### Building tezos..." docker build \ @@ -26,6 +32,7 @@ docker build \ --cache-from "$build_image_name:$image_version" \ --build-arg "BASE_IMAGE=$build_deps_image_name" \ --build-arg "BASE_IMAGE_VERSION=runtime-build-dependencies--$build_deps_image_version" \ + --build-arg "EXECUTABLES=${executables}" \ --build-arg "GIT_SHORTREF=${commit_short_sha}" \ --build-arg "GIT_DATETIME=${commit_datetime}" \ --build-arg "GIT_VERSION=${commit_tag}" \ -- GitLab