From 60d20ec91f866bbd07b129b195fab69c162e8a82 Mon Sep 17 00:00:00 2001 From: Romain Bardou Date: Wed, 23 Nov 2022 17:04:39 +0100 Subject: [PATCH 1/2] Makefile: clean up set of executables to build --- Makefile | 84 ++++++++++++++++------------- build.Dockerfile | 5 +- docs/alpha/smart_rollups.rst | 5 +- scripts/ci/build_full_unreleased.sh | 2 +- scripts/ci/build_static_binaries.sh | 3 +- 5 files changed, 56 insertions(+), 43 deletions(-) diff --git a/Makefile b/Makefile index 4f46a14764c3..b4fd4b3eb587 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,6 @@ PACKAGES_SUBPROJECT:=$(patsubst %.opam,%,$(notdir $(shell find src vendors -name \*.opam -print))) PACKAGES:=$(patsubst %.opam,%,$(notdir $(shell find opam -name \*.opam -print))) -active_protocol_versions_without_number := $(shell cat script-inputs/active_protocol_versions_without_number) -sc_rollup_protocol_versions_without_number := $(shell cat script-inputs/sc_rollup_protocol_versions_without_number) - define directory_of_version src/proto_$(shell echo $1 | tr -- - _) endef @@ -33,16 +30,17 @@ CODE_QUALITY_REPORT := _reports/gl-code-quality-report.json PROFILE?=dev VALID_PROFILES=dev release static -OCTEZ_BIN=octez-node octez-client octez-admin-client octez-signer octez-codec \ - octez-protocol-compiler octez-snoop octez-proxy-server \ - $(foreach p, $(active_protocol_versions_without_number), octez-baker-$(p)) \ - $(foreach p, $(active_protocol_versions_without_number), octez-accuser-$(p)) \ - $(foreach p, $(active_protocol_versions_without_number), octez-tx-rollup-node-$(p)) \ - $(foreach p, $(active_protocol_versions_without_number), octez-tx-rollup-client-$(p)) \ - $(foreach p, $(sc_rollup_protocol_versions_without_number), octez-sc-rollup-node-$(p)) \ - $(foreach p, $(sc_rollup_protocol_versions_without_number), octez-sc-rollup-client-$(p)) +# See the documentation of [~release_status] in [manifest/manifest.mli]. +RELEASED_EXECUTABLES := $(shell cat script-inputs/released-executables) +EXPERIMENTAL_EXECUTABLES := $(shell cat script-inputs/experimental-executables) + +# Executables that developers need at the root of the repository but that +# are not useful for users. +# - scripts/snapshot_alpha.sh expects octez-protocol-compiler to be at the root. +# - Some tests expect octez-snoop to be at the root. +DEV_EXECUTABLES := octez-protocol-compiler octez-snoop -UNRELEASED_OCTEZ_BIN=octez-dal-node octez-wasm-repl-alpha +ALL_EXECUTABLES := $(RELEASED_EXECUTABLES) $(EXPERIMENTAL_EXECUTABLES) $(DEV_EXECUTABLES) # See first mention of TEZOS_WITHOUT_OPAM. ifndef TEZOS_WITHOUT_OPAM @@ -55,6 +53,14 @@ ifeq ($(filter ${VALID_PROFILES},${PROFILE}),) $(error Unexpected dune profile (got: ${PROFILE}, expecting one of: ${VALID_PROFILES})) endif +# This check ensures that the `OCTEZ_EXECUTABLES` variable contains a subset of +# the `ALL_EXECUTABLE` variable. The `OCTEZ_EXECUTABLES` variable is used +# internally to select a subset of which executables to build. +# The reason for the `foreach` is so that we support both newlines and spaces. +ifneq ($(filter ${ALL_EXECUTABLES},${OCTEZ_EXECUTABLES}),$(foreach executable,${OCTEZ_EXECUTABLES},${executable})) +$(error Unexpected list of executables to build, make sure environment variable OCTEZ_EXECUTABLES is unset) +endif + # See first mention of TEZOS_WITHOUT_OPAM. ifdef TEZOS_WITHOUT_OPAM current_ocaml_version := $(shell ocamlc -version) @@ -62,36 +68,46 @@ else current_ocaml_version := $(shell opam exec -- ocamlc -version) endif +# Default target. +# +# Note that you can override the list of executables to build on the command-line, e.g.: +# +# make OCTEZ_EXECUTABLES='octez-node octez-client' +# +# This is more efficient than 'make octez-node octez-client' +# because it only calls 'dune' once. +# +# Targets 'all', 'release', 'experimental-release' and 'static' define different +# default lists of executables to build but they all can be overridden from the command-line. .PHONY: all all: - @$(MAKE) build + @$(MAKE) build OCTEZ_EXECUTABLES?="$(ALL_EXECUTABLES)" .PHONY: release release: - @$(MAKE) build PROFILE=release + @$(MAKE) build PROFILE=release OCTEZ_EXECUTABLES?="$(RELEASED_EXECUTABLES)" + +.PHONY: experimental-release +experimental-release: + @$(MAKE) build PROFILE=release OCTEZ_EXECUTABLES?="$(RELEASED_EXECUTABLES) $(EXPERIMENTAL_EXECUTABLES)" .PHONY: static static: - @$(MAKE) build build-unreleased PROFILE=static + @$(MAKE) build PROFILE=static OCTEZ_EXECUTABLES?="$(RELEASED_EXECUTABLES)" .PHONY: build-parameters build-parameters: @dune build --profile=$(PROFILE) $(COVERAGE_OPTIONS) @copy-parameters -.PHONY: $(OCTEZ_BIN) -$(OCTEZ_BIN): +.PHONY: $(ALL_EXECUTABLES) +$(ALL_EXECUTABLES): dune build $(COVERAGE_OPTIONS) --profile=$(PROFILE) _build/install/default/bin/$@ cp -f _build/install/default/bin/$@ ./ -.PHONY: $(UNRELEASED_OCTEZ_BIN) -$(UNRELEASED_OCTEZ_BIN): - @dune build $(COVERAGE_OPTIONS) --profile=$(PROFILE) _build/install/default/bin/$@ - @cp -f _build/install/default/bin/$@ ./ - # Remove the old names of executables. # Depending on the commit you are updating from (v14.0, v15 or some version of master), # the exact list can vary. We just remove all of them. -# Don't try to generate this list from OCTEZ_BIN: this list should not evolve as +# Don't try to generate this list from *_EXECUTABLES: this list should not evolve as # we add new executables, and this list should contain executables that were built # before (e.g. old protocol daemons) but that are no longer built. .PHONY: clean-old-names @@ -145,16 +161,19 @@ clean-old-names: @rm -f octez-tx-rollup-client-015-PtLimaPt # See comment of clean-old-names for an explanation regarding why we do not try -# to generate the symbolic links from OCTEZ_BIN. +# to generate the symbolic links from *_EXECUTABLES. .PHONY: build build: clean-old-names ifneq (${current_ocaml_version},${ocaml_version}) $(error Unexpected ocaml version (found: ${current_ocaml_version}, expected: ${ocaml_version})) +endif +ifeq (${OCTEZ_EXECUTABLES},) + $(error The build target requires OCTEZ_EXECUTABLES to be specified. Please use another target (e.g. 'make' or 'make release') and make sure that environment variable OCTEZ_EXECUTABLES is unset) endif @dune build --profile=$(PROFILE) $(COVERAGE_OPTIONS) \ - $(foreach b, $(OCTEZ_BIN), _build/install/default/bin/${b}) \ + $(foreach b, $(OCTEZ_EXECUTABLES), _build/install/default/bin/${b}) \ @copy-parameters - @cp -f $(foreach b, $(OCTEZ_BIN), _build/install/default/bin/${b}) ./ + @cp -f $(foreach b, $(OCTEZ_EXECUTABLES), _build/install/default/bin/${b}) ./ @ln -s octez-node tezos-node @ln -s octez-client tezos-client @ln -s octez-admin-client tezos-admin-client @@ -405,18 +424,9 @@ build-tps: lift-protocol-limits-patch build build-tezt @cp -f ./src/bin_tps_evaluation/tezos-tps-evaluation-estimate-average-block . @cp -f ./src/bin_tps_evaluation/tezos-tps-evaluation-gas-tps . -# Note: this target is an extended copy-paste of the target 'build' -# and must be kept in sync with it, so that 'build-unreleased' builds -# a superset of 'build'. .PHONY: build-unreleased -build-unreleased: -ifneq (${current_ocaml_version},${ocaml_version}) - $(error Unexpected ocaml version (found: ${current_ocaml_version}, expected: ${ocaml_version})) -endif - @dune build --profile=$(PROFILE) $(COVERAGE_OPTIONS) \ - $(foreach b, $(OCTEZ_BIN) $(UNRELEASED_OCTEZ_BIN), _build/install/default/bin/${b}) \ - @copy-parameters - @cp -f $(foreach b, $(OCTEZ_BIN) $(UNRELEASED_OCTEZ_BIN), _build/install/default/bin/${b}) ./ +build-unreleased: all + @echo 'Note: "make build-unreleased" is deprecated. Just use "make".' .PHONY: docker-image-build docker-image-build: diff --git a/build.Dockerfile b/build.Dockerfile index 6ff4a2159a9e..99402471e095 100644 --- a/build.Dockerfile +++ b/build.Dockerfile @@ -14,9 +14,12 @@ COPY --chown=tezos:nogroup Makefile tezos COPY --chown=tezos:nogroup script-inputs/active_protocol_versions tezos/script-inputs/ COPY --chown=tezos:nogroup script-inputs/active_protocol_versions_without_number tezos/script-inputs/ COPY --chown=tezos:nogroup script-inputs/sc_rollup_protocol_versions_without_number tezos/script-inputs/ +COPY --chown=tezos:nogroup script-inputs/released-executables tezos/script-inputs/ +COPY --chown=tezos:nogroup script-inputs/experimental-executables tezos/script-inputs/ COPY --chown=tezos:nogroup dune tezos COPY --chown=tezos:nogroup scripts/version.sh tezos/scripts/ COPY --chown=tezos:nogroup src tezos/src +COPY --chown=tezos:nogroup tezt tezos/tezt COPY --chown=tezos:nogroup opam tezos/opam COPY --chown=tezos:nogroup dune tezos/dune COPY --chown=tezos:nogroup dune-workspace tezos/dune-workspace @@ -25,7 +28,7 @@ COPY --chown=tezos:nogroup vendors tezos/vendors ENV GIT_SHORTREF=${GIT_SHORTREF} ENV GIT_DATETIME=${GIT_DATETIME} ENV GIT_VERSION=${GIT_VERSION} -RUN opam exec -- make -C tezos release +RUN opam exec -- make -C tezos experimental-release # Gather the parameters of all active protocols in 1 place RUN while read -r protocol; do \ mkdir -p tezos/parameters/"$protocol"-parameters && \ diff --git a/docs/alpha/smart_rollups.rst b/docs/alpha/smart_rollups.rst index 86bb19c7a08e..8672a1a6faa0 100644 --- a/docs/alpha/smart_rollups.rst +++ b/docs/alpha/smart_rollups.rst @@ -1020,9 +1020,8 @@ Testing your Kernel in `the Octez repository `_. To get ``octez-wasm-repl``, the easiest way is to build Octez from - source. In addition to the `usual instructions - `_, - the command ``make build-unreleased`` has to be used. + source. See the `usual instructions + `_. For now, ``octez-wasm-repl`` is **not** part of Octez, and is only provided for developers interested in testing Tezos smart rollup diff --git a/scripts/ci/build_full_unreleased.sh b/scripts/ci/build_full_unreleased.sh index 0cc81049a494..e3d6184ef9fd 100755 --- a/scripts/ci/build_full_unreleased.sh +++ b/scripts/ci/build_full_unreleased.sh @@ -19,7 +19,7 @@ diff poetry.lock /home/tezos/poetry.lock diff pyproject.toml /home/tezos/pyproject.toml # 2. Actually build -make build-unreleased +make # 3. Also build the tps evaluation tool which is not part of the default build. # NOTE: We add $COVERAGE_OPTIONS to all dune build commands to enable reuse of diff --git a/scripts/ci/build_static_binaries.sh b/scripts/ci/build_static_binaries.sh index c264344ebd26..04b014c1b0a6 100755 --- a/scripts/ci/build_static_binaries.sh +++ b/scripts/ci/build_static_binaries.sh @@ -20,7 +20,8 @@ echo "Create destination directory" mkdir -pv "octez-binaries/$ARCH" echo "Build and install static binaries" -make static +# shellcheck disable=SC2086 +make static OCTEZ_EXECUTABLES="$(cat $EXECUTABLE_FILES)" echo "Check executables and move them to the destination directory" # Disable https://www.shellcheck.net/wiki/SC2086 because: -- GitLab From 1c29019a6e34fafb6dd0a8860fecfdb50f31eeec Mon Sep 17 00:00:00 2001 From: Romain Bardou Date: Thu, 24 Nov 2022 10:46:53 +0100 Subject: [PATCH 2/2] Manifest: sc_rollup_protocol_versions no longer useful --- build.Dockerfile | 1 - manifest/main.ml | 17 ----------------- .../sc_rollup_protocol_versions_without_number | 1 - 3 files changed, 19 deletions(-) delete mode 100644 script-inputs/sc_rollup_protocol_versions_without_number diff --git a/build.Dockerfile b/build.Dockerfile index 99402471e095..8d975295dded 100644 --- a/build.Dockerfile +++ b/build.Dockerfile @@ -13,7 +13,6 @@ RUN mkdir -p /home/tezos/tezos/scripts /home/tezos/tezos/script-inputs /home/tez COPY --chown=tezos:nogroup Makefile tezos COPY --chown=tezos:nogroup script-inputs/active_protocol_versions tezos/script-inputs/ COPY --chown=tezos:nogroup script-inputs/active_protocol_versions_without_number tezos/script-inputs/ -COPY --chown=tezos:nogroup script-inputs/sc_rollup_protocol_versions_without_number tezos/script-inputs/ COPY --chown=tezos:nogroup script-inputs/released-executables tezos/script-inputs/ COPY --chown=tezos:nogroup script-inputs/experimental-executables tezos/script-inputs/ COPY --chown=tezos:nogroup dune tezos diff --git a/manifest/main.ml b/manifest/main.ml index 1ca37f8c746c..c1e977fa16d7 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -6096,21 +6096,4 @@ let () = write "script-inputs/active_protocol_versions_without_number" @@ fun fmt -> List.iter (write_protocol fmt) Protocol.active -(* TODO: https://gitlab.com/tezos/tezos/-/issues/4194 - - `sc_rollup_protocol_versions_without_number` can be removed and replace by - `active_protocol_versions_without_number` (i.e. `Protocol.active`) when all - active protocol support scoru. *) -(* Generate sc_rollup_protocol_versions_without_number. *) -let () = - let write_protocol fmt protocol = - match Protocol.number protocol with - | Alpha -> Format.fprintf fmt "%s\n" (Protocol.short_hash protocol) - | V v when v >= 16 -> - Format.fprintf fmt "%s\n" (Protocol.short_hash protocol) - | V _ | Other -> () - in - write "script-inputs/sc_rollup_protocol_versions_without_number" @@ fun fmt -> - List.iter (write_protocol fmt) Protocol.active - let () = check ~exclude () diff --git a/script-inputs/sc_rollup_protocol_versions_without_number b/script-inputs/sc_rollup_protocol_versions_without_number deleted file mode 100644 index 4a58007052a6..000000000000 --- a/script-inputs/sc_rollup_protocol_versions_without_number +++ /dev/null @@ -1 +0,0 @@ -alpha -- GitLab