From 0dd5e0016b758d5cccf8c8eae81e8f3b00f60fd9 Mon Sep 17 00:00:00 2001 From: Romain Bardou Date: Wed, 6 Mar 2024 16:41:12 +0100 Subject: [PATCH 1/6] Doc: update instructions to update dependencies --- ...tributing-adding-a-new-opam-dependency.rst | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/docs/developer/contributing-adding-a-new-opam-dependency.rst b/docs/developer/contributing-adding-a-new-opam-dependency.rst index 335745d685b7..38fcb1a9359b 100644 --- a/docs/developer/contributing-adding-a-new-opam-dependency.rst +++ b/docs/developer/contributing-adding-a-new-opam-dependency.rst @@ -27,10 +27,14 @@ To this end: The Docker images for the CI are built by the CI of another repository, the so-called `Tezos opam repository `__. -For legacy reasons, the Tezos opam repository is actually a subset -of the public opam repository containing the same dependencies as the lock file, -plus a few others such as ``odoc`` which is needed by the CI but not to build Octez. -Docker images are built from those package definitions. +The set of dependencies that is used to build those images is also defined +by a lock file and a commit hash from the public opam repository. +Both must be kept synchronized with their counterpart in the ``tezos/tezos`` repository. + +.. note:: + + Docker images contain additional dependencies + such as ``odoc`` which are needed by the CI but not to build Octez. Adding, removing or updating dependencies thus requires to work both on the `main codebase `__ and on @@ -109,15 +113,8 @@ not an issue in general but it might explain some changes unrelated to your work as it does not guarantee that the set of dependencies is actually a valid solution that the opam solver could have chosen. -Third, still in your local copy of Octez, -**execute the** :src:`scripts/update_opam_repo.sh` **script**. -This script creates a file called ``opam_repo.patch``. -This file contains the difference between the current version of the -Tezos opam repository, and what it needs to be to take your new lock file -into account. - -Fourth, **create an MR on the Tezos opam repository that applies -your patch.** This is the *opam repository MR*, its role is to prepare +Third, **create an MR on the Tezos opam repository.** +This is the *opam repository MR*, its role is to prepare the environment for the *Octez MR* that we will create below. In order to create the opam repository MR: @@ -125,9 +122,13 @@ In order to create the opam repository MR: - If you haven’t already done so, clone `the Tezos opam repository `__. - Create a branch from the repository's ``master`` and switch to it. -- Apply the patch generated by :src:`scripts/update_opam_repo.sh` - (``git apply /opam_repo.path``). -- Commit the applied patch. Take note of the commit hash, it will be useful later. +- Update file ``scripts/version.sh`` (in the Tezos opam repository) + to set the value of ``opam_repository_commit_hash`` + to match the value of ``full_opam_repository_tag`` that you have set in + :src:`scripts/version.sh` (in the Octez repository). +- Copy file :src:`opam/virtual/octez-deps.opam.locked` (from the Octez repository) + to the root of the Tezos opam repository. +- Commit the result. Take note of the commit hash, it will be useful later. - Push your branch. - Create the opam repository MR from this branch. @@ -135,10 +136,10 @@ You can test the MR locally using the command ``OPAM_REPOSITORY_TAG= make build-deps``. This will rebuild the dependencies locally using the ```` of the opam-repository. -Fifth, back in your local copy of Octez, **update the** ``opam_repository_tag`` **variable in the** -:src:`scripts/version.sh` **file**. Specifically, set it +Fourth, back in your local copy of Octez, **update the** ``opam_repository_tag`` +**variable in the** :src:`scripts/version.sh` **file**. Specifically, set it to the hash of your commit on the opam repository MR. -Afterwards, you will also need to regenerate the GitLab CI configuration +Afterwards, you will also need to regenerate the GitLab CI configuration by running ``make -C ci`` from the root of the repository. Commit the change of ``scripts/version.sh`` and the GitLab configuration with a title along the lines of “CI: use dependency ``foo``”. @@ -204,9 +205,12 @@ As a developer: a recent version of the public default opam repository. - You update :src:`opam/virtual/octez-deps.opam.locked`, for instance by executing :src:`scripts/update_opam_lock.sh`. -- You execute :src:`scripts/update_opam_repo.sh`. - You open an opam repository MR from ``tezos/opam-repository:`` - onto ``tezos/opam-repository:master`` that includes the generated patch. + onto ``tezos/opam-repository:master`` that updates: + + - variable ``opam_repository_commit_hash`` in ``scripts/version.sh``; + - file ``octez-deps.opam.locked`` at the root. + - You update ``opam_repository_tag`` to the hash of the last commit of your opam repository MR and regenerate the CI configuration. - You push the changes to your Octez MR. -- GitLab From 8be5ae23ba6b1de5a6a3872494eaab543b72c196 Mon Sep 17 00:00:00 2001 From: Romain Bardou Date: Wed, 6 Mar 2024 16:41:32 +0100 Subject: [PATCH 2/6] Scripts: deprecate update_opam_repo.sh --- .gitignore | 3 - scripts/update_opam_repo.sh | 158 ++++-------------------------------- 2 files changed, 17 insertions(+), 144 deletions(-) diff --git a/.gitignore b/.gitignore index 0b44e89b1274..a198981938eb 100644 --- a/.gitignore +++ b/.gitignore @@ -94,9 +94,6 @@ contrib/kaitai-struct-files/doc **/.venv **/.mypy_cache/ -# Output of the update_opam_repo script -opam_repo.patch - # Rust target diff --git a/scripts/update_opam_repo.sh b/scripts/update_opam_repo.sh index 8d57ed8b1a6a..bc57ebbb92f4 100755 --- a/scripts/update_opam_repo.sh +++ b/scripts/update_opam_repo.sh @@ -1,148 +1,24 @@ #!/bin/sh -# Update the repository of opam packages used by tezos. Tezos uses a -# private, shrunk down, opam repository to store all its -# dependencies. This is generated by the official opam repository -# (branch master) and then filtered using opam admin to include only -# the cone of tezos dependencies. This repository is then used to -# create the based opam image used by the CI to compile tezos and to -# generate the docker images. From time to time, when it is necessary -# to update a dependency, this repository should be manually -# refreshed. This script takes care of generating a patch for the -# private opam tezos repository. This patch must be applied manually -# w.r.t. the master branch. The procedure is as follows : -# -# 1. Update the variable `full_opam_repository_tag` in `version.sh` to -# a commit hash from the master branch of the official -# opam-repository. All the required packages will be extracted from -# this snapshot to the repo. -# -# 2. Run this script, it will generate a file `opam_repo.patch` -# -# 3. Review the patch. -# -# 4. In the tezos opam-repository, create a new branch from master and -# apply this patch. Push the patch and create a merge request. A -# new docker image with all the prebuilt dependencies will be -# created by the CI. -# -# 5. Update the variable `opam_repository_tag` in `scripts/version.sh` -# and the variable `build_deps_image_version` in `.gitlab-ci.yml` -# with the hash of the newly created commit in `tezos/opam-repository`. -# -# 6. Enjoy your new dependencies +cat << EOT +This script is deprecated. -set -e +To update dependencies, you have to: -target="$(pwd)"/opam_repo.patch tmp_dir=$(mktemp -dt tezos_deps_opam.XXXXXXXX) +- in tezos/tezos: + - update version constraints in manifest/ and run: make -C manifest + - update full_opam_repository_tag in: scripts/version.sh + - update the lock file in: opam/virtual/octez-deps.opam.locked + (for instance using: scripts/update_opam_lock.sh) -cleanup() { - set +e - echo Cleaning up... - rm -rf "$tmp_dir" - rm -rf Dockerfile -} -trap cleanup EXIT INT +- in tezos/opam-repository: + - update opam_repository_commit_hash in: scripts/version.sh + (to match full_opam_repository_tag from scripts/version.sh from tezos/tezos) + - update the opam lock file: octez-deps.opam.locked + (copy opam/virtual/octez-deps.opam.locked from tezos/tezos) -script_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")" -src_dir="$(dirname "$script_dir")" +More information in the documentation: +http://tezos.gitlab.io/developer/contributing-adding-a-new-opam-dependency.html +EOT -# shellcheck source=scripts/version.sh -. "$script_dir"/version.sh - -## Shallow clone of opam repository (requires git protocol version 2) -export GIT_WORK_TREE="$tmp_dir" -export GIT_DIR="$GIT_WORK_TREE/.git" -git init -git config --local protocol.version 2 -git remote add origin https://github.com/ocaml/opam-repository -git fetch --depth 1 origin "$full_opam_repository_tag" - -## Adding the various tezos packages - -mkdir -p "$tmp_dir"/packages/octez-deps/octez-deps.dev -cp opam/virtual/octez-deps.opam.locked "$tmp_dir"/packages/octez-deps/octez-deps.dev/opam - -## Filtering unrequired packages -cd "$tmp_dir" -git reset --hard "$full_opam_repository_tag" - -## we add a dummy package that conflict with all "hidden" packages -dummy_pkg=dummy-tezos -dummy_path=packages/$dummy_pkg/$dummy_pkg.dev -dummy_opam=$dummy_path/opam -mkdir -p $dummy_path -echo 'opam-version: "2.0"' > $dummy_opam -echo "depends: [ \"ocaml\" { = \"$ocaml_version\" } ]" >> $dummy_opam -echo 'conflicts:[' >> $dummy_opam -grep -r "^flags: *\[ *avoid-version *\]" -l ./ | LC_COLLATE=C sort -u | while read -r f; do - f=$(dirname "$f") - f=$(basename "$f") - p=$(echo "$f" | cut -d '.' -f '1') - v=$(echo "$f" | cut -d '.' -f '2-') - echo "\"$p\" {= \"$v\"}" >> $dummy_opam -done -# FIXME: https://gitlab.com/tezos/tezos/-/issues/5832 -# opam unintentionally picks up a windows dependency. We add a -# conflict here to work around it. -echo '"ocamlbuild" {= "0.14.2+win" }' >> $dummy_opam -echo ']' >> $dummy_opam - -# Opam < 2.1 requires opam-depext as a plugin, later versions include it -# natively: -case $(opam --version) in -2.0.*) opam_depext_dep="opam-depext," ;; -*) opam_depext_dep="" ;; -esac -#shellcheck disable=SC2086 -OPAMSOLVERTIMEOUT=600 opam admin filter --yes --resolve \ - octez-deps,ocaml,ocaml-base-compiler,odoc,${opam_depext_dep}ledgerwallet-tezos,caqti-driver-postgresql,js_of_ocaml-lwt,$dummy_pkg -## - ocaml-base-compiler has to be explicitely listed for the solver -## to not prefer the "variant" `system` of the compiler -## - odoc is used by the CI to generate the doc -## - ledgerwallet-tezos is an optional dependency of signer-services -## we want to have when building released binaries -## - caqti-driver-postgresq is needed by tps measurement software to -## read tezos-indexer databases -## - js_of_ocaml-lwt is an optional dependency of tezt which is needed -## to build tezt.js, and we do want to run some tests using nodejs - -## Adding useful compiler variants -for variant in afl flambda fp; do - git checkout packages/ocaml-option-$variant/ocaml-option-$variant.1 -done - -## Removing temporary hacks -rm -r "$tmp_dir"/packages/octez-deps -rm -r "$tmp_dir"/packages/$dummy_pkg - -## Checkout tezos/opam-repository while keeping the working directory -git remote add tezos $opam_repository_git -git fetch --depth 1 tezos "$opam_repository_tag" -git reset "$opam_repository_tag" - -## opam.2.1 will try to delete opam-depext, we should restore it. -if [ ! -d packages/opam-depext ]; then - git checkout HEAD -- packages/opam-depext -fi - -## Adding safer hashes -cp -rf packages packages.bak - -opam admin add-hashes sha256 sha512 - -(cd "$src_dir" && dune build src/tooling/opam-lint/opam_lint.exe) -for i in $(cd packages && find ./ -name opam); do - "$src_dir/_build/default/src/tooling/opam-lint/opam_lint.exe" "packages/$i" "packages.bak/$i" -done -rm -rf packages.bak - -## Generating the diff! -git add packages -git diff HEAD -- packages > "$target" - -echo -echo "Wrote proposed update in: $target." -echo 'Please add this patch to: https://gitlab.com/tezos/opam-repository' -echo 'And update accordingly the commit hash in: .gitlab/ci/templates.yml and scripts/version.sh' -echo +exit 1 -- GitLab From e7093f702bfff01db4e3987b1f7cd0ba603047f4 Mon Sep 17 00:00:00 2001 From: Romain Bardou Date: Wed, 6 Mar 2024 16:42:58 +0100 Subject: [PATCH 3/6] CI: use Docker images that have version.sh --- .gitlab-ci.yml | 2 +- scripts/version.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3f99449bc614..685910450f13 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -59,7 +59,7 @@ default: interruptible: true variables: - build_deps_image_version: d8bd1a0e555b2f9a32bce18a4022f6851c700bca + build_deps_image_version: df242dbcfa69587ccc182e4f3c3c02288407a097 build_deps_image_name: ${CI_REGISTRY}/tezos/opam-repository rust_toolchain_image_name: ${GCP_REGISTRY}/${CI_PROJECT_PATH}/rust-toolchain client_libs_dependencies_image_name: ${GCP_REGISTRY}/${CI_PROJECT_PATH}/client-libs-dependencies diff --git a/scripts/version.sh b/scripts/version.sh index e3ef7ee0cb91..c4e939d5f634 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -32,7 +32,7 @@ export full_opam_repository_tag=2314da5646931ec7f643bdc9aaa39177971ac857 ## This value MUST be reflected in the `build_deps_image_version` variable ## of `.gitlab-ci.yml`, which is ensured by running `make -C ci` from the root. export opam_repository_url=https://gitlab.com/tezos/opam-repository -export opam_repository_tag="${OPAM_REPOSITORY_TAG:-d8bd1a0e555b2f9a32bce18a4022f6851c700bca}" +export opam_repository_tag="${OPAM_REPOSITORY_TAG:-df242dbcfa69587ccc182e4f3c3c02288407a097}" export opam_repository_git="$opam_repository_url.git" export opam_repository="$opam_repository_git"\#"$opam_repository_tag" -- GitLab From 7038861b8db3eef20d581cd3e005f840e9c5035f Mon Sep 17 00:00:00 2001 From: Romain Bardou Date: Wed, 6 Mar 2024 16:44:00 +0100 Subject: [PATCH 4/6] CI: remove check that uses update_opam_repo.sh --- .gitlab/ci/jobs/test/misc_opam_checks.yml | 5 ----- scripts/opam-check.sh | 19 ------------------- 2 files changed, 24 deletions(-) diff --git a/.gitlab/ci/jobs/test/misc_opam_checks.yml b/.gitlab/ci/jobs/test/misc_opam_checks.yml index d4b0e17e5cb3..ea65f24bebd7 100644 --- a/.gitlab/ci/jobs/test/misc_opam_checks.yml +++ b/.gitlab/ci/jobs/test/misc_opam_checks.yml @@ -8,8 +8,3 @@ misc_opam_checks: script: # checks that all deps of opam packages are already installed - ./scripts/opam-check.sh - artifacts: - when: always - paths: - - opam_repo.patch - expire_in: 1 day diff --git a/scripts/opam-check.sh b/scripts/opam-check.sh index 77b41ed4db4f..bc60809b1fde 100755 --- a/scripts/opam-check.sh +++ b/scripts/opam-check.sh @@ -21,23 +21,4 @@ if ! opam install opam/virtual/octez-deps.opam.locked --deps-only --with-test -- exit 1 fi -# We really want literal backticks here, not command substitution. -# shellcheck disable=SC2016 -echo '## Running `./scripts/update_opam_repo.sh`' -echo -./scripts/update_opam_repo.sh || exit 1 - -if [ -n "$(cat opam_repo.patch)" ]; then - - echo "##################################################" - cat opam_repo.patch - echo "##################################################" - - # We really want literal backticks here, not command substitution. - # shellcheck disable=SC2016 - echo 'Failed! The variables `opam_repository_tag` and `full_opam_repository_tag` are not synchronized. Please read the doc in `./scripts/update_opam_repo.sh` and act accordingly.' - echo - exit 1 -fi - echo "Ok." -- GitLab From 9e5d76fbd7edfab2067e5cc26609b01eade4419b Mon Sep 17 00:00:00 2001 From: Romain Bardou Date: Wed, 6 Mar 2024 16:48:12 +0100 Subject: [PATCH 5/6] CI: add check about opam repository commit hash Co-authored-by: Arvid Jakobsson --- scripts/opam-check.sh | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/scripts/opam-check.sh b/scripts/opam-check.sh index bc60809b1fde..4c34bf9498ec 100755 --- a/scripts/opam-check.sh +++ b/scripts/opam-check.sh @@ -1,5 +1,10 @@ #!/bin/sh +# This script expects a copy of scripts/version.sh from tezos/opam-repository +# to be available at: /home/tezos/version.sh +# This is the case if this script runs in one of the Docker images that are +# built by tezos/opam-repository. + script_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")" # shellcheck source=scripts/version.sh @@ -21,4 +26,28 @@ if ! opam install opam/virtual/octez-deps.opam.locked --deps-only --with-test -- exit 1 fi +# Check that the value of opam_repository_commit_hash in tezos/opam-repository +# matches the value of full_opam_repository_tag in tezos/tezos. +# It happens that tezos/opam-repository keeps its version.sh in the Docker image, +# so we read the value of opam_repository_commit_hash from it. +echo "## Checking opam repository commit hash..." + +# Source the copy of `version.sh` in the image produced by tezos/opam-repository +opam_repository_commit_hash=$( + # '/home/tezos/version.sh' is not in the source repo but in + # one if the images, so shellcheck cannot analyze it. + # shellcheck disable=SC1091 + . /home/tezos/version.sh + echo "$opam_repository_commit_hash" +) + +echo "- opam_repository_commit_hash = $opam_repository_commit_hash (from tezos/opam-repository)" +echo "- full_opam_repository_tag = $full_opam_repository_tag (from tezos/tezos)" + +if [ "$opam_repository_commit_hash" != "$full_opam_repository_tag" ]; then + echo "Error: values do not match." + echo "See http://tezos.gitlab.io/developer/contributing-adding-a-new-opam-dependency.html" + exit 1 +fi + echo "Ok." -- GitLab From ddfada78eb809cec7feb14f882bda5b2fb2b708c Mon Sep 17 00:00:00 2001 From: Romain Bardou Date: Mon, 11 Mar 2024 14:24:19 +0100 Subject: [PATCH 6/6] Scripts: add comment about running in the CI --- scripts/opam-check.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/opam-check.sh b/scripts/opam-check.sh index 4c34bf9498ec..3a98a8516fae 100755 --- a/scripts/opam-check.sh +++ b/scripts/opam-check.sh @@ -1,5 +1,7 @@ #!/bin/sh +# Implementation of CI job 'misc_opam_checks'. + # This script expects a copy of scripts/version.sh from tezos/opam-repository # to be available at: /home/tezos/version.sh # This is the case if this script runs in one of the Docker images that are -- GitLab