diff --git a/.gitignore b/.gitignore index 2325029a19ad7a9ea5aa35c4fdd2eaf5eceedd29..809afe4e4ff2d373db3f41a0c14bd78f17c17c0f 100644 --- a/.gitignore +++ b/.gitignore @@ -92,9 +92,6 @@ __pycache__ **/.venv **/.mypy_cache/ -# Output of the update_opam_repo script -opam_repo.patch - # Rust target diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c77a169f48c18e9d02a7deb602ff20268504ca23..b0cf294bea2de9e2de13e0039510c6afa2ea0d2a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,8 +40,8 @@ workflow: variables: # /!\ CI_REGISTRY is overriden to use a private Docker registry mirror in AWS ECR # in GitLab namespaces `nomadic-labs` and `tezos` - ## This value MUST be the same as `opam_repository_tag` in `scripts/version.sh` - build_deps_image_version: 2703ff3a4dbbfa575a7a3d6dc2fff2a4b23c2045 + ## This value MUST be the same as `base_image_repository_tag` in `scripts/version.sh` + build_deps_image_version: 3ac5259b93529e05d8431cb09ef8225fdfe6a152 build_deps_image_name: "${CI_REGISTRY}/tezos/opam-repository" GIT_STRATEGY: fetch GIT_DEPTH: "1" diff --git a/.gitlab/ci/jobs/test/misc_opam_checks.yml b/.gitlab/ci/jobs/test/misc_opam_checks.yml index 94ac0f6f3723e2dae7de32f885e3351c0cf6aff2..77349e18beb0160eea005817116ca815c14b64d4 100644 --- a/.gitlab/ci/jobs/test/misc_opam_checks.yml +++ b/.gitlab/ci/jobs/test/misc_opam_checks.yml @@ -5,8 +5,3 @@ misc_opam_checks: script: # checks that all deps of opam packages are already installed - ./scripts/opam-check.sh - artifacts: - when: always - paths: - - opam_repo.patch - expire_in: 1 days diff --git a/Makefile b/Makefile index 2cff9ead9423bb7d98edc3e045618cdda4f6b928..42f153e9b88533436b1cac55a98ab832de298ddf 100644 --- a/Makefile +++ b/Makefile @@ -22,8 +22,8 @@ DOCKER_BARE_IMAGE_VERSION := latest DOCKER_DEBUG_IMAGE_NAME := $(DOCKER_IMAGE_NAME)-debug DOCKER_DEBUG_IMAGE_VERSION := latest DOCKER_DEPS_IMAGE_NAME := registry.gitlab.com/tezos/opam-repository -DOCKER_DEPS_IMAGE_VERSION := runtime-build-dependencies--${opam_repository_tag} -DOCKER_DEPS_MINIMAL_IMAGE_VERSION := runtime-dependencies--${opam_repository_tag} +DOCKER_DEPS_IMAGE_VERSION := runtime-build-dependencies--${base_image_repository_tag} +DOCKER_DEPS_MINIMAL_IMAGE_VERSION := runtime-dependencies--${base_image_repository_tag} COVERAGE_REPORT := _coverage_report COBERTURA_REPORT := _coverage_report/cobertura.xml CODE_QUALITY_REPORT := _reports/gl-code-quality-report.json diff --git a/devtools/testnet_experiment_tools/docker/create_docker_image.sh b/devtools/testnet_experiment_tools/docker/create_docker_image.sh index 86b9896960aeb767dea78837733803c45eaad2be..7282c4a37f79945b3801c2e683521934ba635114 100755 --- a/devtools/testnet_experiment_tools/docker/create_docker_image.sh +++ b/devtools/testnet_experiment_tools/docker/create_docker_image.sh @@ -25,7 +25,7 @@ cd "$src_dir" image_name="${1:-tezos-}" image_version="${2:-latest}" build_deps_image_name=${3:-registry.gitlab.com/tezos/opam-repository} -build_deps_image_version=${4:-$opam_repository_tag} +build_deps_image_version=${4:-$base_image_repository_tag} executables=${5:-$(cat "$devtools_docker_dir"/executables)} commit_short_sha="${6:-$(git rev-parse --short HEAD)}" docker_target="${7:-without-evm-artifacts}" diff --git a/docs/developer/contributing-adding-a-new-opam-dependency.rst b/docs/developer/contributing-adding-a-new-opam-dependency.rst index 7865f9728a44d590ab8d976c23d0bbab545c7b69..d2e6d0fec5820842325fad7e8f2b6c12af279088 100644 --- a/docs/developer/contributing-adding-a-new-opam-dependency.rst +++ b/docs/developer/contributing-adding-a-new-opam-dependency.rst @@ -12,17 +12,19 @@ If you have already read this guide and only need a refresher, skip to the Background ---------- -The Octez project is built under a system that is somewhat stricter than -the default for OCaml project. Specifically, the Octez project maintains -a dedicated opam package repository that is a strict subset of the opam -default one; all binaries are built with dependencies from this subset -only. +OCaml dependencies of Octez are listed in an opam lock file, +namely :src:`opam/virtual/octez-deps.opam.locked`. +The CI of Octez uses Docker images where dependencies +are already compiled. Those Docker images are built by the CI of +another repository: +`tezos/opam-repository `__. +This other repository also has a copy of ``octez-deps.opam.locked`` +at its root. For this reason, adding or updating a dependency requires to work both -on `the main codebase `__ and on `the -dedicated opam package -repository `__. Moreover, work -between those two components must happen in a specific order. +on `the main codebase `__ and on +`tezos/opam-repository `__. +Moreover, work between those two components must happen in a specific order. The rest of this document explains the process from the point-of-view of a developer (you). The instructions below assume you have already @@ -70,46 +72,28 @@ the CI of the dedicated ``opam-repository``. You must follow the steps below in order to produce the necessary Docker images, allowing your work to eventually be merged. -First, in your local copy of Octez, **update the** -``full_opam_repository_tag`` **variable in the** :src:`scripts/version.sh` -**file**. You -should set this variable to the hash of the ``HEAD`` commit on -`the default opam repository `__. -(Note: this is not always necessary, but it is simpler for you to do so -than to check whether it is necessary to do so.) - -Second, still in your local copy of Octez, **execute the** -:src:`scripts/update_opam_repo.sh` **script**. This script uses the content of -your :src:`opam/` directory to create a file -called ``opam_repo.patch``. This file represents the diff between the current -dedicated opam repository and the dedicated opam repository that your MR -needs. - -Note that the diff may include a few more changes than what you strictly need. -Specifically, it might include some updates of some other dependencies. This is -not an issue in general but it might explain some changes unrelated to your -work. - -Third, **create an MR on the dedicated opam repository that includes -your patch.** This is the *opam repository MR*, its role is to prepare -the environment for your existing *Octez MR*. - -In order to create the opam repository MR: - -- If you haven’t already done so, clone `the dedicated opam repository `__. +First, **update the lock file** as follows, from your local copy of ``tezos/tezos`` +(remember that this assumes that you already have installed your new dependency +with ``opam install foo``):: + + opam lock opam/virtual/octez-deps.opam opam/virtual/octez-dev-deps.opam + mv octez-deps.opam.locked opam/virtual + rm octez-dev-deps.opam.locked + +Second, copy ``opam/virtual/octez-deps.opam.locked`` into your local copy of +``tezos/opam-repository`` and **create an MR**: + +- If you haven’t already done so, clone `tezos/opam-repository `__. - Create a branch from the repository's ``master`` and switch to it. -- Apply the patch generated by :src:`scripts/update_opam_repo.sh` (``git apply /opam_repo.path``). -- Commit the applied patch. +- Copy ``opam/virtual/octez-deps.opam.locked`` from your clone of ``tezos/tezos`` + into the root of your clone of ``tezos/opam-repository``. +- Commit the changes. - Push your branch. - Create the opam repository MR from this branch. -You can test the MR locally using the command -``OPAM_REPOSITORY_TAG= make build-deps``. This will rebuild the -dependencies locally using the ```` of the opam-repository. - -Fourth, back in your local copy of Octez, **update the variables in the** +Third, back in your local copy of Octez, **update the variables in the** :src:`.gitlab-ci.yml` **and** :src:`scripts/version.sh` **files**. Specifically, set -the ``build_deps_image_version`` and the ``opam_repository_tag`` variables +the ``build_deps_image_version`` and the ``base_image_repository_tag`` variables to the hash of the ``HEAD`` commit of the opam repository MR. Commit this change with a title along the lines of “CI: use dependency ``foo``”. @@ -119,7 +103,7 @@ opam-repository and the associated Docker images. Do note that the CI on your branch of Octez will only be able to run after the CI on your branch of opam-repository has completed. -Fifth, still in your local copy of Octez, **push these changes and open or +Finally, still in your local copy of Octez, **push these changes and open or update the MR**. Make sure you add links referencing the opam-repository MR from the Octez MR and vice-versa. This gives the reviewers the necessary context to review. @@ -162,13 +146,14 @@ TL;DR As a developer: -- You have an Octez MR from ``/tezos:`` onto ``tezos/tezos:master`` introducing a dependency to ``foo``. +- You have a Octez MR from ``/tezos:`` onto ``tezos/tezos:master`` introducing a dependency to ``foo``. +- You install your dependency with ``opam install foo``. - You amend the :src:`manifest/main.ml` file to declare the dependency. - You propagate the changes to ``opam`` and ``dune`` files by running ``make -C manifest`` -- You update the ``full_opam_repository_tag`` to the HEAD commit hash from the public default opam repository. -- You execute :src:`scripts/update_opam_repo.sh`. +- You run ``opam lock opam/virtual/octez-deps.opam`` to update the lock file. + Alternatively, you execute :src:`scripts/update_opam_repo.sh` to update all dependencies. - You open an opam repository MR from ``tezos/opam-repository:`` onto ``tezos/opam-repository:master`` that includes the generated patch. -- You update ``build_deps_image_version`` and ``opam_repository_tag`` to the hash of the ``HEAD`` commit of your opam repository MR. +- You update ``build_deps_image_version`` and ``base_image_repository_tag`` to the hash of the ``HEAD`` commit of your opam repository MR. - You push the changes to your Octez MR. - You update the description of your MRs to include links. diff --git a/nix/opam-repo.nix b/nix/opam-repo.nix index 9137e39221ee0fa5785a12911e12dfa27c71bca6..a932612a90c15a1f94f9ca5148360efe86cf2faf 100644 --- a/nix/opam-repo.nix +++ b/nix/opam-repo.nix @@ -7,7 +7,7 @@ } '' . $src - echo -n $full_opam_repository_tag > $out + echo -n $base_image_repository_tag | cut -d'#' -f2 > $out '' ); in diff --git a/nix/tezos-opam-repo.nix b/nix/tezos-opam-repo.nix index 1be740253741d972890d9ed51f173d0bbb671f6c..65f92a8d6df03976a4321c735ffcef358f01552d 100644 --- a/nix/tezos-opam-repo.nix +++ b/nix/tezos-opam-repo.nix @@ -7,7 +7,7 @@ } '' . $src - echo -n $opam_repository_tag > $out + echo -n $base_image_repository_tag > $out '' ); in diff --git a/opam/virtual/octez-deps.opam.locked b/opam/virtual/octez-deps.opam.locked new file mode 100644 index 0000000000000000000000000000000000000000..c3095dafa5c7ac9d2141f183dba606178ae39c15 --- /dev/null +++ b/opam/virtual/octez-deps.opam.locked @@ -0,0 +1,240 @@ +opam-version: "2.0" +name: "octez-deps" +version: "~dev" +synopsis: + "Virtual package depending on Octez dependencies (profile: octez-deps)" +description: + "Install this package to install all dependencies needed to build the subset of Octez denoted by profile octez-deps." +maintainer: "contact@tezos.com" +authors: "Tezos devteam" +license: "MIT" +homepage: "https://www.tezos.com/" +bug-reports: "https://gitlab.com/tezos/tezos/issues" +depends: [ + "aches" {= "1.0.0"} + "aches-lwt" {= "1.0.0"} + "alcotest" {= "1.7.0"} + "angstrom" {= "0.15.0"} + "asetmap" {= "0.8.1"} + "asn1-combinators" {= "0.2.6"} + "astring" {= "0.8.5"} + "base" {= "v0.15.1"} + "base-bigarray" {= "base"} + "base-bytes" {= "base"} + "base-threads" {= "base"} + "base-unix" {= "base"} + "base64" {= "3.5.1"} + "bheap" {= "2.0.0"} + "bigarray-compat" {= "1.1.0"} + "bigstring" {= "0.3"} + "bigstringaf" {= "0.9.1"} + "bisect_ppx" {= "2.8.1"} + "bos" {= "0.2.1"} + "ca-certs" {= "0.2.3"} + "camlp-streams" {= "5.0.1"} + "camlzip" {= "1.11"} + "caqti" {= "1.9.0"} + "caqti-dynload" {= "1.3.0"} + "caqti-lwt" {= "1.9.0"} + "checkseum" {= "0.5.1"} + "class_group_vdf" {= "0.0.4"} + "cmdliner" {= "1.2.0"} + "cohttp" {= "5.3.0"} + "cohttp-lwt" {= "5.3.0"} + "cohttp-lwt-unix" {= "5.3.0"} + "conduit" {= "6.2.0"} + "conduit-lwt" {= "6.2.0"} + "conduit-lwt-unix" {= "6.2.0"} + "conf-autoconf" {= "0.1"} + "conf-cmake" {= "1"} + "conf-g++" {= "1.0"} + "conf-gmp" {= "4"} + "conf-gmp-powm-sec" {= "3"} + "conf-hidapi" {= "0"} + "conf-libev" {= "4-12"} + "conf-libffi" {= "2.0.0"} + "conf-pkg-config" {= "3"} + "conf-rust" {= "0.1"} + "conf-rust-2021" {= "1"} + "conf-which" {= "1"} + "conf-zlib" {= "1"} + "cppo" {= "1.6.9"} + "csexp" {= "1.5.2"} + "cstruct" {= "6.2.0"} + "cstruct-lwt" {= "6.2.0"} + "ctypes" {= "0.20.2"} + "ctypes-foreign" {= "0.18.0"} + "ctypes_stubs_js" {= "0.1"} + "data-encoding" {= "0.7.1"} + "decompress" {= "1.5.2"} + "digestif" {= "1.1.4"} + "dmap" {= "0.5"} + "domain-name" {= "0.4.0"} + "dune" {= "3.10.0"} + "dune-build-info" {= "3.10.0"} + "dune-configurator" {= "3.10.0"} + "duration" {= "0.2.1"} + "either" {= "1.0.0"} + "eqaf" {= "0.9"} + "ezjsonm" {= "1.3.0"} + "fix" {= "20230505"} + "fmt" {= "0.9.0"} + "fpath" {= "0.7.3"} + "gen" {= "1.1"} + "gmap" {= "0.3.0"} + "hacl-star" {= "0.7.1"} + "hacl-star-raw" {= "0.7.1"} + "hashcons" {= "1.3"} + "hex" {= "1.5.0"} + "hidapi" {= "1.1.2"} + "hkdf" {= "1.0.4"} + "index" {= "1.6.1"} + "integers" {= "0.7.0"} + "integers_stubs_js" {= "1.0"} + "ipaddr" {= "5.5.0"} + "ipaddr-sexp" {= "5.5.0"} + "irmin" {= "3.7.2"} + "irmin-pack" {= "3.7.2"} + "jane-street-headers" {= "v0.15.0"} + "jingoo" {= "1.4.4"} + "js_of_ocaml" {= "5.4.0"} + "js_of_ocaml-compiler" {= "5.4.0"} + "js_of_ocaml-lwt" {= "5.4.0"} + "js_of_ocaml-ppx" {= "5.4.0"} + "json-data-encoding" {= "0.12.1"} + "json-data-encoding-bson" {= "0.12.1"} + "jsonm" {= "1.0.2"} + "jst-config" {= "v0.15.1"} + "lambda-term" {= "3.3.1"} + "ledgerwallet" {= "0.3.0"} + "ledgerwallet-tezos" {= "0.3.0"} + "logs" {= "0.7.0"} + "lru" {= "0.3.1"} + "lwt" {= "5.7.0"} + "lwt-canceler" {= "0.3"} + "lwt-exit" {= "1.0"} + "lwt-watcher" {= "0.2"} + "lwt_react" {= "1.1.2"} + "macaddr" {= "5.5.0"} + "magic-mime" {= "1.3.0"} + "menhir" {= "20230608"} + "menhirLib" {= "20230608"} + "menhirSdk" {= "20230608"} + "mew" {= "0.1.0"} + "mew_vi" {= "0.5.0"} + "mirage-crypto" {= "0.11.1"} + "mirage-crypto-ec" {= "0.11.1"} + "mirage-crypto-pk" {= "0.11.1"} + "mirage-crypto-rng" {= "0.11.1"} + "mirage-crypto-rng-lwt" {= "0.11.1"} + "mtime" {= "1.4.0"} + "num" {= "1.4"} + "ocaml" {= "4.14.1"} + "ocaml-base-compiler" {= "4.14.1"} + "ocaml-compiler-libs" {= "v0.12.4"} + "ocaml-config" {= "2"} + "ocaml-migrate-parsetree" {= "2.4.0"} + "ocaml-options-vanilla" {= "1"} + "ocaml-syntax-shims" {= "1.0.0"} + "ocaml-version" {= "3.5.0"} + "ocamlbuild" {= "0.14.2"} + "ocamlfind" {= "1.9.6"} + "ocamlformat" {= "0.24.1"} + "ocamlgraph" {= "2.0.0"} + "ocp-indent" {= "1.8.1"} + "ocp-ocamlres" {= "0.4"} + "ocplib-endian" {= "1.2"} + "odoc-parser" {= "2.0.0"} + "ometrics" {= "0.2.1"} + "optint" {= "0.3.0"} + "parsexp" {= "v0.15.0"} + "pbkdf" {= "1.2.0"} + "pprint" {= "20220103"} + "ppx_assert" {= "v0.15.0"} + "ppx_base" {= "v0.15.0"} + "ppx_cold" {= "v0.15.0"} + "ppx_compare" {= "v0.15.0"} + "ppx_derivers" {= "1.2.1"} + "ppx_deriving" {= "5.2.1"} + "ppx_enumerate" {= "v0.15.0"} + "ppx_expect" {= "v0.15.1"} + "ppx_hash" {= "v0.15.0"} + "ppx_here" {= "v0.15.0"} + "ppx_import" {= "1.9.1"} + "ppx_inline_test" {= "v0.15.1"} + "ppx_irmin" {= "3.7.2"} + "ppx_optcomp" {= "v0.15.0"} + "ppx_repr" {= "0.7.0"} + "ppx_sexp_conv" {= "v0.15.0"} + "ppxlib" {= "0.25.1"} + "prbnmcn-basic-structures" {= "0.0.1"} + "prbnmcn-linalg" {= "0.0.1"} + "prbnmcn-stats" {= "0.0.6"} + "pringo" {= "1.3"} + "progress" {= "0.2.1"} + "prometheus" {= "1.2"} + "prometheus-app" {= "1.2"} + "psq" {= "0.2.1"} + "ptime" {= "1.1.0"} + "pure-splitmix" {= "0.3"} + "pyml" {= "20220905"} + "qcheck-alcotest" {= "0.21.1"} + "qcheck-core" {= "0.21.1"} + "re" {= "1.10.4"} + "react" {= "1.2.2"} + "repr" {= "0.7.0"} + "resto" {= "1.2"} + "resto-acl" {= "1.2"} + "resto-cohttp" {= "1.2"} + "resto-cohttp-client" {= "1.2"} + "resto-cohttp-self-serving-client" {= "1.2"} + "resto-cohttp-server" {= "1.2"} + "resto-directory" {= "1.2"} + "result" {= "1.5"} + "ringo" {= "1.0.0"} + "rresult" {= "0.7.0"} + "rusage" {= "1.0.0"} + "secp256k1-internal" {= "0.4.0"} + "sedlex" {= "2.5"} + "semaphore-compat" {= "1.0.1"} + "seq" {= "base"} + "seqes" {= "0.2"} + "sexplib" {= "v0.15.1"} + "sexplib0" {= "v0.15.1"} + "stdcompat" {= "19"} + "stdint" {= "0.7.2"} + "stdio" {= "v0.15.0"} + "stdlib-shims" {= "0.3.0"} + "stringext" {= "1.6.0"} + "tar" {= "2.5.1"} + "tar-unix" {= "2.5.1"} + "terminal" {= "0.2.1"} + "tezos-rust-libs" {= "1.6"} + "tezos-sapling-parameters" {= "1.1.0"} + "tezt" {= "3.1.1"} + "time_now" {= "v0.15.0"} + "tls" {= "0.17.1"} + "tls-lwt" {= "0.17.1"} + "topkg" {= "1.0.7"} + "trie" {= "1.0.0"} + "uchar" {= "0.0.2"} + "uri" {= "4.2.0"} + "uri-sexp" {= "4.2.0"} + "utop" {= "2.10.0"} + "uucp" {= "15.0.0"} + "uuseg" {= "15.0.0"} + "uutf" {= "1.0.3"} + "vector" {= "1.0.0"} + "x509" {= "0.16.5"} + "yaml" {= "3.1.0"} + "yojson" {= "2.1.0"} + "zarith" {= "1.12"} + "zarith_stubs_js" {= "v0.16.0"} + "zed" {= "3.2.1"} +] +conflicts: [ + "checkseum" {= "0.5.0"} + "hacl_x25519" + "result" {< "1.5"} +] +dev-repo: "git+https://gitlab.com/tezos/tezos.git" \ No newline at end of file diff --git a/scripts/ci/build_full_unreleased.sh b/scripts/ci/build_full_unreleased.sh index 6934eba2dc51b8e106a791bb4e46ea3b8c0f0e9b..57122a86aa42ca6231b90684224944e89da4ff0f 100755 --- a/scripts/ci/build_full_unreleased.sh +++ b/scripts/ci/build_full_unreleased.sh @@ -11,9 +11,9 @@ old_protocol_store=$(mktemp -d) . ./scripts/version.sh # 1. Some basic, fast sanity checks -if [ "${build_deps_image_version}" != "${opam_repository_tag}" ] ; then +if [ "${build_deps_image_version}" != "${base_image_repository_tag}" ] ; then echo "Inconsistent dependencies hash between 'scripts/version.sh' and '.gitlab/ci/templates.yml'." ; - echo "${build_deps_image_version} != ${opam_repository_tag}" ; + echo "${build_deps_image_version} != ${base_image_repository_tag}" ; exit 1 ; fi diff --git a/scripts/create_docker_image.sh b/scripts/create_docker_image.sh index 69b11e98cd9d343a3fd75637c498299b88ed9a19..f33875372dfd85222b8a16718f6282ad573df1bb 100755 --- a/scripts/create_docker_image.sh +++ b/scripts/create_docker_image.sh @@ -11,7 +11,7 @@ cd "$src_dir" image_name="${1:-tezos-}" image_version="${2:-latest}" build_deps_image_name=${3:-registry.gitlab.com/tezos/opam-repository} -build_deps_image_version=${4:-$opam_repository_tag} +build_deps_image_version=${4:-$base_image_repository_tag} executables=${5:-$(cat script-inputs/released-executables)} commit_short_sha="${6:-$(git rev-parse --short HEAD)}" docker_target="${7:-without-evm-artifacts}" diff --git a/scripts/install_build_deps.raw.sh b/scripts/install_build_deps.raw.sh index dec44650775a8bce57d31ceeabe62997f036951b..a887b94aedca90141df1d08999439e22de4fac17 100755 --- a/scripts/install_build_deps.raw.sh +++ b/scripts/install_build_deps.raw.sh @@ -24,7 +24,7 @@ case $(opam --version) in * ) opam install --depext-only opam/virtual/octez-deps.opam ;; esac -opam install opam/virtual/octez-deps.opam --deps-only --criteria="-notuptodate,-changed,-removed" +opam install opam/virtual/octez-deps.opam.locked --deps-only if [ "$1" = "--tps" ]; then opam install caqti-driver-postgresql diff --git a/scripts/install_build_deps.sh b/scripts/install_build_deps.sh index cfc2ddf47abb5260c044095405fcb3017ce695b6..e2456268b1af8619f095b578a08e1fd5f3132807 100755 --- a/scripts/install_build_deps.sh +++ b/scripts/install_build_deps.sh @@ -102,5 +102,5 @@ if [ -n "$dev" ]; then # enough (for [ppx_yojson_conv_lib] in particular), so we add a # minimal bound to ensure it won’t be picked by opam. # utop is constrained to avoid reinstalling in all the times. - opam install --yes opam/virtual/octez-deps.opam opam/virtual/octez-dev-deps.opam --deps-only --criteria="-changed,-removed" + opam install --yes opam/virtual/octez-deps.opam.locked opam/virtual/octez-dev-deps.opam --deps-only --criteria="-changed,-removed" fi diff --git a/scripts/opam-check.sh b/scripts/opam-check.sh index 5db0d4e247b96d719b2c4773bd298c941f3c95de..7e3d744633cccb4b214bda33795c5a29a7a33d79 100755 --- a/scripts/opam-check.sh +++ b/scripts/opam-check.sh @@ -8,11 +8,12 @@ script_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")" echo "## Checking installed dependencies..." echo -if ! opam install opam/virtual/octez-deps.opam --deps-only --with-test --show-actions | grep "Nothing to do." > /dev/null 2>&1 ; then +# In opam 2.1, --show-actions could be replaced with --check. +if ! opam install opam/virtual/octez-deps.opam.locked --deps-only --with-test --show-actions | grep "Nothing to do." > /dev/null 2>&1 ; then echo echo 'Failure! Missing actions:' echo - opam install opam/virtual/octez-deps.opam --deps-only --with-test --show-actions + opam install opam/virtual/octez-deps.opam.locked --deps-only --with-test --show-actions echo # We really want literal backticks here, not command substitution. # shellcheck disable=SC2016 @@ -21,23 +22,4 @@ if ! opam install opam/virtual/octez-deps.opam --deps-only --with-test --show-ac exit 1 fi -# We really want literal backticks here, not command substitution. -# shellcheck disable=SC2016 -echo '## Running `./scripts/update_opam_repo.sh`' -echo -./scripts/update_opam_repo.sh || exit 1 - -if [ -n "$(cat opam_repo.patch)" ] ; then - - echo "##################################################" - cat opam_repo.patch - echo "##################################################" - - # We really want literal backticks here, not command substitution. - # shellcheck disable=SC2016 - echo 'Failed! The variables `opam_repository_tag` and `full_opam_repository_tag` are not synchronized. Please read the doc in `./scripts/update_opam_repo.sh` and act accordingly.' - echo - exit 1 -fi - echo "Ok." diff --git a/scripts/update_opam_repo.sh b/scripts/update_opam_repo.sh index da2015915d26b2cce10ab4d0b3cbc00a937aba1c..c54d148b933c45d22db644f85e5f27a204b3bf67 100755 --- a/scripts/update_opam_repo.sh +++ b/scripts/update_opam_repo.sh @@ -1,158 +1,77 @@ #!/bin/sh -# Update the repository of opam packages used by tezos. Tezos uses a -# private, shrunk down, opam repository to store all its -# dependencies. This is generated by the official opam repository -# (branch master) and then filtered using opam admin to include only -# the cone of tezos dependencies. This repository is then used to -# create the based opam image used by the CI to compile tezos and to -# generate the docker images. From time to time, when it is necessary -# to update a dependency, this repository should be manually -# refreshed. This script takes care of generating a patch for the -# private opam tezos repository. This patch must be applied manually -# w.r.t. the master branch. The procedure is as follows : +# Dependencies are listed in 'opam/virtual/octez-deps.opam.locked'. +# This lock file is used by 'make build-deps'. # -# 1. Update the variable `full_opam_repository_tag` in `version.sh` to -# a commit hash from the master branch of the official -# opam-repository. All the required packages will be extracted from -# this snapshot to the repo. +# The lock file must be copied to the root of the 'tezos/opam-repository' repository. +# The CI of 'tezos/opam-repository' builds Docker images that contain +# pre-built versions of the packages listed in the lock file. +# Those Docker images are used by the CI of 'tezos/tezos'. # -# 2. Run this script, it will generate a file `opam_repo.patch` +# To update the version of a dependency, or to add or remove a dependency, +# it is thus enough to update 'opam/virtual/octez-deps.opam.locked' and its copy +# in 'tezos/opam-repository'. # -# 3. Review the patch. +# To update 'opam/virtual/octez-deps.opam.locked', you can either modify the file +# manually (which is error-prone), or use 'opam lock'. +# 'opam lock' is built-in in opam 2.1. # -# 4. In the tezos opam-repository, create a new branch from master and -# apply this patch. Push the patch and create a merge request. A -# new docker image with all the prebuilt dependencies will be -# created by the CI. +# This script uses 'opam lock' to update 'opam/virtual/octez-deps.opam.locked' +# to update all dependencies. If you only want to update a single dependency, +# use 'opam install' to install this dependency, and then run: # -# 5. Update the variable `opam_repository_tag` in `scripts/version.sh` -# and the variable `build_deps_image_version` in `.gitlab-ci.yml` -# with the hash of the newly created commit in `tezos/opam-repository`. +# opam lock opam/virtual/octez-deps.opam opam/virtual/octez-dev-deps.opam +# mv octez-deps.opam.locked opam/virtual +# rm octez-dev-deps.opam.locked # -# 6. Enjoy your new dependencies +# The reason octez-dev-deps.opam should be passed to 'opam lock' is to ensure +# that the set of packages that are found are compatible with development dependencies. +# The lock file of octez-dev-deps.opam is not important in itself though. set -e -target="$(pwd)"/opam_repo.patch tmp_dir=$(mktemp -dt tezos_deps_opam.XXXXXXXX) - -cleanup () { - set +e - echo Cleaning up... - rm -rf "$tmp_dir" - rm -rf Dockerfile -} -trap cleanup EXIT INT - script_dir="$(cd "$(dirname "$0")" && echo "$(pwd -P)/")" src_dir="$(dirname "$script_dir")" +cd "$src_dir" -# shellcheck source=scripts/version.sh -. "$script_dir"/version.sh - -## Shallow clone of opam repository (requires git protocol version 2) -export GIT_WORK_TREE="$tmp_dir" -export GIT_DIR="$GIT_WORK_TREE/.git" -git init -git config --local protocol.version 2 -git remote add origin https://github.com/ocaml/opam-repository -git fetch --depth 1 origin "$full_opam_repository_tag" - -## Adding the various tezos packages - -mkdir -p "$tmp_dir"/packages/octez-deps/octez-deps.dev -cp opam/virtual/octez-deps.opam "$tmp_dir"/packages/octez-deps/octez-deps.dev/opam - -## Filtering unrequired packages -cd "$tmp_dir" -git reset --hard "$full_opam_repository_tag" - -## we add a dummy package that conflict with all "hidden" packages -dummy_pkg=dummy-tezos -dummy_path=packages/$dummy_pkg/$dummy_pkg.dev -dummy_opam=$dummy_path/opam -mkdir -p $dummy_path -echo 'opam-version: "2.0"' > $dummy_opam -# Opam doesn't seem to be deterministic when resolving constraints from mirage-crypto-pk -# (("mirage-no-solo5" & "mirage-no-xen") | "zarith-freestanding" | "mirage-runtime" {>= "4.0"}) -# - Sometime installing mirage-no-xen + mirage-no-solo5 -# - Sometime installing mirage-runtime -# According to mirage devs, mirage-runtime is the correct dependency to install. -# In addition "inotify" is a "{os = linux}" dependency that has to be -# in the repo for irmin to be installable on linux but is not selected -# by the solver. -echo "depends: [ \"ocaml\" { = \"$ocaml_version\" } \"mirage-runtime\" { >= \"4.0.0\" } \"inotify\" ]" >> $dummy_opam -echo 'conflicts:[' >> $dummy_opam -grep -r "^flags: *\[ *avoid-version *\]" -l ./ | LC_COLLATE=C sort -u | while read -r f; -do - f=$(dirname "$f") - f=$(basename "$f") - p=$(echo "$f" | cut -d '.' -f '1') - v=$(echo "$f" | cut -d '.' -f '2-') - echo "\"$p\" {= \"$v\"}" >> $dummy_opam -done -# FIXME: https://gitlab.com/tezos/tezos/-/issues/5832 -# opam unintentionally picks up a windows dependency. We add a -# conflict here to work around it. -echo '"ocamlbuild" {= "0.14.2+win" }' >> $dummy_opam -echo ']' >> $dummy_opam - -# Opam < 2.1 requires opam-depext as a plugin, later versions include it -# natively: -case $(opam --version) in - 2.0.* ) opam_depext_dep="opam-depext," ;; - * ) opam_depext_dep="" ;; -esac -#shellcheck disable=SC2086 -OPAMSOLVERTIMEOUT=600 opam admin filter --yes --resolve \ - octez-deps,ocaml,ocaml-base-compiler,odoc,${opam_depext_dep}ledgerwallet-tezos,caqti-driver-postgresql,js_of_ocaml-lwt,$dummy_pkg -## - ocaml-base-compiler has to be explicitely listed for the solver -## to not prefer the "variant" `system` of the compiler -## - odoc is used by the CI to generate the doc -## - ledgerwallet-tezos is an optional dependency of signer-services -## we want to have when building released binaries -## - caqti-driver-postgresq is needed by tps measurement software to -## read tezos-indexer databases -## - js_of_ocaml-lwt is an optional dependency of tezt which is needed -## to build tezt.js, and we do want to run some tests using nodejs +. scripts/version.sh -## Adding useful compiler variants -for variant in afl flambda fp ; do - git checkout packages/ocaml-option-$variant/ocaml-option-$variant.1 -done +echo "Running: 'opam repository set-url'" +opam repository set-url tezos --dont-select "$opam_repository" || \ + opam repository add tezos --dont-select "$opam_repository" > /dev/null 2>&1 -## Removing temporary hacks -rm -r "$tmp_dir"/packages/octez-deps -rm -r "$tmp_dir"/packages/$dummy_pkg - -## Generating the diff! -git remote add tezos $opam_repository_git -git fetch --depth 1 tezos "$opam_repository_tag" -git reset "$opam_repository_tag" - -## opam.2.1 will try to delete opam-depext, we should restore it. -if [ ! -d packages/opam-depext ]; then - git checkout HEAD -- packages/opam-depext +# Check if the default opam repo was set in this switch +default_switch= +if opam remote -s | grep -q default ; then + default_switch=yes fi -## Adding safer hashes -cp -rf packages packages.bak - -opam admin add-hashes sha256 sha512 - -(cd "$src_dir" && dune build src/tooling/opam-lint/opam_lint.exe) -for i in $(cd packages && find ./ -name opam); -do - "$src_dir/_build/default/src/tooling/opam-lint/opam_lint.exe" "packages/$i" "packages.bak/$i" -done -rm -rf packages.bak - -## -git add packages -git diff HEAD -- packages > "$target" +# Remove default directory (will put it back at the end). +echo "Running: 'opam repository remove default'" +opam repository remove default > /dev/null 2>&1 + +echo "Running: 'opam update'" +opam update +echo "Running: 'opam install'" +opam install --yes opam/virtual/octez-deps.opam opam/virtual/octez-dev-deps.opam --criteria=-not-up-to-date,+removed +echo "Running: 'opam lock'" +opam lock opam/virtual/octez-deps.opam opam/virtual/octez-dev-deps.opam +mv octez-deps.opam.locked opam/virtual +rm octez-dev-deps.opam.locked + +if [ -n "$default_switch" ]; then + echo "Running: 'opam repository add default'" + opam remote add default --rank=-1 > /dev/null 2>&1 || true +fi echo -echo "Wrote proposed update in: $target." -echo 'Please add this patch to: https://gitlab.com/tezos/opam-repository' -echo 'And update accordingly the commit hash in: .gitlab/ci/templates.yml and scripts/version.sh' +echo "You can now:" +echo +echo "- copy 'opam/virtual/octez-deps.opam.locked' to the root of" +echo " https://gitlab.com/tezos/opam-repository" +echo " and create a merge request for it." +echo +echo "- update the commit hash accordingly in:" +echo " - '.gitlab-ci.yml' (variable 'build_deps_image_version')" +echo " - 'scripts/version.sh' (variable 'base_image_repository_tag')" echo diff --git a/scripts/version.sh b/scripts/version.sh index 5bd5f9862345641ddc2384ed1db1c77ce66485dc..a7283bc1a039325e8acccd86c670503a9c2d6656 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -24,16 +24,10 @@ export recommended_node_version=16.18.1 # `image:`. export alpine_version='3.17' -## full_opam_repository is a commit hash of the public OPAM repository, i.e. -## https://github.com/ocaml/opam-repository -export full_opam_repository_tag=d091786ce64a52275ae4925b8eaca10460edebc4 - -## opam_repository is an additional, tezos-specific opam repository. +export opam_repository=https://github.com/ocaml/opam-repository.git\#d091786ce64a52275ae4925b8eaca10460edebc4 +export base_image_repository_url=https://gitlab.com/tezos/opam-repository ## This value MUST be the same as `build_deps_image_version` in `.gitlab-ci.yml` -export opam_repository_url=https://gitlab.com/tezos/opam-repository -export opam_repository_tag="${OPAM_REPOSITORY_TAG:-2703ff3a4dbbfa575a7a3d6dc2fff2a4b23c2045}" -export opam_repository_git="$opam_repository_url.git" -export opam_repository="$opam_repository_git"\#"$opam_repository_tag" +export base_image_repository_tag="3ac5259b93529e05d8431cb09ef8225fdfe6a152" ## Other variables, used both in Makefile and scripts export COVERAGE_OUTPUT=_coverage_output