diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a874e6bff891c0fa508c9586ce32b90308b1e3df..c6433bafb08b56ff2f193a9b8c2ef260b6d634d4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,219 +1,143 @@ ---- +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. -# General setup -default: - interruptible: true - -# Basic configuration to guard against double-pipelines workflow: - name: "[$PIPELINE_TYPE] $CI_COMMIT_TITLE" + name: '[$PIPELINE_TYPE] $CI_COMMIT_TITLE' rules: - # /!\ These rules should be mutually exclusive and kept in sync - # with the corresponding 'if'-rules on the pipeline includes - # below. - - # Allow 'Before merging' pipelines - - if: '$CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "merge_request_event"' - variables: - PIPELINE_TYPE: 'before_merging' - # Allow 'Latest release' pipelines - - if: '$CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "latest-release"' - variables: - PIPELINE_TYPE: 'latest_release' - # Allow 'Test latest release' pipelines for dry running latest - # release pipelines in the nomadic-labs/tezos CI. - - if: '$CI_PROJECT_NAMESPACE != "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "latest-release-test"' - variables: - PIPELINE_TYPE: 'latest_release_test' - # Allow 'Master branch' pipelines - - if: '$CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "master"' - variables: - PIPELINE_TYPE: 'master_branch' - # Allow 'Release tag' pipelines - - if: '$CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG =~ /^v\d+\.\d+(?:\-rc\d+)?$/' - variables: - PIPELINE_TYPE: 'release_tag' - # Allow 'Beta release tag' pipelines - - if: '$CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG =~ /^v\d+\.\d+\-beta\d*$/' - variables: - PIPELINE_TYPE: 'beta_release_tag' - # Allow 'Test release tag' pipelines for dry running release tag - # pipelines in the nomadic-labs/tezos CI. - - if: '$CI_PROJECT_NAMESPACE != "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG =~ /^v\d+\.\d+(?:\-(rc|beta)\d*)?$/' - variables: - PIPELINE_TYPE: 'release_tag_test' - # Allow 'Non-release tag' pipelines - - if: '$CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG != null && $CI_COMMIT_TAG !~ /^v\d+\.\d+(?:\-(rc|beta)\d*)?$/' - variables: - PIPELINE_TYPE: 'non_release_tag' - # Allow 'Test non-release tag' pipelines for dry running non-release tag - # pipelines in the nomadic-labs/tezos CI. - - if: '$CI_PROJECT_NAMESPACE != "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG != null && $CI_COMMIT_TAG !~ /^v\d+\.\d+(?:\-(rc|beta)\d*)?$/' - variables: - PIPELINE_TYPE: 'non_release_tag_test' - # Allow 'Scheduled pipeline for extended test' pipelines - - if: '$CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' - variables: - PIPELINE_TYPE: 'schedule_extended_test' - # Disallow all other pipelines - - when: never - + - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "merge_request_event" + variables: + PIPELINE_TYPE: before_merging + when: always + - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH + == "latest-release" + variables: + PIPELINE_TYPE: latest_release + when: always + - if: $CI_PROJECT_NAMESPACE != "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH + == "latest-release-test" + variables: + PIPELINE_TYPE: latest_release_test + when: always + - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH + == "master" + variables: + PIPELINE_TYPE: master_branch + when: always + - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG + =~ /^v\d+\.\d+(?:\-rc\d+)?$/ + variables: + PIPELINE_TYPE: release_tag + when: always + - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG + =~ /^v\d+\.\d+\-beta\d*$/ + variables: + PIPELINE_TYPE: beta_release_tag + when: always + - if: $CI_PROJECT_NAMESPACE != "tezos" && $CI_PIPELINE_SOURCE == "push" && ($CI_COMMIT_TAG + =~ /^v\d+\.\d+(?:\-rc\d+)?$/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+\-beta\d*$/) + variables: + PIPELINE_TYPE: release_tag_test + when: always + - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG + != null && $CI_COMMIT_TAG !~ /^v\d+\.\d+(?:\-rc\d+)?$/ && $CI_COMMIT_TAG !~ + /^v\d+\.\d+\-beta\d*$/ + variables: + PIPELINE_TYPE: non_release_tag + when: always + - if: $CI_PROJECT_NAMESPACE != "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG + != null && $CI_COMMIT_TAG !~ /^v\d+\.\d+(?:\-rc\d+)?$/ && $CI_COMMIT_TAG !~ + /^v\d+\.\d+\-beta\d*$/ + variables: + PIPELINE_TYPE: non_release_tag_test + when: always + - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS" + variables: + PIPELINE_TYPE: schedule_extended_test + when: always +default: + interruptible: true variables: - # /!\ CI_REGISTRY is overriden to use a private Docker registry mirror in AWS ECR - # in GitLab namespaces `nomadic-labs` and `tezos` - ## This value MUST be the same as `opam_repository_tag` in `scripts/version.sh` build_deps_image_version: 2e1cee010a93b56f8aface30e795e57cf6678bde - build_deps_image_name: "${CI_REGISTRY}/tezos/opam-repository" - # This image is not automatically mirrored to AWS ECR and should be - # fetch from the GitLab registry, so do not use CI_REGISTRY here. - rust_toolchain_image_name: "registry.gitlab.com/${CI_PROJECT_PATH}/rust-toolchain" - client_libs_dependencies_image_name: "registry.gitlab.com/${CI_PROJECT_PATH}/client-libs-dependencies" + build_deps_image_name: ${CI_REGISTRY}/tezos/opam-repository + rust_toolchain_image_name: registry.gitlab.com/${CI_PROJECT_PATH}/rust-toolchain + client_libs_dependencies_image_name: registry.gitlab.com/${CI_PROJECT_PATH}/client-libs-dependencies GIT_STRATEGY: fetch GIT_DEPTH: "1" GET_SOURCES_ATTEMPTS: "2" ARTIFACT_DOWNLOAD_ATTEMPTS: "2" - # Sets the number of tries before failing opam downloads. OPAMRETRIES: "5" - - # An addition to working around a bug in gitlab-runner's default - # unzipping implementation (https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27496), - # this setting cuts cache creation time. FF_USE_FASTZIP: "true" - # TODO: https://gitlab.com/tezos/tezos/-/issues/6764 - # "false" is the GitLab default but we've overridden it in the runner settings. - # This should be fixed at the runner level but we reset it to the - # default here in the meantime. - FF_KUBERNETES_HONOR_ENTRYPOINT: "false" - - # If `RUNTEZTALIAS` is true, then Tezt tests are included in the @runtest - # alias. We set it to false to deactivate these tests in the unit - # test jobs, as they already run in the Tezt jobs. It is set to true - # in the opam jobs where we want to run the tests `--with-test`. It is set - # to true in the `unit:js_component`, as there is not global Tezt job - # for js tests. RUNTEZTALIAS: "false" - -# Image templates -.image_template__runtime_e2etest_dependencies: - image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} - -.image_template__runtime_build_test_dependencies: - image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} - -.image_template__runtime_build_dependencies: - image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} - -.image_template__runtime_prebuild_dependencies: - image: ${build_deps_image_name}:runtime-prebuild-dependencies--${build_deps_image_version} - -.image_template__client_libs_dependencies: - image: ${client_libs_dependencies_image_name}:${client_libs_dependencies_image_tag} - -.image_template__rust_toolchain: - # Warning: we are relying on ill-specified behavior from GitLab that - # allows the expansion of dotenv variables (here: # $rust_toolchain_image_tag) - # in the image field (https://gitlab.com/gitlab-org/gitlab-runner/-/issues/37361). - image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} - -# Match GitLab executors version and directly use the Docker socket -# The Docker daemon is already configured, experimental features are enabled -# The following environment variables are already set: -# - BUILDKIT_PROGRESS -# - DOCKER_DRIVER -# - DOCKER_VERSION -# https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-socket-binding -.image_template__docker: - # https://gitlab.com/tezos/docker-images/ci-docker - image: "${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0" - -.image_template__alpine: - # The Alpine version should be kept up to date with the version used - # for the `build_deps_image_name` images and specified in the - # variable `alpine_version` in `scripts/version.sh`. This is checked - # by the jobs `trigger` and `sanity_ci`. - image: alpine:3.18 - -# The "manual" stage exists to fix a UI problem that occurs when mixing -# manual and non-manual jobs. + FF_KUBERNETES_HONOR_ENTRYPOINT: "false" stages: - - trigger - - sanity - - build - - test - - test_coverage - - packaging - - doc - - prepare_release - - publish_release_gitlab - - publish_release - - publish_package_gitlab - - manual - -# Dummy job. -# -# This fixes the "configuration must contain at least one -# visible job" error in GitLab when using includes. -# -# For more info, see: https://gitlab.com/gitlab-org/gitlab/-/issues/341693 +- trigger +- sanity +- build +- test +- test_coverage +- packaging +- doc +- prepare_release +- publish_release_gitlab +- publish_release +- publish_package_gitlab +- manual dummy_job: rules: - - if: '"foo" == "bar"' + - if: $foo == "bar" && $foo != "bar" + when: on_success script: - - 'echo "This job will never execute"' - + - echo "This job will never execute" include: - # /!\ These rules should be be mutually exclusive and kept in sync - # with the corresponding 'if'-rules on the workflow rules above. - - # Common templates - - local: .gitlab/ci/jobs/shared/templates.yml - - # Before merging - - local: .gitlab/ci/pipelines/before_merging.yml - rules: - - if: '$CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "merge_request_event"' - - # Latest release - - local: .gitlab/ci/pipelines/latest_release.yml - rules: - - if: '$CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "latest-release"' - - # Test latest release - - local: .gitlab/ci/pipelines/latest_release_test.yml - rules: - - if: '$CI_PROJECT_NAMESPACE != "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "latest-release-test"' - - # Master branch - - local: .gitlab/ci/pipelines/master_branch.yml - rules: - - if: '$CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "master"' - - # Release tag - - local: .gitlab/ci/pipelines/release_tag.yml - rules: - - if: '$CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG =~ /^v\d+\.\d+(?:\-rc\d+)?$/' - - # Beta release tag - - local: .gitlab/ci/pipelines/beta_release_tag.yml - rules: - - if: '$CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG =~ /^v\d+\.\d+\-beta\d*$/' - - # Test release tag - - local: .gitlab/ci/pipelines/release_tag_test.yml - rules: - - if: '$CI_PROJECT_NAMESPACE != "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG =~ /^v\d+\.\d+(?:\-(rc|beta)\d*)?$/' - - # Non-release tag - - local: .gitlab/ci/pipelines/non_release_tag.yml - rules: - - if: '$CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG != null && $CI_COMMIT_TAG !~ /^v\d+\.\d+(?:\-(rc|beta)\d*)?$/' - - # Test non-release tag - - local: .gitlab/ci/pipelines/non_release_tag_test.yml - rules: - - if: '$CI_PROJECT_NAMESPACE != "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG != null && $CI_COMMIT_TAG !~ /^v\d+\.\d+(?:\-(rc|beta)\d*)?$/' - - # Scheduled pipeline for extended test - - local: .gitlab/ci/pipelines/schedule_extended_test.yml - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' +- .gitlab/ci/jobs/shared/images.yml +- .gitlab/ci/jobs/shared/templates.yml +- local: .gitlab/ci/pipelines/before_merging.yml + rules: + - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "merge_request_event" + when: always +- local: .gitlab/ci/pipelines/latest_release.yml + rules: + - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH + == "latest-release" + when: always +- local: .gitlab/ci/pipelines/latest_release_test.yml + rules: + - if: $CI_PROJECT_NAMESPACE != "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH + == "latest-release-test" + when: always +- local: .gitlab/ci/pipelines/master_branch.yml + rules: + - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH + == "master" + when: always +- local: .gitlab/ci/pipelines/release_tag.yml + rules: + - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG + =~ /^v\d+\.\d+(?:\-rc\d+)?$/ + when: always +- local: .gitlab/ci/pipelines/beta_release_tag.yml + rules: + - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG + =~ /^v\d+\.\d+\-beta\d*$/ + when: always +- local: .gitlab/ci/pipelines/release_tag_test.yml + rules: + - if: $CI_PROJECT_NAMESPACE != "tezos" && $CI_PIPELINE_SOURCE == "push" && ($CI_COMMIT_TAG + =~ /^v\d+\.\d+(?:\-rc\d+)?$/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+\-beta\d*$/) + when: always +- local: .gitlab/ci/pipelines/non_release_tag.yml + rules: + - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG + != null && $CI_COMMIT_TAG !~ /^v\d+\.\d+(?:\-rc\d+)?$/ && $CI_COMMIT_TAG !~ + /^v\d+\.\d+\-beta\d*$/ + when: always +- local: .gitlab/ci/pipelines/non_release_tag_test.yml + rules: + - if: $CI_PROJECT_NAMESPACE != "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG + != null && $CI_COMMIT_TAG !~ /^v\d+\.\d+(?:\-rc\d+)?$/ && $CI_COMMIT_TAG !~ + /^v\d+\.\d+\-beta\d*$/ + when: always +- local: .gitlab/ci/pipelines/schedule_extended_test.yml + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS" + when: always diff --git a/.gitlab/ci/jobs/sanity/sanity_ci.yml b/.gitlab/ci/jobs/sanity/sanity_ci.yml index 3e95741b43519d5475bbc721fdbf18760dd3f4a6..1953d5c94cd1a441cb9235f36f98fb74f7cec2f8 100644 --- a/.gitlab/ci/jobs/sanity/sanity_ci.yml +++ b/.gitlab/ci/jobs/sanity/sanity_ci.yml @@ -1,7 +1,7 @@ sanity_ci: extends: - .default_settings_template - - .image_template__runtime_prebuild_dependencies + - .image_template__runtime_build_dependencies stage: sanity before_script: - ./scripts/ci/take_ownership.sh @@ -12,3 +12,5 @@ sanity_ci: # Check that the opam-repo images' Alpine version corresponds to # the value in scripts/version.sh. - ./scripts/ci/check_alpine_version.sh + # Check that .gitlab-ci.yml is up to date + - make -C ci check diff --git a/.gitlab/ci/jobs/shared/images.yml b/.gitlab/ci/jobs/shared/images.yml new file mode 100644 index 0000000000000000000000000000000000000000..db81563733e15ce853a9b5dd6552987899af3edc --- /dev/null +++ b/.gitlab/ci/jobs/shared/images.yml @@ -0,0 +1,19 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +.image_template__alpine: + image: alpine:3.18 +.image_template__client_libs_dependencies: + image: ${client_libs_dependencies_image_name}:${client_libs_dependencies_image_tag} +.image_template__docker: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 +.image_template__runtime_build_dependencies: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} +.image_template__runtime_build_test_dependencies: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} +.image_template__runtime_e2etest_dependencies: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} +.image_template__runtime_prebuild_dependencies: + image: ${build_deps_image_name}:runtime-prebuild-dependencies--${build_deps_image_version} +.image_template__rust_toolchain: + image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} diff --git a/ci/README.md b/ci/README.md index 3b98c6b6bf9855b34b636a67693a3924adabd89f..621b8f22c115b0dce0c0a556a8cc2f2062b285d2 100644 --- a/ci/README.md +++ b/ci/README.md @@ -8,5 +8,15 @@ This directory is structured like this: - `lib_gitlab_ci`: contains a partial, slightly opiniated, AST of [GitLab CI/CD YAML syntax](https://docs.gitlab.com/ee/ci/yaml/). - `bin`: contains a set of helpers for creating the Octez-specific - GitLab CI configuration files and the skeleton of an executable that will be used for - for writing `.gitlab-ci.yml` using those helpers. + GitLab CI configuration files and an executable that generates part + of the CI configuration using those helpers. + +## Usage + +To regenerate `.gitlab-ci.yml` (from the root of the repo): + + make -C ci all + +To check that `.gitlab-ci.yml` is up-to-date (from the root of the repo): + + make -C ci check diff --git a/ci/bin/main.ml b/ci/bin/main.ml index e7d4dc6a488d426b26f45c4cf91db795bdc395e4..53adfc8486121817db0de3cb3137b64f77bbb4ec 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -7,9 +7,269 @@ (* Main entrypoint of CI-in-OCaml. - Currently does nothing. - - Here we will register the set of pipelines, stages and images used to + Here we register the set of pipelines, stages and images and generate the top-level [.gitlab-ci.yml] file. *) -let () = () +open Gitlab_ci +open Gitlab_ci.Types +open Gitlab_ci.Util +open Tezos_ci + +(* Sets up the [default:] top-level configuration element. *) +let default = default ~interruptible:true () + +(* Define [stages:] + + The "manual" stage exists to fix a UI problem that occurs when mixing + manual and non-manual jobs. *) +module Stages = struct + let _trigger = Stage.register "trigger" + + let _sanity = Stage.register "sanity" + + let _build = Stage.register "build" + + let _test = Stage.register "test" + + let _test_coverage = Stage.register "test_coverage" + + let _packaging = Stage.register "packaging" + + let _doc = Stage.register "doc" + + let _prepare_release = Stage.register "prepare_release" + + let _publish_release_gitlab = Stage.register "publish_release_gitlab" + + let _publish_release = Stage.register "publish_release" + + let _publish_package_gitlab = Stage.register "publish_package_gitlab" + + let _manual = Stage.register "manual" +end + +(* Get the [build_deps_image_version] from the environment, which is + typically set by sourcing [scripts/version.sh]. This is used to write + [build_deps_image_version] in the top-level [variables:], used to + specify the versions of the [build_deps] images. *) +let build_deps_image_version = + match Sys.getenv_opt "opam_repository_tag" with + | None -> + failwith + "Please set the environment variable [opam_repository_tag], by e.g. \ + sourcing [scripts/version.sh] before running." + | Some v -> v + +(* Get the [alpine_version] from the environment, which is typically + set by sourcing [scripts/version.sh]. This is used to set the tag + of the image {!Images.alpine}. *) +let alpine_version = + match Sys.getenv_opt "alpine_version" with + | None -> + failwith + "Please set the environment variable [alpine_version], by e.g. \ + sourcing [scripts/version.sh] before running." + | Some v -> v + +(* Top-level [variables:] *) +let variables : variables = + [ + (* /!\ CI_REGISTRY is overriden to use a private Docker registry mirror in AWS ECR + in GitLab namespaces `nomadic-labs` and `tezos` + /!\ This value MUST be the same as `opam_repository_tag` in `scripts/version.sh` *) + ("build_deps_image_version", build_deps_image_version); + ("build_deps_image_name", "${CI_REGISTRY}/tezos/opam-repository"); + ( "rust_toolchain_image_name", + "registry.gitlab.com/${CI_PROJECT_PATH}/rust-toolchain" ); + ( "client_libs_dependencies_image_name", + "registry.gitlab.com/${CI_PROJECT_PATH}/client-libs-dependencies" ); + ("GIT_STRATEGY", "fetch"); + ("GIT_DEPTH", "1"); + ("GET_SOURCES_ATTEMPTS", "2"); + ("ARTIFACT_DOWNLOAD_ATTEMPTS", "2"); + (* Sets the number of tries before failing opam downloads. *) + ("OPAMRETRIES", "5"); + (* An addition to working around a bug in gitlab-runner's default + unzipping implementation + (https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27496), + this setting cuts cache creation time. *) + ("FF_USE_FASTZIP", "true"); + (* If RUNTEZTALIAS is true, then Tezt tests are included in the + @runtest alias. We set it to false to deactivate these tests in + the unit test jobs, as they already run in the Tezt jobs. It is + set to true in the opam jobs where we want to run the tests + --with-test. *) + ("RUNTEZTALIAS", "false"); + (* TODO: https://gitlab.com/tezos/tezos/-/issues/6764 + "false" is the GitLab default but we've overridden it in the runner settings. + This should be fixed at the runner level but we reset it to the + default here in the meantime. *) + ("FF_KUBERNETES_HONOR_ENTRYPOINT", "false"); + ] + +(* Register images. + + The set of registered images are written to + [.gitlab/ci/jobs/shared/images.yml] for interoperability with + hand-written .yml files. + + For documentation on the [runtime_X_dependencies] and the + [rust_toolchain] images, refer to + {{:https://gitlab.com/tezos/opam-repository/} + tezos/opam-repository}. *) +module Images = struct + let _runtime_e2etest_dependencies = + Image.register + ~name:"runtime_e2etest_dependencies" + ~image_path: + "${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version}" + + let _runtime_build_test_dependencies = + Image.register + ~name:"runtime_build_test_dependencies" + ~image_path: + "${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version}" + + let _runtime_build_dependencies = + Image.register + ~name:"runtime_build_dependencies" + ~image_path: + "${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version}" + + let _runtime_prebuild_dependencies = + Image.register + ~name:"runtime_prebuild_dependencies" + ~image_path: + "${build_deps_image_name}:runtime-prebuild-dependencies--${build_deps_image_version}" + + let _client_libs_dependencies = + Image.register + ~name:"client_libs_dependencies" + ~image_path: + "${client_libs_dependencies_image_name}:${client_libs_dependencies_image_tag}" + + let _rust_toolchain = + (* Warning: we are relying on ill-specified behavior from GitLab that allows + the expansion of dotenv variables (here: $rust_toolchain_image_tag) in + the image field. + See: https://gitlab.com/gitlab-org/gitlab-runner/-/issues/37361. *) + Image.register + ~name:"rust_toolchain" + ~image_path:"${rust_toolchain_image_name}:${rust_toolchain_image_tag}" + + (* Match GitLab executors version and directly use the Docker socket + The Docker daemon is already configured, experimental features are enabled + The following environment variables are already set: + - [BUILDKIT_PROGRESS] + - [DOCKER_DRIVER] + - [DOCKER_VERSION] + For more info, see {{:https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-socket-binding}} here. + + This image is defined in {{:https://gitlab.com/tezos/docker-images/ci-docker}tezos/docker-images/ci-docker}. *) + let _docker = + Image.register + ~name:"docker" + ~image_path:"${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0" + + (* The Alpine version should be kept up to date with the version + used for the [build_deps_image_name] images and specified in the + variable [alpine_version] in [scripts/version.sh]. This is + checked by the jobs [trigger] and [sanity_ci]. *) + let _alpine = + Image.register ~name:"alpine" ~image_path:("alpine:" ^ alpine_version) +end + +(* Dummy job. + + This fixes the "configuration must contain at least one + visible job" error in GitLab when using includes. + + For more info, see: https://gitlab.com/gitlab-org/gitlab/-/issues/341693 *) +let job_dummy : job = + Util.job + ~rules: + [job_rule ~if_:If.(var "foo" == str "bar" && var "foo" != str "bar") ()] + ~name:"dummy_job" + ~script:[{|echo "This job will never execute"|}] + () + +(* Register pipelines types. Pipelines types are used to generate + workflow rules and includes of the files where the jobs of the + pipeline is defined. At the moment, all these pipelines are defined + manually in .yml, but will eventually be generated. *) +let () = + (* Matches release tags, e.g. [v1.2.3] or [v1.2.3-rc4]. *) + let release_tag_re = "/^v\\d+\\.\\d+(?:\\-rc\\d+)?$/" in + (* Matches beta release tags, e.g. [v1.2.3-beta5]. *) + let beta_release_tag_re = "/^v\\d+\\.\\d+\\-beta\\d*$/" in + let open Rules in + let open Pipeline in + (* Matches either release tags or beta release tags, e.g. [v1.2.3], + [v1.2.3-rc4] or [v1.2.3-beta5]. *) + let has_any_release_tag = + If.(has_tag_match release_tag_re || has_tag_match beta_release_tag_re) + in + let has_non_release_tag = + If.(Predefined_vars.ci_commit_tag != null && not has_any_release_tag) + in + register "before_merging" If.(on_tezos_namespace && merge_request) ; + register + "latest_release" + If.(on_tezos_namespace && push && on_branch "latest-release") ; + register + "latest_release_test" + If.(not_on_tezos_namespace && push && on_branch "latest-release-test") ; + register "master_branch" If.(on_tezos_namespace && push && on_branch "master") ; + register + "release_tag" + If.(on_tezos_namespace && push && has_tag_match release_tag_re) ; + register + "beta_release_tag" + If.(on_tezos_namespace && push && has_tag_match beta_release_tag_re) ; + register + "release_tag_test" + If.(not_on_tezos_namespace && push && has_any_release_tag) ; + register + "non_release_tag" + If.(on_tezos_namespace && push && has_non_release_tag) ; + register + "non_release_tag_test" + If.(not_on_tezos_namespace && push && has_non_release_tag) ; + register "schedule_extended_test" schedule_extended_tests + +(* Split pipelines and writes image templates *) +let config = + (* Split pipelines types into workflow and includes *) + let workflow, includes = Pipeline.workflow_includes () in + (* Write image templates. + + This is a temporary stop-gap and only necessary for jobs that are + not define in OCaml. Once all jobs have been migrated, this can + be removed. *) + let image_templates_include = + let filename = ".gitlab/ci/jobs/shared/images.yml" in + let image_template (name, image_path) : string * Yaml.value = + let name = ".image_template__" ^ name in + (name, `O [("image", `String (Image.name image_path))]) + in + let config : Yaml.value = `O (List.map image_template (Image.all ())) in + Base.write_yaml ~header:Tezos_ci.header filename config ; + {local = filename; rules = []} + in + let includes = + image_templates_include + :: {local = ".gitlab/ci/jobs/shared/templates.yml"; rules = []} + :: includes + in + [ + Workflow workflow; + Default default; + Variables variables; + Stages (Stage.to_string_list ()); + Job job_dummy; + Include includes; + ] + +let () = + let filename = Base.(project_root // ".gitlab-ci.yml") in + To_yaml.to_file ~header:Tezos_ci.header ~filename config diff --git a/ci/bin/rules.ml b/ci/bin/rules.ml index 5b85c8cb6da8a74e6ba72362523312e6aae078fb..f0a004d363d2677df83fd708203f93e1ada16d70 100644 --- a/ci/bin/rules.ml +++ b/ci/bin/rules.ml @@ -40,9 +40,6 @@ let not_on_tezos_namespace = Predefined_vars.ci_project_namespace != str "tezos" let has_tag_match tag = Predefined_vars.ci_commit_tag =~ tag -let has_tag_not_match tag = - Predefined_vars.(ci_commit_tag != null && ci_commit_tag =~! tag) - let assigned_to_marge_bot = Predefined_vars.ci_merge_request_assignees =~ "/nomadic-margebot/" diff --git a/ci/bin/rules.mli b/ci/bin/rules.mli index 344aa4dd0e326095d4023eebcad4e9655d10aafd..6093c88724b27fd2cb654756786467a42294f243 100644 --- a/ci/bin/rules.mli +++ b/ci/bin/rules.mli @@ -40,12 +40,9 @@ val on_tezos_namespace : If.t (** A rule that is true if [CI_PROJECT_NAMESPACE] is not [tezos]. *) val not_on_tezos_namespace : If.t -(** A rule that is true if [CI_COMMIT_TAG] is defined and matches the given regexp. *) +(** A rule that is true if [CI_COMMIT_TAG] matches the given regexp. *) val has_tag_match : string -> If.t -(** A rule that is true if [CI_COMMIT_TAG] is defined but does not matches the given regexp. *) -val has_tag_not_match : string -> If.t - (** A rule that is true if the comma-separated list [CI_MERGE_REQUEST_LABELS] contains a given label. *) val has_mr_label : string -> If.t diff --git a/ci/lib_gitlab_ci/if.ml b/ci/lib_gitlab_ci/if.ml index 5ca99e71df2a654bbc77245ddbf7fd979f27372f..fad1596bcd0702a55fee7e2b157f31ab6427ea16 100644 --- a/ci/lib_gitlab_ci/if.ml +++ b/ci/lib_gitlab_ci/if.ml @@ -100,3 +100,11 @@ let ( || ) = or_ let ( =~ ) = match_ let ( =~! ) = unmatch + +let rec not = function + | And (a, b) -> Or (not a, not b) + | Or (a, b) -> And (not a, not b) + | Eq (x, b) -> Neq (x, b) + | Neq (x, b) -> Eq (x, b) + | Match (x, s) -> Unmatch (x, s) + | Unmatch (x, s) -> Match (x, s) diff --git a/ci/lib_gitlab_ci/if.mli b/ci/lib_gitlab_ci/if.mli index f4ecc535fd3cf56d240c2d3d41989a12dea7720e..3446561cc4569733861a909fb84e455124f66f69 100644 --- a/ci/lib_gitlab_ci/if.mli +++ b/ci/lib_gitlab_ci/if.mli @@ -71,3 +71,12 @@ val ( =~ ) : Var.t -> string -> t Example: [var "foo" =~! str "/bar/"] translates to [$foo !~ "/bar/"]. *) val ( =~! ) : Var.t -> string -> t + +(** Negation of a predicate. + + If [t] evaluates to true, then [not t] evaluates to false. Note + that [if:] expressions have no native negation operator. Therefore + this function works by rewriting the expression using de Morgan's + laws and swapping (negated) operators for their (un)negated + counter-parts. *) +val not : t -> t diff --git a/ci/makefile b/ci/makefile new file mode 100644 index 0000000000000000000000000000000000000000..b8675789b46a5b163bf941fbd4efdfd7268530b2 --- /dev/null +++ b/ci/makefile @@ -0,0 +1,16 @@ +.PHONY: all +all: + (cd .. && . ./scripts/version.sh && dune exec ci/bin/main.exe) + +# Used in the CI to verify that [.gitlab-ci.yml] is up to date. +.PHONY: check +check: + @git diff --exit-code HEAD -- ../.gitlab-ci.yml || (echo "Cannot check generated [.gitlab-ci.yml] file, some changes are uncommitted"; exit 1) + @$(MAKE) all + @git diff --exit-code HEAD -- ../.gitlab-ci.yml > /dev/null || ( \ + echo "Repository not clean after 'make -C ci'."; \ + echo "You should not edit generated GitLab CI .yml files directly."; \ + echo "Edit ci/bin/main.ml instead."; \ + echo "Then run 'make -C ci' and commit the difference."; \ + exit 1 \ + )