diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1c94fa156776939994656906f55e076a4318298a..04c25a3747760ad71cc2595e2ae585c0295316e5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,25 +8,69 @@ default: workflow: rules: # This workflow entry allows pipelines for schedules events in which case it checks the - # TZ_SCHEDULE_KIND which must be set in the pipeline scheduling interface - # In either case it sets the TZ_PIPELINE_KIND variable to allow jobs to - # selectively run on some pipelines but not all. + # CI_PIPELINE_SOURCE which is automatically set by Gitlab. # See https://docs.gitlab.com/ee/ci/yaml/README.html#workflow for additional # details. - if: '$CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' - variables: - TZ_PIPELINE_KIND: "SCHEDULE" when: always # Switch between branch pipelines and merge request pipelines. # https://docs.gitlab.com/13.12/ee/ci/yaml/README.html#switch-between-branch-pipelines-and-merge-request-pipelines - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS' when: never + # No branch pipelines outside of the tezos namespace, except for testing releases. + # The goal is to avoid triggering two pipelines for each MR: + # - one when pushing the branch, before the MR exists; + # - and one when the MR is created. + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != "latest-release-test" && $CI_PROJECT_NAMESPACE != "tezos"' + when: never # This workflow entry allows pipelines for push events (including force-push, push of fixup commits, rebase, etc.) - if: '$CI_PIPELINE_SOURCE == "push"' when: always - when: never # default +variables: + # /!\ CI_REGISTRY is overriden to use a private Docker registry mirror in AWS ECR + # in GitLab namespaces `nomadic-labs` and `tezos` + ## This value MUST be the same as `opam_repository_tag` in `scripts/version.sh` + build_deps_image_version: ba1ace10fe5b100b54bc51ab67e7afd1967a4e5f + build_deps_image_name: "${CI_REGISTRY}/tezos/opam-repository" + GIT_STRATEGY: fetch + GIT_DEPTH: "1" + GET_SOURCES_ATTEMPTS: "2" + ARTIFACT_DOWNLOAD_ATTEMPTS: "2" + # Sets the number of tries before failing opam downloads. + OPAMRETRIES: "5" + + # An addition to working around a bug in gitlab-runner's default + # unzipping implementation (https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27496), + # this setting cuts cache creation time. + FF_USE_FASTZIP: "true" + +# Image templates +.image_template__runtime_build_test_dependencies_template: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + +.image_template__runtime_build_dependencies_template: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + +.image_template__runtime_prebuild_dependencies_template: + image: ${build_deps_image_name}:runtime-prebuild-dependencies--${build_deps_image_version} + +# Match GitLab executors version and directly use the Docker socket +# The Docker daemon is already configured, experimental features are enabled +# The following environment variables are already set: +# - BUILDKIT_PROGRESS +# - DOCKER_DRIVER +# - DOCKER_VERSION +# https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-socket-binding +.image_template__docker: + # https://gitlab.com/tezos/docker-images/ci-docker + image: "${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.6.0" + +.image_template__alpine: + image: alpine:3.15 + # The "manual" stage exists to fix a UI problem that occurs when mixing # manual and non-manual jobs. stages: @@ -42,48 +86,63 @@ stages: - publish_release - manual -# All the jobs belonging to a stage X should be in -# .gitlab/ci/X.yml. The stage "manual" is an exception to this rule, -# as jobs of that stage are not logically related. +# Trigger +# +# §1: The purpose of this job is to launch the CI manually in certain cases. +# The objective is not to run computing when it is not +# necessary and the decision to do so belongs to the developer +# +# §2: Gitlab CI needs at least one job definition, otherwise we're stuck with +# this error: 'Jobs config should contain at least one visible job' +trigger: + extends: + - .image_template__runtime_build_test_dependencies_template + stage: trigger + rules: + - if: '($CI_PIPELINE_SOURCE == "merge_request_event") && ($CI_PROJECT_NAMESPACE == "tezos") && ($CI_MERGE_REQUEST_ASSIGNEES !~ /nomadic-margebot/)' + when: manual + - when: always + allow_failure: false + timeout: "10m" + script: + - echo 'Trigger pipeline 🤠' -# If the configuration for stage X is long, it can be broken up in -# subfiles .gitlab/ci/X/a.yml, .gitlab/ci/X/b.yml, etc (see e.g. the -# stage test). +include: -# Finally, templates shall not be defined in one stage file and used -# in another. That is, a template defined in test.yml shall not be -# used in doc.yml (but it's fine to use it in a subfile test/X.yml). + # Common templates + - local: .gitlab/ci/jobs/shared/templates.yml -include: - - .gitlab/ci/templates.yml - - # Actual jobs are defined in these included files - - - .gitlab/ci/trigger.yml - - .gitlab/ci/packaging.yml - - .gitlab/ci/opam-ci.yml - - .gitlab/ci/sanity.yml - - .gitlab/ci/build.yml - - .gitlab/ci/test/templates.yml - - .gitlab/ci/test/lints.yml - - .gitlab/ci/test/unit.yml - - .gitlab/ci/test/integration.yml - - .gitlab/ci/test/liquidity-baking-scripts-integrity.yml - - .gitlab/ci/test/tezt.yml - - .gitlab/ci/doc.yml - - .gitlab/ci/doc/test-scripts.yml - - .gitlab/ci/build_release.yml - - .gitlab/ci/publish_release.yml - - # The job `unified_coverage` must have the same name on merge requests and the default branch - # /!\ Limitations of using rules with include: - # https://docs.gitlab.com/ee/ci/yaml/includes.html#use-variables-with-include - # Only run on merge requests that do not have the label `ci--no-coverage` - - local: .gitlab/ci/test_coverage.yml + # Before merging + - local: .gitlab/ci/pipelines/before_merging.yml rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS !~ /(?:^|[,])ci--no-coverage(?:$|[,])/' - # Only run on default branch - - local: .gitlab/ci/test_coverage_default.yml + - if: '($CI_PIPELINE_SOURCE == "merge_request_event") && ($CI_PROJECT_NAMESPACE == "tezos")' + + # Latest release and test latest release + - local: .gitlab/ci/pipelines/latest_release.yml + rules: + - if: '($CI_COMMIT_BRANCH == "latest-release") && ($CI_PROJECT_NAMESPACE == "tezos")' + + # Test latest release + - local: .gitlab/ci/pipelines/latest_release_test.yml + rules: + - if: '($CI_COMMIT_BRANCH == "latest-release-test") && ($CI_PROJECT_NAMESPACE != "tezos")' + + # Master branch + - local: .gitlab/ci/pipelines/master_branch.yml + rules: + - if: '($CI_PROJECT_NAMESPACE == "tezos") && ($CI_COMMIT_BRANCH == "master")' + + # Release tag + - local: .gitlab/ci/pipelines/release_tag.yml + rules: + - if: '($CI_COMMIT_TAG =~ /^v\d+\.\d+(?:\-rc\d+)?$/) && ($CI_PROJECT_NAMESPACE == "tezos")' + + # Test release tag + - local: .gitlab/ci/pipelines/release_tag_test.yml + rules: + - if: '($CI_COMMIT_TAG =~ /^v\d+\.\d+(?:\-rc\d+)?$/) && ($CI_PROJECT_NAMESPACE != "tezos")' + + # Scheduled pipeline for extended tests + - local: .gitlab/ci/pipelines/schedule_extended_test.yml rules: - # Cannot use TEZOS_DEFAULT_NAMESPACE here, see limitations above - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + - if: '($CI_PIPELINE_SOURCE == "schedule") && ($TZ_SCHEDULE_KIND == "EXTENDED_TESTS")' diff --git a/.gitlab/ci/Makefile b/.gitlab/ci/Makefile index 7a32cee03ff69d5c83e7299727cc34d92378a11c..1bc0826ce8959ee5b4badc478d272a87cd40b106 100644 --- a/.gitlab/ci/Makefile +++ b/.gitlab/ci/Makefile @@ -5,8 +5,8 @@ endif last_proto_name := $(shell find ${CI_PROJECT_DIR}/src -name "proto_[0-9][0-9][0-9]_*" | awk -F'/' '{print $$NF}' | sort -r | head -1) last_proto_version := $(shell echo ${last_proto_name} | cut -d'_' -f2 | sed 's/^0*//') -$(eval new_proto_version=$(shell echo $$(($(last_proto_version)+1)))) -new_proto_version := $(shell if [ "${new_proto_version}" -lt "100" ]; then echo 0${new_proto_version}; else echo ${new_proto_version}; fi) +new_proto_version_int:=$(shell echo $$(($(last_proto_version)+1))) +new_proto_version := $(shell if [ "${new_proto_version_int}" -lt "100" ]; then echo 0${new_proto_version_int}; else echo ${new_proto_version_int}; fi) .PHONY: test-protocol-compiles test-protocol-compiles: diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml deleted file mode 100644 index 46b35a6ac3166421525661123ce2e77b5e5b3e0a..0000000000000000000000000000000000000000 --- a/.gitlab/ci/build.yml +++ /dev/null @@ -1,97 +0,0 @@ ---- - -.build: - extends: .build_template - variables: - ARCH: "" - # We fix the value of GIT_{SHORTREF,DATETIME,VERSION} (these are - # read by src/lib_version and output by the binaries `--version` - # option). Fixing these values on development builds improves - # cache usage. - GIT_SHORTREF: "00000000" - GIT_DATETIME: "1970-01-01 00:00:00 +0000%" - GIT_VERSION: "dev" - script: - - make -C ${CI_PROJECT_DIR}/.gitlab/ci build-full-unreleased - cache: - key: "$CI_COMMIT_REF_SLUG" - paths: - - _build - artifacts: - name: "build-$ARCH-$CI_COMMIT_REF_SLUG" - paths: - - octez-* - - tezos-* - - src/proto_*/parameters/*.json - - _build/default/src/lib_protocol_compiler/bin/main_native.exe - expire_in: 1 day - when: on_success - -build_x86_64: - extends: - - .build - - .rules_template__development_and_extended - # This job is the main bottleneck since a lot of tests depend on it. - # So we start it even before sanity_ci. - - .needs_template__wait_for_trigger_job - variables: - ARCH: "x86_64" - -build_arm64: - extends: - - .build - - .rules_template__development_arm64 - variables: - ARCH: "arm64" - # Disable coverage for arm64 - COVERAGE_OPTIONS: "" - tags: - - arm64 - -build_get_contracts: - extends: - - .build_template - - .rules_template__development - script: - - dune build @devtools/get_contracts/check - -# similar to the build template above, this template -# compiles all binaries using the static profile. -.build_static_binaries_template: - extends: - - .default_settings_template - - .image_template__runtime_build_dependencies_template - stage: build - variables: - ARCH: "" - script: - - make -C ${CI_PROJECT_DIR}/.gitlab/ci build-static-binaries - - artifacts: - paths: - - "octez-binaries/$ARCH/*" - -# x86_64 static binaries are necessary for development branches, as we want to make -# sure the build does not break. We also need it for release tags, to as its artifacts -# are published in the stage `publish_release`. For this reason, we build it in all pipelines, -# except on latest release branches. -build:static-x86_64-linux-binaries: - extends: - - .build_static_binaries_template - - .rules_template__not_on_latest_release - # Even though not many tests depend on static executables, some of those that do - # are limiting factors in the total duration of pipelines. - # So we start this job as early as possible, without waiting for sanity_ci. - - .needs_template__wait_for_trigger_job - variables: - ARCH: "x86_64" - -# arm64 static binaries are only built for master and releases. -build:static-arm64-linux-binaries: - extends: - - .build_static_binaries_template - - .rules_template__master_and_releases - variables: - ARCH: "arm64" - tags: - - arm64 diff --git a/.gitlab/ci/build_release.yml b/.gitlab/ci/build_release.yml deleted file mode 100644 index 39f020768af57bd326edee5820365d149f2ead36..0000000000000000000000000000000000000000 --- a/.gitlab/ci/build_release.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- -.build_docker_release_template: - extends: - - .default_settings_template - - .image_template__docker - - .docker_registry_auth # Sets up a before_script - variables: - IMAGE_ARCH_PREFIX: "" - script: - - ./scripts/ci/docker_release.sh - -docker:manual_build_amd64: - extends: - - .build_docker_release_template - - .rules_template__development_docker - variables: - IMAGE_ARCH_PREFIX: "amd64_" - stage: manual - needs: [] - -docker:manual_build_arm64: - extends: - - .build_docker_release_template - - .rules_template__development_docker - variables: - IMAGE_ARCH_PREFIX: "arm64_" - stage: manual - needs: [] - tags: - - arm64 - -docker:build_amd64: - extends: - - .build_docker_release_template - - .rules_template__master_and_releases - variables: - IMAGE_ARCH_PREFIX: "amd64_" - stage: build - -docker:build_arm64: - extends: - - .build_docker_release_template - - .rules_template__master_and_releases - variables: - IMAGE_ARCH_PREFIX: "arm64_" - stage: build - tags: - - arm64 diff --git a/.gitlab/ci/doc.yml b/.gitlab/ci/doc.yml deleted file mode 100644 index 362e9800293ff2fa7c2d260c8cbc09e1b73d91fd..0000000000000000000000000000000000000000 --- a/.gitlab/ci/doc.yml +++ /dev/null @@ -1,62 +0,0 @@ ---- - -# Warning: the documentation:linkcheck job must have at least the same -# restrictions in the rules as documentation:build_all, otherwise the CI -# may complain that documentation:linkcheck depends on documentation:build_all -# which does not exist. -documentation:build_all: - extends: - - .default_settings_template - - .image_template__runtime_build_test_dependencies_template - - .rules_template__development - - .needs_template__wait_for_trigger_job - stage: build - before_script: - - . $HOME/.venv/bin/activate - script: - - make -C ${CI_PROJECT_DIR}/.gitlab/ci doc-build-all - artifacts: - expose_as: 'Documentation - excluding old protocols' - paths: - # Path must be terminated with / to expose artifact (gitlab-org/gitlab#/36706) - - docs/_build/ - expire_in: 1 week - -documentation:linkcheck: - extends: - - .default_settings_template - - .image_template__runtime_build_test_dependencies_template - - .rules_template__development_documentation - stage: doc - needs: [] - before_script: - - . ./scripts/version.sh - - . $HOME/.venv/bin/activate - script: - - make -C ${CI_PROJECT_DIR}/.gitlab/ci doc-linkcheck - allow_failure: true - -# here we use this hack to publish the tezos documentation on -# gitlab.io because we want to publish the doc for the project -# tezos under https://tezos.gitlab.io and not https://tezos.gitlab.io/tezos -# The latter follows the gitlab url convention of -# https://.gitlab.io// -# Notice that we push only if CI_COMMIT_REF_NAME is really master . -# This allows to test the release workflow -publish:documentation: - extends: - - .default_settings_template - - .image_template__runtime_build_test_dependencies_template - - .rules_template__master - stage: doc - before_script: - - echo "${CI_PK_GITLAB_DOC}" > ~/.ssh/id_ed25519 - - echo "${CI_KH}" > ~/.ssh/known_hosts - - chmod 400 ~/.ssh/id_ed25519 - # Load the environment poetry previously created in the docker image. - # Give access to the Python dependencies/executables - - . $HOME/.venv/bin/activate - script: - - make -C ${CI_PROJECT_DIR}/.gitlab/ci doc-publish - # Publish jobs are uninterruptible to avoid publishing partial results. - interruptible: false diff --git a/.gitlab/ci/jobs/build/arm64.yml b/.gitlab/ci/jobs/build/arm64.yml new file mode 100644 index 0000000000000000000000000000000000000000..9d46c3239ccd588708b057b9c792d11bbdd3a395 --- /dev/null +++ b/.gitlab/ci/jobs/build/arm64.yml @@ -0,0 +1,19 @@ +include: .gitlab/ci/jobs/build/common.yml + +build_arm64: + extends: + - .build + variables: + ARCH: "arm64" + # Disable coverage for arm64 + COVERAGE_OPTIONS: "" + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' + when: always + # Run when there is label on the merge request + - if: '$CI_MERGE_REQUEST_LABELS =~ /(?:^|[,])ci--arm64(?:$|[,])/' + when: on_success + - when: manual + allow_failure: true + tags: + - arm64 \ No newline at end of file diff --git a/.gitlab/ci/jobs/build/common.yml b/.gitlab/ci/jobs/build/common.yml new file mode 100644 index 0000000000000000000000000000000000000000..4029dba148ffd5e0ac46fe6a8af921c4f9bc2515 --- /dev/null +++ b/.gitlab/ci/jobs/build/common.yml @@ -0,0 +1,53 @@ +.build: + extends: .build_template + variables: + ARCH: "" + # We fix the value of GIT_{SHORTREF,DATETIME,VERSION} (these are + # read by src/lib_version and output by the binaries `--version` + # option). Fixing these values on development builds improves + # cache usage. + GIT_SHORTREF: "00000000" + GIT_DATETIME: "1970-01-01 00:00:00 +0000%" + GIT_VERSION: "dev" + script: + - make -C ${CI_PROJECT_DIR}/.gitlab/ci build-full-unreleased + cache: + key: "$CI_COMMIT_REF_SLUG" + paths: + - _build + artifacts: + name: "build-$ARCH-$CI_COMMIT_REF_SLUG" + paths: + - octez-* + - tezos-* + - src/proto_*/parameters/*.json + - _build/default/src/lib_protocol_compiler/bin/main_native.exe + expire_in: 1 day + when: on_success + +# similar to the build template above, this template +# compiles all binaries using the static profile. +.build_static_binaries_template: + extends: + - .default_settings_template + - .image_template__runtime_build_dependencies_template + stage: build + variables: + ARCH: "" + script: + - make -C ${CI_PROJECT_DIR}/.gitlab/ci build-static-binaries + + artifacts: + paths: + - "octez-binaries/$ARCH/*" + +.build_docker_release_template: + extends: + - .default_settings_template + - .image_template__docker + variables: + IMAGE_ARCH_PREFIX: "" + before_script: + - ./scripts/ci/docker_registry_auth.sh + script: + - ./scripts/ci/docker_release.sh diff --git a/.gitlab/ci/jobs/build/docker_amd64_release.yml b/.gitlab/ci/jobs/build/docker_amd64_release.yml new file mode 100644 index 0000000000000000000000000000000000000000..150e956c66f89ae6d066c946d4ec8ead8c4fc868 --- /dev/null +++ b/.gitlab/ci/jobs/build/docker_amd64_release.yml @@ -0,0 +1,9 @@ +include: .gitlab/ci/jobs/build/common.yml + +docker:amd64: + extends: + - .build_docker_release_template + variables: + IMAGE_ARCH_PREFIX: "amd64_" + CI_DOCKER_HUB: "true" + stage: build \ No newline at end of file diff --git a/.gitlab/ci/jobs/build/docker_amd64_test.yml b/.gitlab/ci/jobs/build/docker_amd64_test.yml new file mode 100644 index 0000000000000000000000000000000000000000..c13d9ed960324c1384eafe46118aaf23cd5d438c --- /dev/null +++ b/.gitlab/ci/jobs/build/docker_amd64_test.yml @@ -0,0 +1,9 @@ +include: .gitlab/ci/jobs/build/common.yml + +docker:amd64: + extends: + - .build_docker_release_template + variables: + IMAGE_ARCH_PREFIX: "amd64_" + CI_DOCKER_HUB: "false" + stage: build \ No newline at end of file diff --git a/.gitlab/ci/jobs/build/docker_amd64_test_manual.yml b/.gitlab/ci/jobs/build/docker_amd64_test_manual.yml new file mode 100644 index 0000000000000000000000000000000000000000..659eec9e64394e2984167f13aa0d661ffb1cee52 --- /dev/null +++ b/.gitlab/ci/jobs/build/docker_amd64_test_manual.yml @@ -0,0 +1,12 @@ +include: .gitlab/ci/jobs/build/common.yml + +docker:amd64: + extends: + - .build_docker_release_template + stage: manual + needs: [] + when: manual + variables: + IMAGE_ARCH_PREFIX: "amd64_" + CI_DOCKER_HUB: "false" + allow_failure: true diff --git a/.gitlab/ci/jobs/build/docker_arm64_release.yml b/.gitlab/ci/jobs/build/docker_arm64_release.yml new file mode 100644 index 0000000000000000000000000000000000000000..7a011674930bbee3f3b7f292f92d4cfbf6b49f1b --- /dev/null +++ b/.gitlab/ci/jobs/build/docker_arm64_release.yml @@ -0,0 +1,11 @@ +include: .gitlab/ci/jobs/build/common.yml + +docker:arm64: + extends: + - .build_docker_release_template + stage: build + variables: + IMAGE_ARCH_PREFIX: "arm64_" + CI_DOCKER_HUB: "true" + tags: + - arm64 \ No newline at end of file diff --git a/.gitlab/ci/jobs/build/docker_arm64_test.yml b/.gitlab/ci/jobs/build/docker_arm64_test.yml new file mode 100644 index 0000000000000000000000000000000000000000..6e28da20c8424ace565dc7915712f4c0cdd402da --- /dev/null +++ b/.gitlab/ci/jobs/build/docker_arm64_test.yml @@ -0,0 +1,11 @@ +include: .gitlab/ci/jobs/build/common.yml + +docker:arm64: + extends: + - .build_docker_release_template + stage: build + variables: + IMAGE_ARCH_PREFIX: "arm64_" + CI_DOCKER_HUB: "false" + tags: + - arm64 \ No newline at end of file diff --git a/.gitlab/ci/jobs/build/docker_arm64_test_manual.yml b/.gitlab/ci/jobs/build/docker_arm64_test_manual.yml new file mode 100644 index 0000000000000000000000000000000000000000..5ebc0af0233fff7cade66080f453e35ca695fe2a --- /dev/null +++ b/.gitlab/ci/jobs/build/docker_arm64_test_manual.yml @@ -0,0 +1,14 @@ +include: .gitlab/ci/jobs/build/common.yml + +docker:arm64: + extends: + - .build_docker_release_template + stage: manual + needs: [] + when: manual + variables: + IMAGE_ARCH_PREFIX: "arm64_" + CI_DOCKER_HUB: "false" + allow_failure: true + tags: + - arm64 diff --git a/.gitlab/ci/jobs/build/get_contracts.yml b/.gitlab/ci/jobs/build/get_contracts.yml new file mode 100644 index 0000000000000000000000000000000000000000..ec505d8d83af1ba147fee242241df0e71a2754d0 --- /dev/null +++ b/.gitlab/ci/jobs/build/get_contracts.yml @@ -0,0 +1,7 @@ +include: .gitlab/ci/jobs/build/common.yml + +build_get_contracts: + extends: + - .build_template + script: + - dune build @devtools/get_contracts/check \ No newline at end of file diff --git a/.gitlab/ci/jobs/build/static_arm64_linux_binaries.yml b/.gitlab/ci/jobs/build/static_arm64_linux_binaries.yml new file mode 100644 index 0000000000000000000000000000000000000000..86b191e70192b9339ab86d164faf43b61a8ac220 --- /dev/null +++ b/.gitlab/ci/jobs/build/static_arm64_linux_binaries.yml @@ -0,0 +1,10 @@ +include: .gitlab/ci/jobs/build/common.yml + +# arm64 static binaries are only built for master and releases. +build:static-arm64-linux-binaries: + extends: + - .build_static_binaries_template + variables: + ARCH: "arm64" + tags: + - arm64 \ No newline at end of file diff --git a/.gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml b/.gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml new file mode 100644 index 0000000000000000000000000000000000000000..255cf05d7b1d93130000d0d46a85eb01eb78684d --- /dev/null +++ b/.gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml @@ -0,0 +1,15 @@ +include: .gitlab/ci/jobs/build/common.yml + +# x86_64 static binaries are necessary for development branches, as we want to make +# sure the build does not break. We also need it for release tags, to as its artifacts +# are published in the stage `publish_release`. For this reason, we build it in all pipelines, +# except on latest release branches. +build:static-x86_64-linux-binaries: + extends: + - .build_static_binaries_template + # Even though not many tests depend on static executables, some of those that do + # are limiting factors in the total duration of pipelines. + # So we start this job as early as possible, without waiting for sanity_ci. + needs: [trigger] + variables: + ARCH: "x86_64" diff --git a/.gitlab/ci/jobs/build/x86_64.yml b/.gitlab/ci/jobs/build/x86_64.yml new file mode 100644 index 0000000000000000000000000000000000000000..70918fd9280d117cd13dcce68f84ea8486d552ce --- /dev/null +++ b/.gitlab/ci/jobs/build/x86_64.yml @@ -0,0 +1,10 @@ +include: .gitlab/ci/jobs/build/common.yml + +build_x86_64: + extends: + - .build + # This job is the main bottleneck since a lot of tests depend on it. + # So we start it even before sanity_ci. + needs: [trigger] + variables: + ARCH: "x86_64" diff --git a/.gitlab/ci/jobs/coverage/common.yml b/.gitlab/ci/jobs/coverage/common.yml new file mode 100644 index 0000000000000000000000000000000000000000..f4efb7267e63f2f42032ebd17c32c07b2a67b21c --- /dev/null +++ b/.gitlab/ci/jobs/coverage/common.yml @@ -0,0 +1,19 @@ +.template__coverage_report: + extends: .template__coverage + stage: test_coverage + before_script: + # Load the environment poetry previously created in the docker image + # Give access to the Python dependencies/executables + - . "$HOME/.venv/bin/activate" + coverage: '/Coverage: ([^%]+%)/' + artifacts: + expose_as: 'Coverage report' + reports: + coverage_report: + coverage_format: cobertura + path: _coverage_report/cobertura.xml + paths: + - _coverage_report/ + - $BISECT_FILE + expire_in: 15 days + when: always diff --git a/.gitlab/ci/test_coverage.yml b/.gitlab/ci/jobs/coverage/coverage.yml similarity index 98% rename from .gitlab/ci/test_coverage.yml rename to .gitlab/ci/jobs/coverage/coverage.yml index 3bee8eca15ae2157048d8c8e2340e3044b1662ba..a6caef61f64535d5a220d1c769a4fe7c0879c4fb 100644 --- a/.gitlab/ci/test_coverage.yml +++ b/.gitlab/ci/jobs/coverage/coverage.yml @@ -2,6 +2,8 @@ # This job fetches coverage files by precedent test stage. It creates the html, # summary and cobertura reports. It also provide a coverage % for the merge request. +include: .gitlab/ci/jobs/coverage/common.yml + unified_coverage: extends: - .default_settings_template diff --git a/.gitlab/ci/test_coverage_default.yml b/.gitlab/ci/jobs/coverage/coverage_default.yml similarity index 93% rename from .gitlab/ci/test_coverage_default.yml rename to .gitlab/ci/jobs/coverage/coverage_default.yml index 055550d82858bc6e6e53bc57ed639ef6e7b6c543..85a95262e10f2f7d1da7eda8d166b09a7e6acd3b 100644 --- a/.gitlab/ci/test_coverage_default.yml +++ b/.gitlab/ci/jobs/coverage/coverage_default.yml @@ -1,8 +1,9 @@ ---- # This job fetches coverage files from the most recently merged MR on the default branch. # It creates the html, summary and cobertura reports. It also provide a coverage % for the # merge request. +include: .gitlab/ci/jobs/coverage/common.yml + unified_coverage: extends: - .default_settings_template diff --git a/.gitlab/ci/jobs/doc/build_all.yml b/.gitlab/ci/jobs/doc/build_all.yml new file mode 100644 index 0000000000000000000000000000000000000000..83e9477deb022f7152502e5e04af3ec081fd82d9 --- /dev/null +++ b/.gitlab/ci/jobs/doc/build_all.yml @@ -0,0 +1,20 @@ +# Warning: the documentation:linkcheck job must have at least the same +# restrictions in the rules as documentation:build_all, otherwise the CI +# may complain that documentation:linkcheck depends on documentation:build_all +# which does not exist. +documentation:build_all: + extends: + - .default_settings_template + - .image_template__runtime_build_test_dependencies_template + stage: build + needs: [trigger] + before_script: + - . $HOME/.venv/bin/activate + script: + - make -C ${CI_PROJECT_DIR}/.gitlab/ci doc-build-all + artifacts: + expose_as: 'Documentation - excluding old protocols' + paths: + # Path must be terminated with / to expose artifact (gitlab-org/gitlab#/36706) + - docs/_build/ + expire_in: 1 week diff --git a/.gitlab/ci/jobs/doc/linkcheck.yml b/.gitlab/ci/jobs/doc/linkcheck.yml new file mode 100644 index 0000000000000000000000000000000000000000..02ffc163c2109a8f1c84d04513254435ef91a06a --- /dev/null +++ b/.gitlab/ci/jobs/doc/linkcheck.yml @@ -0,0 +1,25 @@ +documentation:linkcheck: + extends: + - .default_settings_template + - .image_template__runtime_build_test_dependencies_template + stage: doc + needs: [] + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' + when: always + - if: '$CI_MERGE_REQUEST_ID' + changes: + - docs/introduction/*.sh + - docs/developer/*.sh + - docs/scripts/test_doc_scripts.sh + when: on_success + # Run when there is label on the merge request + - if: '$CI_MERGE_REQUEST_LABELS =~ /(?:^|[,])ci--docs(?:$|[,])/' + when: on_success + - when: manual + before_script: + - . ./scripts/version.sh + - . $HOME/.venv/bin/activate + script: + - make -C ${CI_PROJECT_DIR}/.gitlab/ci doc-linkcheck + allow_failure: true diff --git a/.gitlab/ci/jobs/doc/publish.yml b/.gitlab/ci/jobs/doc/publish.yml new file mode 100644 index 0000000000000000000000000000000000000000..1b4856b5eb887695110801e94958b721f7cfabf4 --- /dev/null +++ b/.gitlab/ci/jobs/doc/publish.yml @@ -0,0 +1,25 @@ +include: .gitlab/ci/jobs/shared/templates.yml + +# here we use this hack to publish the tezos documentation on +# gitlab.io because we want to publish the doc for the project +# tezos under https://tezos.gitlab.io and not https://tezos.gitlab.io/tezos +# The latter follows the gitlab url convention of +# https://.gitlab.io// +# Notice that we push only if CI_COMMIT_REF_NAME is really master . +# This allows to test the release workflow +publish:documentation: + extends: + - .default_settings_template + - .image_template__runtime_build_test_dependencies_template + stage: doc + before_script: + - echo "${CI_PK_GITLAB_DOC}" > ~/.ssh/id_ed25519 + - echo "${CI_KH}" > ~/.ssh/known_hosts + - chmod 400 ~/.ssh/id_ed25519 + # Load the environment poetry previously created in the docker image. + # Give access to the Python dependencies/executables + - . $HOME/.venv/bin/activate + script: + - make -C ${CI_PROJECT_DIR}/.gitlab/ci doc-publish + # Publish jobs are uninterruptible to avoid publishing partial results. + interruptible: false diff --git a/.gitlab/ci/doc/test-scripts.yml b/.gitlab/ci/jobs/doc/tests.yml similarity index 85% rename from .gitlab/ci/doc/test-scripts.yml rename to .gitlab/ci/jobs/doc/tests.yml index dfe3ff513fe21eb0a18448bf574129c4ceba01f0..09ede4856c0fee62e0238e8fe633ecb38ee35239 100644 --- a/.gitlab/ci/doc/test-scripts.yml +++ b/.gitlab/ci/jobs/doc/tests.yml @@ -1,8 +1,23 @@ .base-test-doc-scripts-template: stage: doc needs: [] + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' + when: always + - if: '$CI_MERGE_REQUEST_ID' + changes: + - docs/introduction/*.sh + - docs/developer/*.sh + - docs/scripts/test_doc_scripts.sh + when: on_success + # Run when there is label on the merge request + - if: '$CI_MERGE_REQUEST_LABELS =~ /(?:^|[,])ci--docs(?:$|[,])/' + when: on_success + - when: manual + allow_failure: true before_script: # Trigger an allowed fail on runner that do not have the tezos tag + # This condition mean this job MUST be run under Tezos namespace - if ! echo $CI_RUNNER_TAGS | grep -qe '\btezos\b'; then echo -e "\e[33m/.\ This test is skipped on runners lacking the tezos tag\e[0m"; exit 137; @@ -14,7 +29,6 @@ extends: - .default_settings_template - .base-test-doc-scripts-template - - .rules_template__development_documentation script: - ./docs/introduction/install-bin-ubuntu.sh @@ -30,7 +44,6 @@ install_bin_focal: extends: - .default_settings_template - .base-test-doc-scripts-template - - .rules_template__development_documentation script: - ./docs/introduction/install-bin-ubuntu.sh rc @@ -46,7 +59,6 @@ install_bin_rc_focal: extends: - .default_settings_template - .base-test-doc-scripts-template - - .rules_template__development_documentation script: - ./docs/introduction/install-bin-fedora.sh @@ -58,7 +70,6 @@ install_bin_fedora_35: extends: - .default_settings_template - .base-test-doc-scripts-template - - .rules_template__development_documentation script: - ./docs/introduction/install-bin-fedora.sh rc @@ -70,7 +81,6 @@ install_bin_rc_fedora_35: extends: - .default_settings_template - .base-test-doc-scripts-template - - .rules_template__development_documentation script: - ./docs/introduction/install-opam.sh @@ -86,7 +96,6 @@ install_opam_focal: extends: - .default_settings_template - .base-test-doc-scripts-template - - .rules_template__development_documentation compile_release_sources_buster: image: ocaml/opam:debian-10 @@ -104,7 +113,6 @@ compile_sources_buster: extends: - .default_settings_template - .base-test-doc-scripts-template - - .rules_template__development_documentation script: - ./docs/developer/install-python-debian-ubuntu.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} diff --git a/.gitlab/ci/opam-ci.yml b/.gitlab/ci/jobs/packaging/opam_package.yml similarity index 96% rename from .gitlab/ci/opam-ci.yml rename to .gitlab/ci/jobs/packaging/opam_package.yml index 7c0f5a32d280875d9ac5e1292005db6464449a8f..3b20995115bf8655c4b80d08e36a579f647ad193 100644 --- a/.gitlab/ci/opam-ci.yml +++ b/.gitlab/ci/jobs/packaging/opam_package.yml @@ -48,12 +48,9 @@ .rules_template__trigger_opam_batch_1: rules: # Run on scheduled builds. - - if: '$TZ_PIPELINE_KIND == "SCHEDULE" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' + - if: '$CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' when: delayed start_in: 1 minutes - # Never run on branch pipelines for master. - - if: '$CI_COMMIT_BRANCH == $TEZOS_DEFAULT_BRANCH' - when: never # Run when there is label on the merge request - if: '$CI_MERGE_REQUEST_LABELS =~ /(?:^|[,])ci--opam(?:$|[,])/' when: delayed @@ -79,12 +76,9 @@ .rules_template__trigger_opam_batch_2: rules: # Run on scheduled builds. - - if: '$TZ_PIPELINE_KIND == "SCHEDULE" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' + - if: '$CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' when: delayed start_in: 2 minutes - # Never run on branch pipelines for master. - - if: '$CI_COMMIT_BRANCH == $TEZOS_DEFAULT_BRANCH' - when: never # Run when there is label on the merge request - if: '$CI_MERGE_REQUEST_LABELS =~ /(?:^|[,])ci--opam(?:$|[,])/' when: delayed @@ -110,12 +104,9 @@ .rules_template__trigger_opam_batch_3: rules: # Run on scheduled builds. - - if: '$TZ_PIPELINE_KIND == "SCHEDULE" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' + - if: '$CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' when: delayed start_in: 3 minutes - # Never run on branch pipelines for master. - - if: '$CI_COMMIT_BRANCH == $TEZOS_DEFAULT_BRANCH' - when: never # Run when there is label on the merge request - if: '$CI_MERGE_REQUEST_LABELS =~ /(?:^|[,])ci--opam(?:$|[,])/' when: delayed @@ -141,12 +132,9 @@ .rules_template__trigger_opam_batch_4: rules: # Run on scheduled builds. - - if: '$TZ_PIPELINE_KIND == "SCHEDULE" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' + - if: '$CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' when: delayed start_in: 4 minutes - # Never run on branch pipelines for master. - - if: '$CI_COMMIT_BRANCH == $TEZOS_DEFAULT_BRANCH' - when: never # Run when there is label on the merge request - if: '$CI_MERGE_REQUEST_LABELS =~ /(?:^|[,])ci--opam(?:$|[,])/' when: delayed @@ -172,12 +160,9 @@ .rules_template__trigger_opam_batch_5: rules: # Run on scheduled builds. - - if: '$TZ_PIPELINE_KIND == "SCHEDULE" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' + - if: '$CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' when: delayed start_in: 5 minutes - # Never run on branch pipelines for master. - - if: '$CI_COMMIT_BRANCH == $TEZOS_DEFAULT_BRANCH' - when: never # Run when there is label on the merge request - if: '$CI_MERGE_REQUEST_LABELS =~ /(?:^|[,])ci--opam(?:$|[,])/' when: delayed @@ -203,12 +188,9 @@ .rules_template__trigger_opam_batch_6: rules: # Run on scheduled builds. - - if: '$TZ_PIPELINE_KIND == "SCHEDULE" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' + - if: '$CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' when: delayed start_in: 6 minutes - # Never run on branch pipelines for master. - - if: '$CI_COMMIT_BRANCH == $TEZOS_DEFAULT_BRANCH' - when: never # Run when there is label on the merge request - if: '$CI_MERGE_REQUEST_LABELS =~ /(?:^|[,])ci--opam(?:$|[,])/' when: delayed @@ -234,12 +216,9 @@ .rules_template__trigger_opam_batch_7: rules: # Run on scheduled builds. - - if: '$TZ_PIPELINE_KIND == "SCHEDULE" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' + - if: '$CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' when: delayed start_in: 7 minutes - # Never run on branch pipelines for master. - - if: '$CI_COMMIT_BRANCH == $TEZOS_DEFAULT_BRANCH' - when: never # Run when there is label on the merge request - if: '$CI_MERGE_REQUEST_LABELS =~ /(?:^|[,])ci--opam(?:$|[,])/' when: delayed diff --git a/.gitlab/ci/jobs/packaging/opam_prepare.yml b/.gitlab/ci/jobs/packaging/opam_prepare.yml new file mode 100644 index 0000000000000000000000000000000000000000..c0e5dd3c94ce0be1c59f2becc0f6ad668f8e1c38 --- /dev/null +++ b/.gitlab/ci/jobs/packaging/opam_prepare.yml @@ -0,0 +1,37 @@ +# This template is used by the opam test jobs defined in opam-ci.yml +.opam_template: + extends: + - .default_settings_template + - .image_template__runtime_prebuild_dependencies_template + stage: packaging + # FIXME: https://gitlab.com/nomadic-labs/tezos/-/issues/663 + # FIXME: https://gitlab.com/nomadic-labs/tezos/-/issues/664 + # At the time of writing, the opam tests were quite flaky. + # Therefore, a retry was added. This should be removed once the + # underlying tests have been fixed. + retry: 2 + script: + - make -C ${CI_PROJECT_DIR}/.gitlab/ci opam-configure + needs: [opam:prepare] + dependencies: [opam:prepare] + after_script: + # Stores logs in opam_logs for artifacts and outputs an excerpt on failure. + - OPAM_LOGS=opam_logs ./scripts/ci/opam_handle_output.sh + artifacts: + paths: + - opam_logs/ + expire_in: 1 week + when: always + +opam:prepare: + extends: + - .default_settings_template + - .image_template__runtime_prebuild_dependencies_template + - .rules_template__trigger_opam_batch_1 + stage: packaging + needs: [trigger] + script: + - make -C ${CI_PROJECT_DIR}/.gitlab/ci opam-prepare + artifacts: + paths: + - _opam-repo-for-release/ diff --git a/.gitlab/ci/jobs/prepare_release/docker_merge_manifests_release.yml b/.gitlab/ci/jobs/prepare_release/docker_merge_manifests_release.yml new file mode 100644 index 0000000000000000000000000000000000000000..9f645d7e8e9de7a9e70d6bfa8d60ec0d86cf5d5c --- /dev/null +++ b/.gitlab/ci/jobs/prepare_release/docker_merge_manifests_release.yml @@ -0,0 +1,16 @@ +include: .gitlab/ci/jobs/publish/common.yml + +# Note: here we rely on $IMAGE_ARCH_PREFIX to be empty. +# Otherwise, $DOCKER_IMAGE_TAG would contain $IMAGE_ARCH_PREFIX too. +# $IMAGE_ARCH_PREFIX is only used when building Docker images, +# here we handle all architectures so there is no such variable. +docker:merge_manifests: + extends: + - .image_template__docker + stage: prepare_release + variables: + CI_DOCKER_HUB: "true" + before_script: + - ./scripts/ci/docker_registry_auth.sh + script: + - ./scripts/ci/docker_merge_manifests.sh \ No newline at end of file diff --git a/.gitlab/ci/jobs/prepare_release/docker_merge_manifests_test.yml b/.gitlab/ci/jobs/prepare_release/docker_merge_manifests_test.yml new file mode 100644 index 0000000000000000000000000000000000000000..fecff647b86cdb55e69221b80f2ee3560f6948d5 --- /dev/null +++ b/.gitlab/ci/jobs/prepare_release/docker_merge_manifests_test.yml @@ -0,0 +1,16 @@ +include: .gitlab/ci/jobs/publish/common.yml + +# Note: here we rely on $IMAGE_ARCH_PREFIX to be empty. +# Otherwise, $DOCKER_IMAGE_TAG would contain $IMAGE_ARCH_PREFIX too. +# $IMAGE_ARCH_PREFIX is only used when building Docker images, +# here we handle all architectures so there is no such variable. +docker:merge_manifests: + extends: + - .image_template__docker + stage: prepare_release + variables: + CI_DOCKER_HUB: "false" + before_script: + - ./scripts/ci/docker_registry_auth.sh + script: + - ./scripts/ci/docker_merge_manifests.sh \ No newline at end of file diff --git a/.gitlab/ci/jobs/publish/common.yml b/.gitlab/ci/jobs/publish/common.yml new file mode 100644 index 0000000000000000000000000000000000000000..292fc0b355609c2b8846fb2cc0ae18974abf72f8 --- /dev/null +++ b/.gitlab/ci/jobs/publish/common.yml @@ -0,0 +1,58 @@ +# +# Templates for building deb & rpm packages. +# +# The AWS resources used for the packaging are managed on https://gitlab.com/nomadic-labs/iac/terraform/octez-packaging +# +# Specifically we use 2 S3 buckets to host the deb and rpm packages and one IAM role which has the ability to +# push changes. The jobs use the OpenID protocol to connect to AWS, receive temporary credentials and assume +# the IAM role. +# +# Details can be found on the following links: +# - https://docs.gitlab.com/ee/ci/cloud_services/index.html#configure-a-conditional-role-with-oidc-claims +# - https://docs.gitlab.com/ee/ci/cloud_services/aws/index.html + +.build_deb_packages: + stage: publish_release + image: ${CI_REGISTRY}/tezos/docker-images/ci-package:latest-ubuntu@sha256:76c894f509b84e106c9d056882c63cb497a2778f7c7dbbd1c73abe2568ee446e + needs: + - "build:static-x86_64-linux-binaries" + rules: + # NOTE: These jobs are disabled until the upstream scripts (https://github.com/serokell/tezos-packaging) + # provide support for the lima protocol and drop support for jakarta. + - when: never + variables: + TEZOS_PACKAGING_VERSION: "872ac62526c49bd59b8621ee06c066141e940dbd" + TEZOS_PACKAGING_REPO: "https://github.com/serokell/tezos-packaging" + TEZOS_LICENSE_VERSION: "master" + ROLE_ARN: "arn:aws:iam::623103086665:role/octez-packaging-bucket-writer" + OCTEZ_DEB_BUCKET: "octez-deb-pkgs" + AWS_BUCKET_REGION: "eu-west-1" + PACKAGE_FORMAT: "deb" + ARCH: "amd64" + TEZOS_BINARIES: "octez-binaries/x86_64" + script: + - ./scripts/ci/create_linux_package.sh + - ./scripts/ci/upload_linux_package.sh + +.build_rpm_packages: + stage: publish_release + image: ${CI_REGISTRY}/tezos/docker-images/ci-package:latest-fedora@sha256:35bfacc001e4c5247679e2f79c36ab1c83bd893a1e909c5c1afd1beff44c6895 + needs: + - "build:static-x86_64-linux-binaries" + variables: + TEZOS_PACKAGING_VERSION: "872ac62526c49bd59b8621ee06c066141e940dbd" + TEZOS_PACKAGING_REPO: "https://github.com/serokell/tezos-packaging" + TEZOS_LICENSE_VERSION: "master" + ROLE_ARN: "arn:aws:iam::623103086665:role/octez-packaging-bucket-writer" + OCTEZ_RPM_BUCKET: "octez-rpm-pkgs" + AWS_BUCKET_REGION: "eu-west-1" + PACKAGE_FORMAT: "rpm" + ARCH: "amd64" + TEZOS_BINARIES: "octez-binaries/x86_64" + script: + - ./scripts/ci/create_linux_package.sh + - ./scripts/ci/upload_linux_package.sh + rules: + # NOTE: These jobs are disabled until the upstream scripts (https://github.com/serokell/tezos-packaging) + # provide support for the lima protocol and drop support for jakarta. + - when: never diff --git a/.gitlab/ci/jobs/publish/deb_packages_stable_amd64.yml b/.gitlab/ci/jobs/publish/deb_packages_stable_amd64.yml new file mode 100644 index 0000000000000000000000000000000000000000..65e480760a2d7add850093717bf36774993c3997 --- /dev/null +++ b/.gitlab/ci/jobs/publish/deb_packages_stable_amd64.yml @@ -0,0 +1,13 @@ +# +# Deb packages +# + +include: .gitlab/ci/jobs/publish/common.yml + +deb-packages-stable-amd64: + extends: + - .build_deb_packages + variables: + CODENAME: "stable" + before_script: + - export TEZOS_VERSION=$(git tag --points-at HEAD | grep -oE '^v[0-9]{1,3}\.[0-9]{1,3}$' ||:) diff --git a/.gitlab/ci/jobs/publish/deb_packages_unstable_amd64.yml b/.gitlab/ci/jobs/publish/deb_packages_unstable_amd64.yml new file mode 100644 index 0000000000000000000000000000000000000000..660a297a3a72245e5f3bca39377068341b541ed9 --- /dev/null +++ b/.gitlab/ci/jobs/publish/deb_packages_unstable_amd64.yml @@ -0,0 +1,12 @@ +# +# Deb packages +# + +include: .gitlab/ci/jobs/publish/common.yml + +deb-packages-unstable-amd64: + extends: + - .build_deb_packages + variables: + CODENAME: "unstable" + TEZOS_VERSION: v0.0+$CI_COMMIT_SHORT_SHA diff --git a/.gitlab/ci/jobs/publish/docker_promote_to_latest_release.yml b/.gitlab/ci/jobs/publish/docker_promote_to_latest_release.yml new file mode 100644 index 0000000000000000000000000000000000000000..2bf1ba7a69001266a94ebc2e22af312dad4565ed --- /dev/null +++ b/.gitlab/ci/jobs/publish/docker_promote_to_latest_release.yml @@ -0,0 +1,12 @@ +include: .gitlab/ci/jobs/publish/common.yml + +docker:promote_to_latest: + extends: + - .image_template__docker + stage: publish_release + variables: + CI_DOCKER_HUB: "true" + before_script: + - ./scripts/ci/docker_registry_auth.sh + script: + - ./scripts/ci/docker_promote_to_latest.sh diff --git a/.gitlab/ci/jobs/publish/docker_promote_to_latest_test.yml b/.gitlab/ci/jobs/publish/docker_promote_to_latest_test.yml new file mode 100644 index 0000000000000000000000000000000000000000..b6dd6413efc88c7c204d7caedbbdc530096e026b --- /dev/null +++ b/.gitlab/ci/jobs/publish/docker_promote_to_latest_test.yml @@ -0,0 +1,12 @@ +include: .gitlab/ci/jobs/publish/common.yml + +docker:promote_to_latest: + extends: + - .image_template__docker + stage: publish_release + variables: + CI_DOCKER_HUB: "false" + before_script: + - ./scripts/ci/docker_registry_auth.sh + script: + - ./scripts/ci/docker_promote_to_latest.sh \ No newline at end of file diff --git a/.gitlab/ci/jobs/publish/gitlab.yml b/.gitlab/ci/jobs/publish/gitlab.yml new file mode 100644 index 0000000000000000000000000000000000000000..e2446314334a9294bd446bf8c7ea82fe67575212 --- /dev/null +++ b/.gitlab/ci/jobs/publish/gitlab.yml @@ -0,0 +1,14 @@ +include: .gitlab/ci/jobs/publish/common.yml + +gitlab:release: + # https://gitlab.com/tezos/docker-images/ci-release + image: "${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0" + stage: publish_release_gitlab + # Publish jobs are uninterruptible to avoid publishing partial results. + interruptible: false + dependencies: + - build:static-x86_64-linux-binaries + - build:static-arm64-linux-binaries + - docker:merge_manifests + script: + - make -C ${CI_PROJECT_DIR}/.gitlab/ci gitlab-release diff --git a/.gitlab/ci/jobs/publish/opam.yml b/.gitlab/ci/jobs/publish/opam.yml new file mode 100644 index 0000000000000000000000000000000000000000..a61df2437deb46442c733f7a4cae5deff67a0ac8 --- /dev/null +++ b/.gitlab/ci/jobs/publish/opam.yml @@ -0,0 +1,13 @@ +include: .gitlab/ci/jobs/publish/common.yml + +opam:release: + extends: + - .image_template__runtime_build_test_dependencies_template + stage: publish_release + # Publish jobs are uninterruptible to avoid publishing partial results. + interruptible: false + dependencies: + - gitlab:release + script: + # create opam release and push to github (also pushes latest master branch) + - make -C ${CI_PROJECT_DIR}/.gitlab/ci opam-release diff --git a/.gitlab/ci/jobs/publish/rpm_packages_stable_amd64.yml b/.gitlab/ci/jobs/publish/rpm_packages_stable_amd64.yml new file mode 100644 index 0000000000000000000000000000000000000000..81085657f7a7c7c1d58f8101b55e184ff4219a35 --- /dev/null +++ b/.gitlab/ci/jobs/publish/rpm_packages_stable_amd64.yml @@ -0,0 +1,13 @@ +# +# RPM packages +# + +include: .gitlab/ci/jobs/publish/common.yml + +rpm-packages-stable-amd64: + extends: + - .build_rpm_packages + variables: + CODENAME: "stable" + before_script: + - export TEZOS_VERSION=$(git tag --points-at HEAD | grep -oE '^v[0-9]{1,3}\.[0-9]{1,3}$' ||:) diff --git a/.gitlab/ci/jobs/publish/rpm_packages_unstable_amd64.yml b/.gitlab/ci/jobs/publish/rpm_packages_unstable_amd64.yml new file mode 100644 index 0000000000000000000000000000000000000000..160ab6cfbfa5f3832c27ddcf8b2e4ce936f145fc --- /dev/null +++ b/.gitlab/ci/jobs/publish/rpm_packages_unstable_amd64.yml @@ -0,0 +1,12 @@ +# +# RPM packages +# + +include: .gitlab/ci/jobs/publish/common.yml + +rpm-packages-unstable-amd64: + extends: + - .build_rpm_packages + variables: + CODENAME: "unstable" + TEZOS_VERSION: v0.0+$CI_COMMIT_SHORT_SHA diff --git a/.gitlab/ci/jobs/sanity/docker_hadolint.yml b/.gitlab/ci/jobs/sanity/docker_hadolint.yml new file mode 100644 index 0000000000000000000000000000000000000000..3a1cf7db927d4f07a1c929169ce6bb6e1ff79514 --- /dev/null +++ b/.gitlab/ci/jobs/sanity/docker_hadolint.yml @@ -0,0 +1,15 @@ +docker:hadolint: + extends: + - .default_settings_template + image: hadolint/hadolint:2.9.3-debian + stage: sanity + rules: + # Only run on merge requests when Dockerfiles have changed + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + changes: + - build.Dockerfile + - Dockerfile + allow_failure: true + script: + - hadolint build.Dockerfile + - hadolint Dockerfile \ No newline at end of file diff --git a/.gitlab/ci/jobs/sanity/sanity_ci.yml b/.gitlab/ci/jobs/sanity/sanity_ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..b20a46426632cee317c06f399582563c4bd15b3c --- /dev/null +++ b/.gitlab/ci/jobs/sanity/sanity_ci.yml @@ -0,0 +1,7 @@ +sanity_ci: + extends: + - .default_settings_template + - .image_template__runtime_build_test_dependencies_template + stage: sanity + script: + - make -C ${CI_PROJECT_DIR}/.gitlab/ci sanity-ci diff --git a/.gitlab/ci/jobs/shared/templates.yml b/.gitlab/ci/jobs/shared/templates.yml new file mode 100644 index 0000000000000000000000000000000000000000..11a67fd6b1c959e3cb37d5d2aed0718f79622829 --- /dev/null +++ b/.gitlab/ci/jobs/shared/templates.yml @@ -0,0 +1,30 @@ +# Basic, specialised, minimal, orthogonal templates + +# Some settings we want by default on all jobs that cannot be set +# through the key `default` in `.gitlab-ci.yml`. +.default_settings_template: + # `dependencies` are empty be default. Each individual job or + # template requiring artifacts from preceeding jobs should declare + # their dependencies locally. + dependencies: [] + +# Add variables for bisect_ppx instrumentation +.template__coverage: + variables: + COVERAGE_OPTIONS: "--instrument-with bisect_ppx" + BISECT_FILE: "$CI_PROJECT_DIR/_coverage_output/" + SLACK_COVERAGE_CHANNEL: "C02PHBE7W73" + +.build_template: + extends: + - .default_settings_template + - .image_template__runtime_build_test_dependencies_template + - .template__coverage + stage: build + before_script: + # FIXME: https://gitlab.com/tezos/tezos/-/issues/2865 + - sudo chown -R $(id -u):$(id -g) $CI_PROJECT_DIR + - . ./scripts/version.sh + # Load the environment poetry previously created in the docker image. + # Give access to the Python dependencies/executables + - . $HOME/.venv/bin/activate \ No newline at end of file diff --git a/.gitlab/ci/jobs/test/check_precommit_hook.yml b/.gitlab/ci/jobs/test/check_precommit_hook.yml new file mode 100644 index 0000000000000000000000000000000000000000..b1690e048ca6288d9fa8622df5f08eb6170a1829 --- /dev/null +++ b/.gitlab/ci/jobs/test/check_precommit_hook.yml @@ -0,0 +1,6 @@ +check_precommit_hook: + extends: + - .test_template + needs: [trigger] + script: + - make -C ${CI_PROJECT_DIR}/.gitlab/ci check-precommit diff --git a/.gitlab/ci/jobs/test/check_scripts.yml b/.gitlab/ci/jobs/test/check_scripts.yml new file mode 100644 index 0000000000000000000000000000000000000000..a2319c5cb5e6a6214524a8ae5789d80620f1f563 --- /dev/null +++ b/.gitlab/ci/jobs/test/check_scripts.yml @@ -0,0 +1,46 @@ +############################################################ +## Stage: run scripts to check they are working properly ## +############################################################ + +# Note: those two jobs actually probably don't need the build_x86_64 job +# to have finished, but we don't want to start before build_x86_64 has finished either. +script:prepare_migration_test: + extends: .test_template + needs: ["build_x86_64"] + before_script: + - make + script: + - make -C ${CI_PROJECT_DIR}/.gitlab/ci integration-test-prepare-migration-test + +script:snapshot_alpha_and_link: + extends: .test_template + needs: ["build_x86_64"] + script: + - make -C ${CI_PROJECT_DIR}/.gitlab/ci integration-test-snapshot-alpha-and-link + +script:test-gen-genesis: + extends: + - .default_settings_template + - .image_template__runtime_build_test_dependencies_template + stage: test + needs: [trigger] + before_script: + - cd scripts/gen-genesis + script: + - dune build gen_genesis.exe + +script:test_release_versions: + extends: .test_template + script: + # FIXME: https://gitlab.com/tezos/tezos/-/issues/2865 + - sudo chown -R $(id -u):$(id -g) ${CI_PROJECT_DIR} + - ./scripts/test_release_version.sh + +check_scripts_b58_prefix: + # Can be changed to a python image, but using the build docker image to keep + # in sync with the python version used for the tests + extends: + - .test_template + needs: [trigger] + script: + - make -C ${CI_PROJECT_DIR}/.gitlab/ci check-scripts-b58-prefix diff --git a/.gitlab/ci/jobs/test/common.yml b/.gitlab/ci/jobs/test/common.yml new file mode 100644 index 0000000000000000000000000000000000000000..463437794d58a9aac0150ef50c3c9dd91ba87a67 --- /dev/null +++ b/.gitlab/ci/jobs/test/common.yml @@ -0,0 +1,80 @@ +.template__code_quality: + variables: + CODE_QUALITY_REPORT: "_reports/gl-code-quality-report.json" + +.test_template: + extends: + - .default_settings_template + - .image_template__runtime_build_test_dependencies_template + - .template__coverage + stage: test + before_script: + - . ./scripts/version.sh + # Load the environment poetry previously created in the docker image. + # Give access to the Python dependencies/executables + - . $HOME/.venv/bin/activate + retry: 2 + +# Merge coverage files after the execution +.template__coverage_files: + extends: .template__coverage + artifacts: + name: "coverage-files-$CI_JOB_ID" + paths: + - $BISECT_FILE + expire_in: 1 day + when: on_success + +# Definition for the environment to run all integration tests. +# This is also used by Tezt tests. +# In general we do not have to run make, which takes a while, +# because the binaries have been produced by the build job and are +# in the cache. But if they are not, we need to build them. +# Ideally we should also check that the baker / accuser / endorser +# exist (some tests use them) but their actual name depend on the protocol. +.integration_template: + extends: + - .test_template + - .template__coverage_files + dependencies: ["build_x86_64"] + # Start immediately after 'build_x86_64' and don't wait for + # intermediate stages to succeed + needs: ["build_x86_64"] + +# definition for the environment to run all integration tests +# integration tests are run only on x86_64 architectures (for now) +.pytest_template: + extends: + - .test_template + dependencies: ["build_x86_64"] + # Start immediately after 'build_x86_64' and don't wait for + # intermediate stages to succeed + needs: ["build_x86_64"] + before_script: + # Load the environment poetry previously created in the docker image. + # Give access to the Python dependencies/executables + - . $HOME/.venv/bin/activate + - mkdir tests_python/tmp + - cd tests_python + + +.pytest_integration: + extends: + - .pytest_template + - .template__coverage_files + # the number of jobs have been choosen to give jobs of maximal + # length ~10 minutes and to accommodate the addition of new protocol + # test suites without increasing wall time of the test stage. + variables: + TESTS: "" + script: + - make -C ${CI_PROJECT_DIR}/.gitlab/ci test-pytest + artifacts: + paths: + - tests_python/tmp/ + - tests_python/reports/*.xml + - $BISECT_FILE + reports: + junit: tests_python/reports/*.xml + expire_in: 7 day + when: always diff --git a/.gitlab/ci/jobs/test/integration_compiler_rejections.yml b/.gitlab/ci/jobs/test/integration_compiler_rejections.yml new file mode 100644 index 0000000000000000000000000000000000000000..36ae2214470fa45ca9efee8313bec5d42906084e --- /dev/null +++ b/.gitlab/ci/jobs/test/integration_compiler_rejections.yml @@ -0,0 +1,8 @@ +# integration:compiler-rejections does not +# require access to the binaries like the "true" integration tests +# below. Therefore, it does not extend the .integration_template. + +integration:compiler-rejections: + extends: .test_template + script: + - make -C ${CI_PROJECT_DIR}/.gitlab/ci integration-test-compiler-rejections \ No newline at end of file diff --git a/.gitlab/ci/jobs/test/integration_pytest_old_protocols.yml b/.gitlab/ci/jobs/test/integration_pytest_old_protocols.yml new file mode 100644 index 0000000000000000000000000000000000000000..193b49c3853047959d61c641a55ca4a2fa728151 --- /dev/null +++ b/.gitlab/ci/jobs/test/integration_pytest_old_protocols.yml @@ -0,0 +1,20 @@ +include: .gitlab/ci/jobs/test/common.yml + +integration:pytest_old_protocols: + extends: + - .pytest_integration + parallel: 16 + variables: + TESTS: "--ignore tests_alpha" + rules: + # Rules for jobs that should appear in the extended test pipeline or + # whenever the pytests change + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + changes: + - tests_python/**/* + when: on_success + - if: '$CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' + when: on_success + - if: '$CI_MERGE_REQUEST_LABELS =~ /(?:^|[,])ci--extended-tests(?:$|[,])/' + when: on_success + - when: never \ No newline at end of file diff --git a/.gitlab/ci/test/liquidity-baking-scripts-integrity.yml b/.gitlab/ci/jobs/test/liquidity_baking_scripts-integrity.yml similarity index 80% rename from .gitlab/ci/test/liquidity-baking-scripts-integrity.yml rename to .gitlab/ci/jobs/test/liquidity_baking_scripts-integrity.yml index c8bea6ec3be3363d1132e798cd18626d44f8b41c..5dcc2628a5be077ddf5166ac758c45d5d1d1b6ba 100644 --- a/.gitlab/ci/test/liquidity-baking-scripts-integrity.yml +++ b/.gitlab/ci/jobs/test/liquidity_baking_scripts-integrity.yml @@ -1,3 +1,5 @@ +include: .gitlab/ci/jobs/test/common.yml + test-liquidity-baking-scripts: extends: .integration_template stage: test diff --git a/.gitlab/ci/jobs/test/misc_checks.yml b/.gitlab/ci/jobs/test/misc_checks.yml new file mode 100644 index 0000000000000000000000000000000000000000..9579e66b0c321d6e856ee233ae255f1cdedfcfa7 --- /dev/null +++ b/.gitlab/ci/jobs/test/misc_checks.yml @@ -0,0 +1,8 @@ +misc_checks: + extends: + - .test_template + # These tests are not flaky at all so we overwrite the retry value + needs: [trigger] + retry: 1 + script: + - make -C ${CI_PROJECT_DIR}/.gitlab/ci lint-misc-check diff --git a/.gitlab/ci/jobs/test/misc_opam_checks.yml b/.gitlab/ci/jobs/test/misc_opam_checks.yml new file mode 100644 index 0000000000000000000000000000000000000000..94ac0f6f3723e2dae7de32f885e3351c0cf6aff2 --- /dev/null +++ b/.gitlab/ci/jobs/test/misc_opam_checks.yml @@ -0,0 +1,12 @@ +misc_opam_checks: + extends: + - .test_template + needs: [trigger] + script: + # checks that all deps of opam packages are already installed + - ./scripts/opam-check.sh + artifacts: + when: always + paths: + - opam_repo.patch + expire_in: 1 days diff --git a/.gitlab/ci/jobs/test/ometrics_code_quality.yml b/.gitlab/ci/jobs/test/ometrics_code_quality.yml new file mode 100644 index 0000000000000000000000000000000000000000..26ab7715bf3875bedcc69c04307fc7f3c690e9ee --- /dev/null +++ b/.gitlab/ci/jobs/test/ometrics_code_quality.yml @@ -0,0 +1,21 @@ +include: .gitlab/ci/jobs/test/common.yml + +ometrics-code-quality: + stage: test + extends: + - .default_settings_template + - .image_template__runtime_build_test_dependencies_template + - .template__code_quality + needs: [trigger] + allow_failure: true + interruptible: true + script: + - OMETRICS_GIT=$CI_MERGE_REQUEST_SOURCE_PROJECT_URL OMETRICS_BRANCH=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME make lint-ometrics-gitlab + artifacts: + expose_as: 'Code quality report' + when: always + reports: + codequality: $CODE_QUALITY_REPORT + paths: + - _reports/ + expire_in: 15 days diff --git a/.gitlab/ci/jobs/test/ometrics_code_quality_default.yml b/.gitlab/ci/jobs/test/ometrics_code_quality_default.yml new file mode 100644 index 0000000000000000000000000000000000000000..aa01c8b261d626189d3d8f04ef58627a82029554 --- /dev/null +++ b/.gitlab/ci/jobs/test/ometrics_code_quality_default.yml @@ -0,0 +1,20 @@ +include: .gitlab/ci/jobs/test/common.yml + +ometrics-code-quality-default: + stage: test + extends: + - .template__code_quality + # See https://gitlab.com/gitlab-org/gitlab/-/issues/215279 + # In short, GitLab requires a code quality report from the target branch. + # As ometrics tries to find code quality issues against the target branch + # (i.e. previously introduced issues will not be reported), we create a fake + # empty report on the default branch which is supposed to be the target branch + # in most cases. + needs: [trigger] + interruptible: true + script: + - mkdir -p _reports/ + - echo "[]" > $CODE_QUALITY_REPORT + artifacts: + paths: + - $CODE_QUALITY_REPORT diff --git a/.gitlab/ci/jobs/test/python_integration_tests.yml b/.gitlab/ci/jobs/test/python_integration_tests.yml new file mode 100644 index 0000000000000000000000000000000000000000..8cdf3bfac1370f4087e8e844653849c1a7e83b49 --- /dev/null +++ b/.gitlab/ci/jobs/test/python_integration_tests.yml @@ -0,0 +1,36 @@ +############################################################ +## Stage: run python integration tests ## +############################################################ + +include: .gitlab/ci/jobs/test/common.yml + +integration:static-binaries: + extends: + - .pytest_template + allow_failure: true + needs: + - build:static-x86_64-linux-binaries + dependencies: + - build:static-x86_64-linux-binaries + script: + - make -C ${CI_PROJECT_DIR}/.gitlab/ci test-static-binaries + +integration:pytest: + extends: + - .pytest_integration + # WARNING: if you increase the number of parallel jobs, you need to + # update test_coverage.yml with the new list of jobs. + parallel: 20 + variables: + TESTS: "tests_alpha" + +integration:pytest_examples: + extends: .pytest_template + script: + - make -C ${CI_PROJECT_DIR}/.gitlab/ci test-pytest-example + +integration:pytest_check_unknown_regression_files: + extends: + - .pytest_template + script: + - make -C ${CI_PROJECT_DIR}/.gitlab/ci test-pytest-check-regression-files \ No newline at end of file diff --git a/.gitlab/ci/jobs/test/semgrep.yml b/.gitlab/ci/jobs/test/semgrep.yml new file mode 100644 index 0000000000000000000000000000000000000000..6532232b7193e2abf6377ab90e649c756e21fd61 --- /dev/null +++ b/.gitlab/ci/jobs/test/semgrep.yml @@ -0,0 +1,17 @@ +semgrep: + extends: + - .default_settings_template + # We specify the image by hash to avoid flakiness. Indeed, if we took the + # latest release, then an update in the parser or analyser could result in new + # errors being found even if the code doesn't change. This would place the + # burden for fixing the code on the wrong dev (the devs who happen to open an + # MR coinciding with the semgrep update rather than the dev who wrote the + # infringing code in the first place). + # Update the hash in scripts/semgrep/README.md too when updating it here + # Last update: 20212-01-03 + image: returntocorp/semgrep-agent:sha-c6cd7cf + stage: test + needs: [trigger] + script: + - echo "OCaml code linting. For information on how to reproduce locally, check out scripts/semgrep/README.md" + - sh ./scripts/semgrep/lint-all-ocaml-sources.sh diff --git a/.gitlab/ci/test/tezt.yml b/.gitlab/ci/jobs/test/tezt.yml similarity index 82% rename from .gitlab/ci/test/tezt.yml rename to .gitlab/ci/jobs/test/tezt.yml index fd5a41d537fc42d2a681597f272143bb76f3b05c..c8f4e013c1034f2451b1a87e29e93c0b4528d6f6 100644 --- a/.gitlab/ci/test/tezt.yml +++ b/.gitlab/ci/jobs/test/tezt.yml @@ -1,3 +1,5 @@ +include: .gitlab/ci/jobs/test/common.yml + # We use the --job option to split tests into jobs of roughly the same # duration. This is based on a file that contains timings of test results, # generated with --record. To rebalance jobs, update this record with: @@ -37,7 +39,6 @@ tezt:static-binaries: # Expansion of .integration_template but without coverage. - .default_settings_template - .image_template__runtime_build_test_dependencies_template - - .rules_template__development - .tezt_template dependencies: # Fetch src/proto_*/parameters/*.json from build_x86_64 @@ -60,31 +61,11 @@ tezt:static-binaries: script: - make -C ${CI_PROJECT_DIR}/.gitlab/ci test-tezt-static-binaries -# Long Tezt tests are not ran in the CI, but we want them to type-check so that -# they can be built in the performance regression test framework executors. -tezt:build-long: - extends: - - .build_template - - .rules_template__development - stage: test - script: - - make -C ${CI_PROJECT_DIR}/.gitlab/ci test-tezt-build-long - -# Tests tezt itself -tezt:self_tests: - extends: - - .build_template - - .rules_template__development - stage: test - script: - - make -C ${CI_PROJECT_DIR}/.gitlab/ci test-tezt-self-test - # Note: if you reactivate this test and if you keep it manual, put it in the "manual" stage. # #tezt:manual:migration: # extends: # - .test_template -# - .rules_template__extended_test_pipeline # before_script: # - export TEZOS_CLIENT_UNSAFE_DISABLE_DISCLAIMER=Y # - curl -s https://api.github.com/repos/Phlogi/tezos-snapshots/releases/latest | jq -r ".assets[] | select(.name) | .browser_download_url" | grep roll | xargs wget -q diff --git a/.gitlab/ci/jobs/test/tezt_build_long.yml b/.gitlab/ci/jobs/test/tezt_build_long.yml new file mode 100644 index 0000000000000000000000000000000000000000..5b0c0b9ca762208a161cc88e64362f8f8e111daa --- /dev/null +++ b/.gitlab/ci/jobs/test/tezt_build_long.yml @@ -0,0 +1,8 @@ +# Long Tezt tests are not ran in the CI, but we want them to type-check so that +# they can be built in the performance regression test framework executors. +tezt:build-long: + extends: + - .build_template + stage: test + script: + - make -C ${CI_PROJECT_DIR}/.gitlab/ci test-tezt-build-long diff --git a/.gitlab/ci/jobs/test/tezt_self_tests.yml b/.gitlab/ci/jobs/test/tezt_self_tests.yml new file mode 100644 index 0000000000000000000000000000000000000000..cdf25d84240ed18e50eaaffaeb6500a81e19c3d7 --- /dev/null +++ b/.gitlab/ci/jobs/test/tezt_self_tests.yml @@ -0,0 +1,7 @@ +# Tests tezt itself +tezt:self_tests: + extends: + - .build_template + stage: test + script: + - make -C ${CI_PROJECT_DIR}/.gitlab/ci test-tezt-self-test diff --git a/.gitlab/ci/test/unit.yml b/.gitlab/ci/jobs/test/unit.yml similarity index 99% rename from .gitlab/ci/test/unit.yml rename to .gitlab/ci/jobs/test/unit.yml index 61030fad67151d4833ec1de76c931c83383db501..307a5be2ca0f7a28bdd922f2940f5c24414fddce 100644 --- a/.gitlab/ci/test/unit.yml +++ b/.gitlab/ci/jobs/test/unit.yml @@ -161,6 +161,7 @@ unit:alpha: script: - scripts/test_wrapper.sh $TEST_TARGETS ${!TEST_TARGETS} - ./scripts/ci/merge_coverage.sh + unit:non-proto-x86_64: extends: - .unit_test_template_x86_64_coverage diff --git a/.gitlab/ci/packaging.yml b/.gitlab/ci/packaging.yml deleted file mode 100644 index 247559395455b6a2ca1a54fa9b4c88976071df97..0000000000000000000000000000000000000000 --- a/.gitlab/ci/packaging.yml +++ /dev/null @@ -1,12 +0,0 @@ -opam:prepare: - extends: - - .default_settings_template - - .image_template__runtime_prebuild_dependencies_template - - .rules_template__trigger_opam_batch_1 - - .needs_template__wait_for_trigger_job - stage: packaging - script: - - make -C ${CI_PROJECT_DIR}/.gitlab/ci opam-prepare - artifacts: - paths: - - _opam-repo-for-release/ diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml new file mode 100644 index 0000000000000000000000000000000000000000..d2bd686919d1b496a2eae3736eed87f245104894 --- /dev/null +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -0,0 +1,43 @@ +include: + # Stage: sanity + - .gitlab/ci/jobs/sanity/sanity_ci.yml + - .gitlab/ci/jobs/sanity/docker_hadolint.yml + + # Stage: package + - .gitlab/ci/jobs/packaging/opam_prepare.yml + - .gitlab/ci/jobs/packaging/opam_package.yml + + # Stage: build + - .gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml + - .gitlab/ci/jobs/build/arm64.yml + - .gitlab/ci/jobs/build/x86_64.yml + - .gitlab/ci/jobs/build/get_contracts.yml + - .gitlab/ci/jobs/doc/build_all.yml + - .gitlab/ci/jobs/build/docker_amd64_test_manual.yml + - .gitlab/ci/jobs/build/docker_arm64_test_manual.yml + + # Stage: test + - .gitlab/ci/jobs/test/check_precommit_hook.yml + - .gitlab/ci/jobs/test/misc_checks.yml + - .gitlab/ci/jobs/test/misc_opam_checks.yml + - .gitlab/ci/jobs/test/ometrics_code_quality.yml + - .gitlab/ci/jobs/test/semgrep.yml + - .gitlab/ci/jobs/test/unit.yml + - .gitlab/ci/jobs/test/integration_compiler_rejections.yml + - .gitlab/ci/jobs/test/check_scripts.yml + - .gitlab/ci/jobs/test/python_integration_tests.yml + - .gitlab/ci/jobs/test/integration_pytest_old_protocols.yml + - .gitlab/ci/jobs/test/liquidity_baking_scripts-integrity.yml + - .gitlab/ci/jobs/test/tezt.yml + - .gitlab/ci/jobs/test/tezt_build_long.yml + - .gitlab/ci/jobs/test/tezt_self_tests.yml + + # Stage: test_coverage + # Only run on merge requests that do not have the label `ci--no-coverage` + - local: .gitlab/ci/jobs/coverage/coverage.yml + rules: + - if: '$CI_MERGE_REQUEST_LABELS !~ /(?:^|[,])ci--no-coverage(?:$|[,])/' + + # Stage: doc + - .gitlab/ci/jobs/doc/linkcheck.yml + - .gitlab/ci/jobs/doc/tests.yml \ No newline at end of file diff --git a/.gitlab/ci/pipelines/latest_release.yml b/.gitlab/ci/pipelines/latest_release.yml new file mode 100644 index 0000000000000000000000000000000000000000..8b61960acac297d286965c23cf4e7fcd43cb0f95 --- /dev/null +++ b/.gitlab/ci/pipelines/latest_release.yml @@ -0,0 +1,3 @@ +include: + # Stage: publish_release + - .gitlab/ci/jobs/publish/docker_promote_to_latest_release.yml \ No newline at end of file diff --git a/.gitlab/ci/pipelines/latest_release_test.yml b/.gitlab/ci/pipelines/latest_release_test.yml new file mode 100644 index 0000000000000000000000000000000000000000..2f99a76759ac7f85a61a77f358b098d0b682887a --- /dev/null +++ b/.gitlab/ci/pipelines/latest_release_test.yml @@ -0,0 +1,30 @@ +include: + # Stage: sanity + - .gitlab/ci/jobs/sanity/sanity_ci.yml + + # Stage: build + - .gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml + - .gitlab/ci/jobs/build/arm64.yml + - .gitlab/ci/jobs/build/x86_64.yml + - .gitlab/ci/jobs/build/get_contracts.yml + - .gitlab/ci/jobs/doc/build_all.yml + + + # Stage: test + - .gitlab/ci/jobs/test/check_precommit_hook.yml + - .gitlab/ci/jobs/test/misc_checks.yml + - .gitlab/ci/jobs/test/misc_opam_checks.yml + - .gitlab/ci/jobs/test/semgrep.yml + - .gitlab/ci/jobs/test/unit.yml + - .gitlab/ci/jobs/test/integration_compiler_rejections.yml + - .gitlab/ci/jobs/test/check_scripts.yml + - .gitlab/ci/jobs/test/python_integration_tests.yml + - .gitlab/ci/jobs/test/integration_pytest_old_protocols.yml + - .gitlab/ci/jobs/test/liquidity_baking_scripts-integrity.yml + - .gitlab/ci/jobs/test/tezt.yml + - .gitlab/ci/jobs/test/tezt_build_long.yml + - .gitlab/ci/jobs/test/tezt_self_tests.yml + + # Stage: doc + - .gitlab/ci/jobs/doc/linkcheck.yml + - .gitlab/ci/jobs/doc/tests.yml \ No newline at end of file diff --git a/.gitlab/ci/pipelines/master_branch.yml b/.gitlab/ci/pipelines/master_branch.yml new file mode 100644 index 0000000000000000000000000000000000000000..36a26c7bfc0d76f601f67f7e10b3a95845284799 --- /dev/null +++ b/.gitlab/ci/pipelines/master_branch.yml @@ -0,0 +1,23 @@ +include: + # Stage: build + - .gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml + - .gitlab/ci/jobs/build/static_arm64_linux_binaries.yml + - .gitlab/ci/jobs/build/arm64.yml + - .gitlab/ci/jobs/build/docker_amd64_release.yml + - .gitlab/ci/jobs/build/docker_arm64_release.yml + + # Stage: test + - .gitlab/ci/jobs/test/ometrics_code_quality_default.yml + + # Stage: test_coverage + - .gitlab/ci/jobs/coverage/coverage_default.yml + + # Stage: doc + - .gitlab/ci/jobs/doc/publish.yml + + # Stage: prepare_release + - .gitlab/ci/jobs/prepare_release/docker_merge_manifests_release.yml + + # Stage: publish_release + - .gitlab/ci/jobs/publish/deb_packages_unstable_amd64.yml + - .gitlab/ci/jobs/publish/rpm_packages_unstable_amd64.yml \ No newline at end of file diff --git a/.gitlab/ci/pipelines/release_tag.yml b/.gitlab/ci/pipelines/release_tag.yml new file mode 100644 index 0000000000000000000000000000000000000000..afcefdbfe8fbc8c77e46df6c7c97a7ba011f0bc6 --- /dev/null +++ b/.gitlab/ci/pipelines/release_tag.yml @@ -0,0 +1,17 @@ +include: + # Stage: build + - .gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml + - .gitlab/ci/jobs/build/static_arm64_linux_binaries.yml + - .gitlab/ci/jobs/build/docker_amd64_release.yml + - .gitlab/ci/jobs/build/docker_arm64_release.yml + + # Stage: prepare_release + - .gitlab/ci/jobs/prepare_release/docker_merge_manifests_release.yml + + # Stage: publish_release_gitlab + - .gitlab/ci/jobs/publish/gitlab.yml + + # Stage: publish_release + - .gitlab/ci/jobs/publish/opam.yml + - .gitlab/ci/jobs/publish/deb_packages_stable_amd64.yml + - .gitlab/ci/jobs/publish/rpm_packages_stable_amd64.yml \ No newline at end of file diff --git a/.gitlab/ci/pipelines/release_tag_test.yml b/.gitlab/ci/pipelines/release_tag_test.yml new file mode 100644 index 0000000000000000000000000000000000000000..0b52e3d4a2f3274bed50ff2e9b17f20cd7040de8 --- /dev/null +++ b/.gitlab/ci/pipelines/release_tag_test.yml @@ -0,0 +1,47 @@ +include: + # Stage: sanity + - .gitlab/ci/jobs/sanity/sanity_ci.yml + - .gitlab/ci/jobs/sanity/docker_hadolint.yml + + # Stage: build + - .gitlab/ci/jobs/build/arm64.yml + - .gitlab/ci/jobs/build/x86_64.yml + - .gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml + - .gitlab/ci/jobs/build/static_arm64_linux_binaries.yml + - .gitlab/ci/jobs/build/docker_amd64_test.yml + - .gitlab/ci/jobs/build/docker_arm64_test.yml + - .gitlab/ci/jobs/build/get_contracts.yml + - .gitlab/ci/jobs/doc/build_all.yml + + # Stage: Coverage + - .gitlab/ci/jobs/coverage/coverage_default.yml + + # Stage: test + - .gitlab/ci/jobs/test/check_precommit_hook.yml + - .gitlab/ci/jobs/test/misc_checks.yml + - .gitlab/ci/jobs/test/misc_opam_checks.yml + - .gitlab/ci/jobs/test/semgrep.yml + - .gitlab/ci/jobs/test/unit.yml + - .gitlab/ci/jobs/test/integration_compiler_rejections.yml + - .gitlab/ci/jobs/test/check_scripts.yml + - .gitlab/ci/jobs/test/python_integration_tests.yml + - .gitlab/ci/jobs/test/integration_pytest_old_protocols.yml + - .gitlab/ci/jobs/test/liquidity_baking_scripts-integrity.yml + - .gitlab/ci/jobs/test/tezt.yml + - .gitlab/ci/jobs/test/tezt_build_long.yml + - .gitlab/ci/jobs/test/tezt_self_tests.yml + + # Stage: prepare_release + - .gitlab/ci/jobs/prepare_release/docker_merge_manifests_test.yml + + # Stage: publish_release_gitlab + - .gitlab/ci/jobs/publish/gitlab.yml + + # Stage: publish_release + - .gitlab/ci/jobs/publish/opam.yml + - .gitlab/ci/jobs/publish/deb_packages_stable_amd64.yml + - .gitlab/ci/jobs/publish/rpm_packages_stable_amd64.yml + + # Stage: doc + - .gitlab/ci/jobs/doc/linkcheck.yml + - .gitlab/ci/jobs/doc/tests.yml \ No newline at end of file diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml new file mode 100644 index 0000000000000000000000000000000000000000..a153c85ba6e49d897f433a65fb14333952283d67 --- /dev/null +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -0,0 +1,20 @@ +include: + # Stage: build + - .gitlab/ci/jobs/build/x86_64.yml + - .gitlab/ci/jobs/build/arm64.yml + - .gitlab/ci/jobs/build/static_x86_64_linux_binaries.yml + + # Stage: test + - .gitlab/ci/jobs/test/integration_pytest_old_protocols.yml + - .gitlab/ci/jobs/test/ometrics_code_quality_default.yml + + # Stage: test_coverage + - .gitlab/ci/jobs/coverage/coverage_default.yml + + # Stage: packaging + - .gitlab/ci/jobs/packaging/opam_prepare.yml + - .gitlab/ci/jobs/packaging/opam_package.yml + + # Stage: doc + - .gitlab/ci/jobs/doc/linkcheck.yml + - .gitlab/ci/jobs/doc/tests.yml diff --git a/.gitlab/ci/publish_release.yml b/.gitlab/ci/publish_release.yml deleted file mode 100644 index f9eacde61e9d7ff85ba1915e6ecfbaf739ffd41e..0000000000000000000000000000000000000000 --- a/.gitlab/ci/publish_release.yml +++ /dev/null @@ -1,152 +0,0 @@ ---- -gitlab:release: - extends: - - .rules_template__release_tag - # https://gitlab.com/tezos/docker-images/ci-release - image: "${CI_REGISTRY}/tezos/docker-images/ci-release:v1.1.0" - stage: publish_release_gitlab - # Publish jobs are uninterruptible to avoid publishing partial results. - interruptible: false - dependencies: - - build:static-x86_64-linux-binaries - - build:static-arm64-linux-binaries - - docker:merge_manifests - script: - - make -C ${CI_PROJECT_DIR}/.gitlab/ci gitlab-release - -opam:release: - extends: - - .default_settings_template - - .image_template__runtime_build_test_dependencies_template - - .rules_template__release_tag - stage: publish_release - # Publish jobs are uninterruptible to avoid publishing partial results. - interruptible: false - dependencies: - - gitlab:release - script: - # create opam release and push to github (also pushes latest master branch) - - make -C ${CI_PROJECT_DIR}/.gitlab/ci opam-release - -# Note: here we rely on $IMAGE_ARCH_PREFIX to be empty. -# Otherwise, $DOCKER_IMAGE_TAG would contain $IMAGE_ARCH_PREFIX too. -# $IMAGE_ARCH_PREFIX is only used when building Docker images, -# here we handle all architectures so there is no such variable. -docker:merge_manifests: - extends: - - .rules_template__master_and_releases - - .image_template__docker - - .docker_registry_auth # Sets up a before_script - stage: prepare_release - script: - - ./scripts/ci/docker_merge_manifests.sh - -docker:promote_to_latest: - extends: - - .rules_template__latest_release - - .image_template__docker - - .docker_registry_auth # Sets up a before_script - stage: publish_release - script: - - ./scripts/ci/docker_promote_to_latest.sh - -# -# Templates for building deb & rpm packages. -# -# The AWS resources used for the packaging are managed on https://gitlab.com/nomadic-labs/iac/terraform/octez-packaging -# -# Specifically we use 2 S3 buckets to host the deb and rpm packages and one IAM role which has the ability to -# push changes. The jobs use the OpenID protocol to connect to AWS, receive temporary credentials and assume -# the IAM role. -# -# Details can be found on the following links: -# - https://docs.gitlab.com/ee/ci/cloud_services/index.html#configure-a-conditional-role-with-oidc-claims -# - https://docs.gitlab.com/ee/ci/cloud_services/aws/index.html - -.build_deb_packages: - stage: publish_release - image: ${CI_REGISTRY}/tezos/docker-images/ci-package:latest-ubuntu@sha256:76c894f509b84e106c9d056882c63cb497a2778f7c7dbbd1c73abe2568ee446e - needs: - - "build:static-x86_64-linux-binaries" - variables: - TEZOS_PACKAGING_VERSION: "872ac62526c49bd59b8621ee06c066141e940dbd" - TEZOS_PACKAGING_REPO: "https://github.com/serokell/tezos-packaging" - TEZOS_LICENSE_VERSION: "master" - ROLE_ARN: "arn:aws:iam::623103086665:role/octez-packaging-bucket-writer" - OCTEZ_DEB_BUCKET: "octez-deb-pkgs" - AWS_BUCKET_REGION: "eu-west-1" - PACKAGE_FORMAT: "deb" - ARCH: "amd64" - TEZOS_BINARIES: "octez-binaries/x86_64" - script: - - ./scripts/ci/create_linux_package.sh - - ./scripts/ci/upload_linux_package.sh - rules: - # NOTE: These jobs are disabled until the upstream scripts (https://github.com/serokell/tezos-packaging) - # provide support for the lima protocol and drop support for jakarta. - - when: never - -.build_rpm_packages: - stage: publish_release - image: ${CI_REGISTRY}/tezos/docker-images/ci-package:latest-fedora@sha256:35bfacc001e4c5247679e2f79c36ab1c83bd893a1e909c5c1afd1beff44c6895 - needs: - - "build:static-x86_64-linux-binaries" - variables: - TEZOS_PACKAGING_VERSION: "872ac62526c49bd59b8621ee06c066141e940dbd" - TEZOS_PACKAGING_REPO: "https://github.com/serokell/tezos-packaging" - TEZOS_LICENSE_VERSION: "master" - ROLE_ARN: "arn:aws:iam::623103086665:role/octez-packaging-bucket-writer" - OCTEZ_RPM_BUCKET: "octez-rpm-pkgs" - AWS_BUCKET_REGION: "eu-west-1" - PACKAGE_FORMAT: "rpm" - ARCH: "amd64" - TEZOS_BINARIES: "octez-binaries/x86_64" - script: - - ./scripts/ci/create_linux_package.sh - - ./scripts/ci/upload_linux_package.sh - rules: - # NOTE: These jobs are disabled until the upstream scripts (https://github.com/serokell/tezos-packaging) - # provide support for the lima protocol and drop support for jakarta. - - when: never - -# -# RPM packages -# - -rpm-packages-unstable-amd64: - extends: - - .rules_template__master - - .build_rpm_packages - variables: - CODENAME: "unstable" - TEZOS_VERSION: v0.0+$CI_COMMIT_SHORT_SHA - -rpm-packages-stable-amd64: - extends: - - .rules_template__release_tag - - .build_rpm_packages - variables: - CODENAME: "stable" - before_script: - - export TEZOS_VERSION=$(git tag --points-at HEAD | grep -oE '^v[0-9]{1,3}\.[0-9]{1,3}$' ||:) - -# -# Deb packages -# - -deb-packages-unstable-amd64: - extends: - - .rules_template__master - - .build_deb_packages - variables: - CODENAME: "unstable" - TEZOS_VERSION: v0.0+$CI_COMMIT_SHORT_SHA - -deb-packages-stable-amd64: - extends: - - .rules_template__release_tag - - .build_deb_packages - variables: - CODENAME: "stable" - before_script: - - export TEZOS_VERSION=$(git tag --points-at HEAD | grep -oE '^v[0-9]{1,3}\.[0-9]{1,3}$' ||:) diff --git a/.gitlab/ci/sanity.yml b/.gitlab/ci/sanity.yml deleted file mode 100644 index 7a296ec469dcf44f02351dbf474ccbe85d72e083..0000000000000000000000000000000000000000 --- a/.gitlab/ci/sanity.yml +++ /dev/null @@ -1,18 +0,0 @@ -sanity_ci: - extends: - - .default_settings_template - - .image_template__runtime_build_test_dependencies_template - - .rules_template__development - stage: sanity - script: - - make -C ${CI_PROJECT_DIR}/.gitlab/ci sanity-ci - -docker:hadolint: - extends: - - .default_settings_template - - .rules_template__development_dockerfile - image: hadolint/hadolint:2.9.3-debian - stage: sanity - script: - - hadolint build.Dockerfile - - hadolint Dockerfile diff --git a/.gitlab/ci/templates.yml b/.gitlab/ci/templates.yml deleted file mode 100644 index b8ae2ecda369912adddbe975a2b8c19afc81a4e5..0000000000000000000000000000000000000000 --- a/.gitlab/ci/templates.yml +++ /dev/null @@ -1,417 +0,0 @@ -variables: - # /!\ CI_REGISTRY is overriden to use a private Docker registry mirror in AWS ECR - # in GitLab namespaces `nomadic-labs` and `tezos` - ## This value MUST be the same as `opam_repository_tag` in `scripts/version.sh` - build_deps_image_version: ba1ace10fe5b100b54bc51ab67e7afd1967a4e5f - build_deps_image_name: "${CI_REGISTRY}/tezos/opam-repository" - GIT_STRATEGY: fetch - GIT_DEPTH: "1" - GET_SOURCES_ATTEMPTS: "2" - ARTIFACT_DOWNLOAD_ATTEMPTS: "2" - # Sets the number of tries before failing opam downloads. - OPAMRETRIES: "5" - - TEZOS_DEFAULT_BRANCH: "master" - TEZOS_DEFAULT_NAMESPACE: "tezos" - - # An addition to working around a bug in gitlab-runner's default - # unzipping implementation (https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27496), - # this setting cuts cache creation time. - FF_USE_FASTZIP: "true" - -# Basic, specialised, minimal, orthogonal templates - -# Some settings we want by default on all jobs that cannot be set -# through the key `default` in `.gitlab-ci.yml`. -.default_settings_template: - # `dependencies` are empty be default. Each individual job or - # template requiring artifacts from preceeding jobs should declare - # their dependencies locally. - dependencies: [] - -# Image templates -.image_template__runtime_build_test_dependencies_template: - image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} - -.image_template__runtime_build_dependencies_template: - image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} - -.image_template__runtime_prebuild_dependencies_template: - image: ${build_deps_image_name}:runtime-prebuild-dependencies--${build_deps_image_version} - -# Match GitLab executors version and directly use the Docker socket -# The Docker daemon is already configured, experimental features are enabled -# The following environment variables are already set: -# - BUILDKIT_PROGRESS -# - DOCKER_DRIVER -# - DOCKER_VERSION -# https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-socket-binding -.image_template__docker: - # https://gitlab.com/tezos/docker-images/ci-docker - image: "${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.6.0" - -.image_template__alpine: - image: alpine:3.15 - -# Rules template - -# Rules for the first pipeline job (gateway to make the whole pipeline manual for development) -.rules_template__trigger: - rules: - # Scheduled pipelines - - if: '$TZ_PIPELINE_KIND == "SCHEDULE"' - when: always - # Default branch in default namespace - - if: '$CI_COMMIT_BRANCH == $TEZOS_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: always - # Nomadic Labs Marge-bot: https://gitlab.com/nomadic-margebot - - if: '$CI_MERGE_REQUEST_ASSIGNEES =~ /nomadic-margebot/' - when: always - # Tags - - if: '$CI_COMMIT_TAG && ($CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE || $CI_PROJECT_NAMESPACE == "nomadic-labs")' - when: always - # 'vX-release' branches - - if: '($CI_COMMIT_BRANCH =~ /^v[0-9]{1,3}-release$/ || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^v[0-9]{1,3}-release$/) && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: always - # Release branches - - if: '$CI_COMMIT_REF_NAME == "latest-release" && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: always - # Test release branches - - if: '$CI_COMMIT_REF_NAME == "test-latest-release" && $CI_PROJECT_NAMESPACE == "nomadic-labs"' - when: always - - when: manual # default - -# Rule to add to all jobs which need to be run only if (manual) trigger job is completed -.needs_template__wait_for_trigger_job: - needs: [trigger] - -# Rules for all the jobs that need to be run on development branches (typically -# those that have an MR, but also some experiments, etc.) -.rules_template__development: - rules: - - if: '$TZ_PIPELINE_KIND == "SCHEDULE"' - when: never - - if: '$CI_COMMIT_BRANCH == $TEZOS_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: never - - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == $TEZOS_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: never - # No development jobs on test latest release - - if: '$CI_COMMIT_REF_NAME == "test-latest-release" && $CI_PROJECT_NAMESPACE == "nomadic-labs"' - when: never - # No development jobs on release branches - - if: '($CI_COMMIT_BRANCH =~ /-release$/ || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /-release$/) && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: never - # No development jobs on release tags - - if: '$CI_COMMIT_TAG && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: never - - when: on_success - -# Same as .rules_template__development, but for manual jobs. -.rules_template__development_manual: - rules: - - if: '$TZ_PIPELINE_KIND == "SCHEDULE"' - when: never - - if: '$CI_COMMIT_BRANCH == $TEZOS_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: never - - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == $TEZOS_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: never - # No development jobs on test latest release - - if: '$CI_COMMIT_REF_NAME == "test-latest-release" && $CI_PROJECT_NAMESPACE == "nomadic-labs"' - when: never - # No development jobs on release branches - - if: '($CI_COMMIT_BRANCH =~ /-release$/ || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /-release$/) && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: never - # No development jobs on release tags - - if: '$CI_COMMIT_TAG && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: never - - when: manual - -# Same as .rules_template__development, but for jobs that should also -# appear in the extended test pipeline -.rules_template__development_and_extended: - rules: - # Run in the scheduled, extended test pipeline - - if: '$TZ_PIPELINE_KIND == "SCHEDULE" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' - when: on_success - # If this is not the extended test pipeline (since the above rule - # did not trigger), then forbid on default branch - - if: '$CI_COMMIT_BRANCH == $TEZOS_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: never - # No development jobs on test latest release - - if: '$CI_COMMIT_REF_NAME == "test-latest-release" && $CI_PROJECT_NAMESPACE == "nomadic-labs"' - when: never - # No development jobs on release branches - - if: '($CI_COMMIT_BRANCH =~ /-release$/ || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /-release$/) && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: never - # No development jobs on release tags - - if: '$CI_COMMIT_TAG && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: never - - when: on_success - -# Rules for all the jobs that are run only for the master branch and the like -# (tags, releases, etc.) -.rules_template__master_and_releases: - rules: - - if: '$TZ_PIPELINE_KIND == "SCHEDULE"' - when: never - # Valid release tag: vX.Y or vX.Y-rcZ - - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+(?:\-rc\d+)?$/ && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: on_success - variables: - CI_DOCKER_HUB: "true" - # No jobs on latest release branches - - if: '($CI_COMMIT_REF_NAME == "latest-release" || $CI_COMMIT_REF_NAME == "test-latest-release")' - when: never - # Match 'vX-release' branches only - - if: '($CI_COMMIT_BRANCH =~ /v[0-9]{1,3}-release$/ || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /v[0-9]{1,3}-release$/) && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: on_success - variables: - CI_DOCKER_HUB: "true" - - if: '$CI_COMMIT_BRANCH == $TEZOS_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: on_success - variables: - CI_DOCKER_HUB: "true" - - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == $TEZOS_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: on_success - variables: - CI_DOCKER_HUB: "true" - # Testing - - if: '$CI_COMMIT_TAG && $CI_PROJECT_NAMESPACE == "nomadic-labs"' - when: on_success - - when: never - -# Rules for all the jobs that are run only for the master branch -.rules_template__master: - rules: - - if: '$TZ_PIPELINE_KIND == "SCHEDULE"' - when: never - - if: '$CI_COMMIT_BRANCH == $TEZOS_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: on_success - - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == $TEZOS_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: on_success - - when: never - -# Rules for all the jobs that are run only on release tags -.rules_template__release_tag: - rules: - # Valid release tag: vX.Y or vX.Y-rcZ - - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+(?:\-rc\d+)?$/ && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: on_success - # Testing - - if: '$CI_COMMIT_TAG && $CI_PROJECT_NAMESPACE == "nomadic-labs"' - when: on_success - - when: never - -# Rules for all the jobs that are run only on latest release branches -.rules_template__latest_release: - rules: - # Production: push to Docker Hub - - if: '$CI_COMMIT_REF_NAME == "latest-release" && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: on_success - variables: - CI_DOCKER_HUB: "true" - # Testing: push to GitLab container registry - - if: '$CI_COMMIT_REF_NAME == "test-latest-release" && $CI_PROJECT_NAMESPACE == "nomadic-labs"' - when: on_success - - when: never - -# Rules for jobs that should run all the time except on latest release branches -.rules_template__not_on_latest_release: - rules: - # No jobs on latest release branches - - if: '($CI_COMMIT_REF_NAME == "latest-release" || $CI_COMMIT_REF_NAME == "test-latest-release")' - when: never - - when: on_success - -# Rules for specific topics: doc, opam, etc. -.rules_template__development_documentation: - rules: - - if: '$TZ_PIPELINE_KIND == "SCHEDULE" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' - when: always - - if: '$CI_COMMIT_TAG && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: never - # No development jobs on latest release branches - - if: '$CI_COMMIT_REF_NAME == "test-latest-release" && $CI_PROJECT_NAMESPACE == "nomadic-labs"' - when: never - # No development jobs on release branches - - if: '($CI_COMMIT_BRANCH =~ /-release$/ || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /-release$/) && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: never - - if: '$CI_COMMIT_BRANCH == $TEZOS_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: never - - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == $TEZOS_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: never - - if: '$CI_MERGE_REQUEST_ID' - changes: - - docs/introduction/*.sh - - docs/developer/*.sh - - docs/scripts/test_doc_scripts.sh - when: on_success - # Run when there is label on the merge request - - if: '$CI_MERGE_REQUEST_LABELS =~ /(?:^|[,])ci--docs(?:$|[,])/' - when: on_success - - when: manual - allow_failure: true - -.rules_template__development_arm64: - rules: - - if: '$TZ_PIPELINE_KIND == "SCHEDULE" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' - when: always - # Run when there is label on the merge request - - if: '$CI_MERGE_REQUEST_LABELS =~ /(?:^|[,])ci--arm64(?:$|[,])/' - when: on_success - # No development jobs on test latest release - - if: '$CI_COMMIT_REF_NAME == "test-latest-release" && $CI_PROJECT_NAMESPACE == "nomadic-labs"' - when: never - # No development jobs on release branches - - if: '($CI_COMMIT_BRANCH =~ /-release$/ || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /-release$/) && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: never - # No development jobs on release tags - - if: '$CI_COMMIT_TAG && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: never - - when: manual - allow_failure: true - -.rules_template__development_docker: - rules: - # No development jobs on test latest release - - if: '$CI_COMMIT_REF_NAME == "test-latest-release" && $CI_PROJECT_NAMESPACE == "nomadic-labs"' - when: never - # No development jobs on release branches - - if: '($CI_COMMIT_BRANCH =~ /-release$/ || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /-release$/) && $CI_PROJECT_NAMESPACE == $TEZOS_DEFAULT_NAMESPACE' - when: never - # Run when there is label on the merge request - - if: '$CI_MERGE_REQUEST_LABELS =~ /(?:^|[,])ci--docker(?:$|[,])/' - when: on_success - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - when: manual - allow_failure: true - - when: never - -.rules_template__development_dockerfile: - rules: - # Only run on merge requests when Dockerfiles have changed - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - changes: - - build.Dockerfile - - Dockerfile - - when: never # default - - -# Rules for jobs that should appear in the extended test pipeline -.rules_template__extended_test_pipeline: - rules: - - if: '$TZ_PIPELINE_KIND == "SCHEDULE" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' - when: on_success - - if: '$CI_MERGE_REQUEST_LABELS =~ /(?:^|[,])ci--extended-tests(?:$|[,])/' - when: on_success - - when: never - -# Rules for jobs that should appear in the extended test pipeline or -# whenever the pytests change -.rules_template__extended_test_pipeline__pytest: - rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - changes: - - tests_python/**/* - when: on_success - - !reference [.rules_template__extended_test_pipeline, rules] - -.rules_template__merge_request_only: - rules: - # Only run on merge requests - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - when: on_success - - when: never # default - -.rules_template__tezos_default_branch_only: - rules: - - if: '$CI_COMMIT_BRANCH == $TEZOS_DEFAULT_BRANCH' - when: on_success - - when: never # default - -# Job templates - -# This template is used by the opam test jobs defined in opam-ci.yml -.opam_template: - extends: - - .default_settings_template - - .image_template__runtime_prebuild_dependencies_template - stage: packaging - # FIXME: https://gitlab.com/nomadic-labs/tezos/-/issues/663 - # FIXME: https://gitlab.com/nomadic-labs/tezos/-/issues/664 - # At the time of writing, the opam tests were quite flaky. - # Therefore, a retry was added. This should be removed once the - # underlying tests have been fixed. - retry: 2 - script: - - make -C ${CI_PROJECT_DIR}/.gitlab/ci opam-configure - needs: [opam:prepare] - dependencies: [opam:prepare] - after_script: - # Stores logs in opam_logs for artifacts and outputs an excerpt on failure. - - OPAM_LOGS=opam_logs ./scripts/ci/opam_handle_output.sh - artifacts: - paths: - - opam_logs/ - expire_in: 1 week - when: always - -# Add variables for bisect_ppx instrumentation -.template__coverage: - variables: - COVERAGE_OPTIONS: "--instrument-with bisect_ppx" - BISECT_FILE: "$CI_PROJECT_DIR/_coverage_output/" - SLACK_COVERAGE_CHANNEL: "C02PHBE7W73" - -# Merge coverage files after the execution -.template__coverage_files: - extends: .template__coverage - artifacts: - name: "coverage-files-$CI_JOB_ID" - paths: - - $BISECT_FILE - expire_in: 1 day - when: on_success - -.template__coverage_report: - extends: .template__coverage - stage: test_coverage - before_script: - # Load the environment poetry previously created in the docker image - # Give access to the Python dependencies/executables - - . "$HOME/.venv/bin/activate" - coverage: '/Coverage: ([^%]+%)/' - artifacts: - expose_as: 'Coverage report' - reports: - coverage_report: - coverage_format: cobertura - path: _coverage_report/cobertura.xml - paths: - - _coverage_report/ - - $BISECT_FILE - expire_in: 15 days - when: always - -.build_template: - extends: - - .default_settings_template - - .image_template__runtime_build_test_dependencies_template - - .template__coverage - - .rules_template__not_on_latest_release - stage: build - before_script: - # FIXME: https://gitlab.com/tezos/tezos/-/issues/2865 - - sudo chown -R $(id -u):$(id -g) $CI_PROJECT_DIR - - . ./scripts/version.sh - # Load the environment poetry previously created in the docker image. - # Give access to the Python dependencies/executables - - . $HOME/.venv/bin/activate - -.docker_registry_auth: - before_script: - - ./scripts/ci/docker_registry_auth.sh - -.template__code_quality: - variables: - CODE_QUALITY_REPORT: "_reports/gl-code-quality-report.json" diff --git a/.gitlab/ci/test/integration.yml b/.gitlab/ci/test/integration.yml deleted file mode 100644 index d085ac153fe0f05f53c4755d7db74e05f5b79e27..0000000000000000000000000000000000000000 --- a/.gitlab/ci/test/integration.yml +++ /dev/null @@ -1,127 +0,0 @@ -# integration:compiler-rejections does not -# require access to the binaries like the "true" integration tests -# below. Therefore, it does not extend the .integration_template. - -integration:compiler-rejections: - extends: .test_template - script: - - make -C ${CI_PROJECT_DIR}/.gitlab/ci integration-test-compiler-rejections - -############################################################ -## Stage: run scripts to check they are working properly ## -############################################################ - -# Note: those two jobs actually probably don't need the build_x86_64 job -# to have finished, but we don't want to start before build_x86_64 has finished either. -script:prepare_migration_test: - extends: .test_template - needs: ["build_x86_64"] - before_script: - - make - script: - - make -C ${CI_PROJECT_DIR}/.gitlab/ci integration-test-prepare-migration-test - -script:snapshot_alpha_and_link: - extends: .test_template - needs: ["build_x86_64"] - script: - - make -C ${CI_PROJECT_DIR}/.gitlab/ci integration-test-snapshot-alpha-and-link - -script:test-gen-genesis: - extends: - - .default_settings_template - - .image_template__runtime_build_test_dependencies_template - - .rules_template__development - - .needs_template__wait_for_trigger_job - stage: test - before_script: - - cd scripts/gen-genesis - script: - - dune build gen_genesis.exe - -script:test_release_versions: - extends: .test_template - script: - # FIXME: https://gitlab.com/tezos/tezos/-/issues/2865 - - sudo chown -R $(id -u):$(id -g) ${CI_PROJECT_DIR} - - ./scripts/test_release_version.sh - -############################################################ -## Stage: run python integration tests ## -############################################################ - -# definition for the environment to run all integration tests -# integration tests are run only on x86_64 architectures (for now) -.pytest_template: - extends: - - .test_template - dependencies: ["build_x86_64"] - # Start immediately after 'build_x86_64' and don't wait for - # intermediate stages to succeed - needs: ["build_x86_64"] - before_script: - # Load the environment poetry previously created in the docker image. - # Give access to the Python dependencies/executables - - . $HOME/.venv/bin/activate - - mkdir tests_python/tmp - - cd tests_python - -integration:static-binaries: - extends: - - .pytest_template - allow_failure: true - needs: - - build:static-x86_64-linux-binaries - dependencies: - - build:static-x86_64-linux-binaries - script: - - make -C ${CI_PROJECT_DIR}/.gitlab/ci test-static-binaries - -integration:pytest_check_unknown_regression_files: - extends: - - .pytest_template - script: - - make -C ${CI_PROJECT_DIR}/.gitlab/ci test-pytest-check-regression-files - -.pytest_integration: - extends: - - .pytest_template - - .template__coverage_files - # the number of jobs have been choosen to give jobs of maximal - # length ~10 minutes and to accommodate the addition of new protocol - # test suites without increasing wall time of the test stage. - variables: - TESTS: "" - script: - - make -C ${CI_PROJECT_DIR}/.gitlab/ci test-pytest - artifacts: - paths: - - tests_python/tmp/ - - tests_python/reports/*.xml - - $BISECT_FILE - reports: - junit: tests_python/reports/*.xml - expire_in: 7 day - when: always - -integration:pytest: - extends: - - .pytest_integration - # WARNING: if you increase the number of parallel jobs, you need to - # update test_coverage.yml with the new list of jobs. - parallel: 20 - variables: - TESTS: "tests_alpha" - -integration:pytest_old_protocols: - extends: - - .pytest_integration - - .rules_template__extended_test_pipeline__pytest - parallel: 16 - variables: - TESTS: "--ignore tests_alpha" - -integration:pytest_examples: - extends: .pytest_template - script: - - make -C ${CI_PROJECT_DIR}/.gitlab/ci test-pytest-example diff --git a/.gitlab/ci/test/lints.yml b/.gitlab/ci/test/lints.yml deleted file mode 100644 index 26e97be4030ee2c33b93bb084b188a556a79dbcf..0000000000000000000000000000000000000000 --- a/.gitlab/ci/test/lints.yml +++ /dev/null @@ -1,97 +0,0 @@ -misc_checks: - extends: - - .test_template - - .needs_template__wait_for_trigger_job - # These tests are not flaky at all so we overwrite the retry value - retry: 1 - script: - - make -C ${CI_PROJECT_DIR}/.gitlab/ci lint-misc-check - -misc_opam_checks: - extends: - - .test_template - - .needs_template__wait_for_trigger_job - script: - # checks that all deps of opam packages are already installed - - ./scripts/opam-check.sh - artifacts: - when: always - paths: - - opam_repo.patch - expire_in: 1 days - -semgrep: - extends: - - .default_settings_template - - .rules_template__development - - .needs_template__wait_for_trigger_job - # We specify the image by hash to avoid flakiness. Indeed, if we took the - # latest release, then an update in the parser or analyser could result in new - # errors being found even if the code doesn't change. This would place the - # burden for fixing the code on the wrong dev (the devs who happen to open an - # MR coinciding with the semgrep update rather than the dev who wrote the - # infringing code in the first place). - # Update the hash in scripts/semgrep/README.md too when updating it here - # Last update: 20212-01-03 - image: returntocorp/semgrep-agent:sha-c6cd7cf - stage: test - script: - - echo "OCaml code linting. For information on how to reproduce locally, check out scripts/semgrep/README.md" - - sh ./scripts/semgrep/lint-all-ocaml-sources.sh - -check_precommit_hook: - extends: - - .test_template - - .needs_template__wait_for_trigger_job - script: - - make -C ${CI_PROJECT_DIR}/.gitlab/ci check-precommit - -check_scripts_b58_prefix: - # Can be changed to a python image, but using the build docker image to keep - # in sync with the python version used for the tests - extends: - - .test_template - - .needs_template__wait_for_trigger_job - script: - - make -C ${CI_PROJECT_DIR}/.gitlab/ci check-scripts-b58-prefix - -ometrics-code-quality-default: - stage: test - extends: - - .template__code_quality - # See https://gitlab.com/gitlab-org/gitlab/-/issues/215279 - # In short, GitLab requires a code quality report from the target branch. - # As ometrics tries to find code quality issues against the target branch - # (i.e. previously introduced issues will not be reported), we create a fake - # empty report on the default branch which is supposed to be the target branch - # in most cases. - - .rules_template__tezos_default_branch_only - - .needs_template__wait_for_trigger_job - interruptible: true - script: - - mkdir -p _reports/ - - echo "[]" > $CODE_QUALITY_REPORT - artifacts: - paths: - - $CODE_QUALITY_REPORT - -ometrics-code-quality: - stage: test - extends: - - .default_settings_template - - .image_template__runtime_build_test_dependencies_template - - .template__code_quality - - .rules_template__merge_request_only - - .needs_template__wait_for_trigger_job - allow_failure: true - interruptible: true - script: - - OMETRICS_GIT=$CI_MERGE_REQUEST_SOURCE_PROJECT_URL OMETRICS_BRANCH=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME make lint-ometrics-gitlab - artifacts: - expose_as: 'Code quality report' - when: always - reports: - codequality: $CODE_QUALITY_REPORT - paths: - - _reports/ - expire_in: 15 days diff --git a/.gitlab/ci/test/templates.yml b/.gitlab/ci/test/templates.yml deleted file mode 100644 index b202ce6db56cdbe178bc1e715ef266bf5a21370b..0000000000000000000000000000000000000000 --- a/.gitlab/ci/test/templates.yml +++ /dev/null @@ -1,29 +0,0 @@ -.test_template: - extends: - - .default_settings_template - - .image_template__runtime_build_test_dependencies_template - - .rules_template__development - - .template__coverage - stage: test - before_script: - - . ./scripts/version.sh - # Load the environment poetry previously created in the docker image. - # Give access to the Python dependencies/executables - - . $HOME/.venv/bin/activate - retry: 2 - -# Definition for the environment to run all integration tests. -# This is also used by Tezt tests. -# In general we do not have to run make, which takes a while, -# because the binaries have been produced by the build job and are -# in the cache. But if they are not, we need to build them. -# Ideally we should also check that the baker / accuser / endorser -# exist (some tests use them) but their actual name depend on the protocol. -.integration_template: - extends: - - .test_template - - .template__coverage_files - dependencies: ["build_x86_64"] - # Start immediately after 'build_x86_64' and don't wait for - # intermediate stages to succeed - needs: ["build_x86_64"] diff --git a/.gitlab/ci/trigger.yml b/.gitlab/ci/trigger.yml deleted file mode 100644 index 231f9609c078a37b4a4a449b48d94abb17847897..0000000000000000000000000000000000000000 --- a/.gitlab/ci/trigger.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -# Note: The purpose of this job is to launch the CI manually in certain cases -# (the rules are defined in the rules_template__trigger fragment of the -# template.yml file). The objective is not to run computing when it is not -# necessary and the decision to do so belongs to the developer - -trigger: - extends: - - .image_template__runtime_build_test_dependencies_template - - .rules_template__trigger - stage: trigger - # This job is simple and fast - allow_failure: false - timeout: "10m" - script: - - echo 'Trigger pipeline 🤠' diff --git a/docs/developer/contributing-adding-a-new-opam-dependency.rst b/docs/developer/contributing-adding-a-new-opam-dependency.rst index 34fb63c97aac051f28e8911662ce890ee1ca2c1c..96b351281a84320e8c779775d1a1f3907f491744 100644 --- a/docs/developer/contributing-adding-a-new-opam-dependency.rst +++ b/docs/developer/contributing-adding-a-new-opam-dependency.rst @@ -104,7 +104,7 @@ In order to create the opam repository MR: - Create the opam repository MR from this branch. Fourth, back in your local copy of Tezos, **update the variables in the** -:src:`.gitlab/ci/templates.yml` **and** :src:`scripts/version.sh` **files**. Specifically, set +:src:`.gitlab-ci.yml` **and** :src:`scripts/version.sh` **files**. Specifically, set the ``build_deps_image_version`` and the ``opam_repository_tag`` variables to the hash of the ``HEAD`` commit of the opam repository MR. Commit this change with a title along the lines of “CI: use dependency diff --git a/docs/developer/protocol_release_checklist.rst b/docs/developer/protocol_release_checklist.rst index c6c4276335c9327460e855615a91529877fc4102..6626b7191a19802eba621a40f2e266635d8c2d96 100644 --- a/docs/developer/protocol_release_checklist.rst +++ b/docs/developer/protocol_release_checklist.rst @@ -62,11 +62,10 @@ The release of a new protocol requires some manual configuration in the CI: - The old ``n-1`` protocol must be removed from those being built and tested This is done by adding and removing the respective protocols to -``active_protocol_versions``, and by changing the ``build`` job in -:src:`.gitlab/ci/build.yml` to ``rm`` the appropriate older protocol. +``active_protocol_versions``, :src:`scripts/remove-old-protocols.sh` will ``rm`` the appropriate older protocol. Additionally, the ``unit:NNN_PROTONAME`` unit test jobs in -:src:`.gitlab/ci/test/unit.yml` must be updated to test the new protocols and stop +:src:`.gitlab/ci/jobs/test/unit.yml` must be updated to test the new protocols and stop testing the old ones, in the same manner as above. **Final Tests and sign-off:** diff --git a/docs/developer/testing.rst b/docs/developer/testing.rst index 3089af6c57cb46fe94ee709143aa0ede09b2d2b3..9209580800cb9a4ee6d1bf7708805d0514201485 100644 --- a/docs/developer/testing.rst +++ b/docs/developer/testing.rst @@ -57,9 +57,9 @@ Acceptance testing more costly and less amenable to automation than integration or system testing. It is often the final step in the testing process, performed before a release. However, in Tezos, acceptance testing - is decoupled from releases, and currently consists in manually running + is decoupled from releases, and currently consists in manually running a net of resilience tests on a regular base. These tests use various - testing frameworks. + testing frameworks. .. Inline testing @@ -509,7 +509,7 @@ pipeline. The grain used varies slightly for different types of tests: Python integration and regression tests - Python tests are grouped in a number of batch jobs (chosen in :src:`.gitlab/ci/test/integration.yml`). This number is + Python tests are grouped in a number of batch jobs (chosen in :src:`.gitlab/ci/jobs/test/python_integration_tests.yml`). This number is chosen to keep the duration of job each lower under 10 minutes on average, and to accommodate the addition of new protocol test suites. @@ -551,7 +551,7 @@ The OCaml package tests (Alcotest & QCheck) Any non-protocol tests located in a folder named ``src/**/test/`` will be picked up automatically by the CI. No intervention is necessary. - Protocol tests must be added to :src:`.gitlab/ci/test/unit.yml` under the + Protocol tests must be added to :src:`.gitlab/ci/jobs/test/unit.yml` under the protocol that they are testing. For example, to run a new protocol test for ``proto_XXX_YYYYYYYY``, add the corresponding ``src/proto_XXX_YYYYYYYY/lib_\*.test_proto`` to the ``unit:XXX_YYYYYYYY`` diff --git a/manifest/manifest.ml b/manifest/manifest.ml index a02d8b9dc4ef8866ba2e64c352c82901ec1cf9e3..61a753eb2a4315df4ddcc612b44a92d79e3fe3ff 100644 --- a/manifest/manifest.ml +++ b/manifest/manifest.ml @@ -2935,7 +2935,7 @@ let generate_opam_ci () = Target.iter_internal_by_opam (fun package_name _internal_pkgs -> let (_ : int) = compute_rank package_name in ()) ; - write ".gitlab/ci/opam-ci.yml" @@ fun fmt -> + write ".gitlab/ci/jobs/packaging/opam_package.yml" @@ fun fmt -> Format.fprintf fmt "# This file was automatically generated, do not edit.@." ; Format.fprintf fmt "# Edit file manifest/manifest.ml instead.@." ; (* Decide whether an opam package should be tested in the CI or @@ -2965,12 +2965,9 @@ let generate_opam_ci () = {|@..rules_template__trigger_opam_batch_%d: rules: # Run on scheduled builds. - - if: '$TZ_PIPELINE_KIND == "SCHEDULE" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' + - if: '$CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' when: delayed start_in: %d minutes - # Never run on branch pipelines for master. - - if: '$CI_COMMIT_BRANCH == $TEZOS_DEFAULT_BRANCH' - when: never # Run when there is label on the merge request - if: '$CI_MERGE_REQUEST_LABELS =~ /(?:^|[,])ci--opam(?:$|[,])/' when: delayed diff --git a/scripts/ci/create_gitlab_release.sh b/scripts/ci/create_gitlab_release.sh index dc0700f723a4444136a23ed306a9233fad8a8f3e..03639e3fceb97f34234fd0e3fc06424b1c61108c 100755 --- a/scripts/ci/create_gitlab_release.sh +++ b/scripts/ci/create_gitlab_release.sh @@ -27,7 +27,7 @@ else gitlab_package_url="https://${CI_SERVER_HOST}${web_path}" fi -if [ "${CI_PROJECT_NAMESPACE}" = "${TEZOS_DEFAULT_NAMESPACE}" ] +if [ "${CI_PROJECT_NAMESPACE}" = "master" ] then ## Production => Docker Hub docker_hub_path='tezos/tezos' diff --git a/scripts/ci/docker_registry_auth.sh b/scripts/ci/docker_registry_auth.sh index 414734d64ca3ce01154b2201ea40224134d0c7ee..7ebd60f2ac3ba8f21c5855711379753b6a320c4a 100755 --- a/scripts/ci/docker_registry_auth.sh +++ b/scripts/ci/docker_registry_auth.sh @@ -27,11 +27,10 @@ echo "CI_DOCKER_HUB=${CI_DOCKER_HUB:-}" echo "CI_PROJECT_NAME=${CI_PROJECT_NAME}" echo "CI_PROJECT_NAMESPACE=${CI_PROJECT_NAMESPACE}" echo "IMAGE_ARCH_PREFIX=${IMAGE_ARCH_PREFIX:-}" -echo "TEZOS_DEFAULT_NAMESPACE=${TEZOS_DEFAULT_NAMESPACE}" # CI_DOCKER_HUB is used to switch to Docker Hub if credentials are available with CI_DOCKER_AUTH # /!\ CI_DOCKER_HUB can be unset, CI_DOCKER_AUTH is only available on protected branches -if [ "${CI_DOCKER_HUB:-}" = 'true' ] && [ "${CI_PROJECT_NAMESPACE}" = "${TEZOS_DEFAULT_NAMESPACE}" ] && [ -n "${CI_DOCKER_AUTH:-}" ] +if [ "${CI_DOCKER_HUB:-}" = 'true' ] && [ "${CI_PROJECT_NAMESPACE}" = "master" ] && [ -n "${CI_DOCKER_AUTH:-}" ] then # Docker Hub docker_image_name="docker.io/${CI_PROJECT_PATH}-" diff --git a/scripts/declare-new-protocol-unit-test/main.ml b/scripts/declare-new-protocol-unit-test/main.ml index 393c8c1f19a085fce67ca78644b8b201044699e1..bc904c837c384d76a05bbfb72984404f20315882 100755 --- a/scripts/declare-new-protocol-unit-test/main.ml +++ b/scripts/declare-new-protocol-unit-test/main.ml @@ -35,9 +35,9 @@ let () = let proto_version = Sys.argv.(1) in let proto_short_hash = Sys.argv.(2) in let new_proto_name = proto_version ^ "_" ^ proto_short_hash in - let in_ch = open_in ".gitlab/ci/test/unit.yml" in + let in_ch = open_in ".gitlab/ci/jobs/test/unit.yml" in Fun.protect ~finally: (fun () -> close_in in_ch) @@ fun () -> - let out_ch = open_out ".gitlab/ci/test/unit2.yml" in + let out_ch = open_out ".gitlab/ci/jobs/test/unit2.yml" in Fun.protect ~finally: (fun () -> close_out out_ch) @@ fun () -> let output_line line = output_string out_ch line; @@ -72,7 +72,7 @@ let () = | exception End_of_file -> error "End of file reached before seeing unit:alpha: - check \ - .gitlab/ci/test/unit.yml" + .gitlab/ci/jobs/test/unit.yml" | "unit:alpha:" as line -> (* Found the job we were looking for, start replacing it. *) output_line (replace line); @@ -82,4 +82,4 @@ let () = find_unit_alpha () in find_unit_alpha (); - Sys.rename ".gitlab/ci/test/unit2.yml" ".gitlab/ci/test/unit.yml" + Sys.rename ".gitlab/ci/jobs/test/unit2.yml" ".gitlab/ci/jobs/test/unit.yml" diff --git a/scripts/lint.sh b/scripts/lint.sh index e8192f06c763b18b4da2d0fa8a025b5667b5f44d..8cd9059f74a1492476f7ef02fedf8206c67be455 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -154,7 +154,7 @@ update_gitlab_ci_yml () { # Check that a rule is not defined twice, which would result in the first # one being ignored. Gitlab linter doesn't warn for it # Job key `unified_coverage` is allowed to be duplicated because we use a conditional include - # on files `.gitlab/ci/test_coverage.yml` and `.gitlab/ci/test_coverage_default.yml` + # on files `.gitlab/ci/coverage/coverage.yml` and `.gitlab/ci/jobs/coverage_default.yml` repeated=$(find .gitlab-ci.yml .gitlab/ci/ -iname \*.yml -exec grep '^[^ #-]' \{\} \; \ | sort \ | grep -v unified_coverage \