diff --git a/.gitlab/ci/jobs/build/common.yml b/.gitlab/ci/jobs/build/common.yml index 063dbd7d87ff460bde5dd9889d19c54e4d89f3c0..98a8b9b274ec2628ecc2666ecb5bd67cde145777 100644 --- a/.gitlab/ci/jobs/build/common.yml +++ b/.gitlab/ci/jobs/build/common.yml @@ -17,61 +17,6 @@ expire_in: 1 day when: on_success -.oc.build_arm64: - extends: - - .oc.build - - .tags_template__build_arm64 - variables: - ARCH: "arm64" - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' - when: always - # Run when there is label on the merge request - - if: '$CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--arm64(?:$|,)/' - when: on_success - - changes: - - src/**/* - - .gitlab/**/* - - .gitlab-ci.yml - when: manual - allow_failure: true - -# similar to the build template above, this template -# compiles all binaries using the static profile. -.oc.build_static_binaries_template: - extends: - - .default_settings_template - - .image_template__runtime_build_dependencies - - .rules__octez_changes - stage: build - variables: - ARCH: "" - before_script: - - ./scripts/ci/take_ownership.sh - - eval $(opam env) - script: - - ./scripts/ci/build_static_binaries.sh - artifacts: - paths: - - "octez-binaries/$ARCH/*" - -.oc.docker:rust-toolchain_template: - extends: - - .image_template__docker - - .docker_auth_template - stage: build - variables: - # These image are not built for external use. - CI_DOCKER_HUB: "false" - # Handle docker initialization, if necessary, in - # './scripts/ci/docker_rust_toolchain_build.sh'. - before_script: [] - script: - - ./scripts/ci/docker_rust_toolchain_build.sh - artifacts: - reports: - dotenv: rust_toolchain_image_tag.env - .oc.docker:client-libs-dependencies_template: extends: - .image_template__docker @@ -89,17 +34,6 @@ reports: dotenv: client_libs_dependencies_image_tag.env -.oc.build_docker_release_template: - extends: - - .image_template__docker - - .docker_auth_template - - .needs__rust_toolchain - variables: - DOCKER_BUILD_TARGET: "without-evm-artifacts" - IMAGE_ARCH_PREFIX: "" - script: - - ./scripts/ci/docker_release.sh - .oc.build_x86_64: extends: - .oc.build @@ -110,21 +44,3 @@ - .needs__trigger variables: ARCH: "x86_64" - -.oc.build:static-x86_64-linux-binaries: - extends: - - .tags_template__build - - .oc.build_static_binaries_template - # Even though not many tests depend on static executables, some of those that do - # are limiting factors in the total duration of pipelines. - # So we start this job as early as possible, without waiting for sanity_ci. - - .needs__trigger - variables: - ARCH: "x86_64" - -.oc.build:static-arm64-linux-binaries: - extends: - - .oc.build_static_binaries_template - - .tags_template__build_arm64 - variables: - ARCH: "arm64" diff --git a/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-master.yml b/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-master.yml deleted file mode 100644 index 8633e3c6de5f9940a3c3df117ea8cabb4624aa00..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-master.yml +++ /dev/null @@ -1,22 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.build:static-arm64-linux-binaries: - image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} - stage: build - tags: - - gcp_arm64 - rules: - - when: always - dependencies: [] - before_script: - - ./scripts/ci/take_ownership.sh - - eval $(opam env) - script: - - ./scripts/ci/build_static_binaries.sh - variables: - ARCH: arm64 - EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables - artifacts: - paths: - - octez-binaries/$ARCH/* diff --git a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-master.yml b/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-master.yml deleted file mode 100644 index 8e721cb125d42bb8d61b90f7ab6678486eb6917d..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-master.yml +++ /dev/null @@ -1,25 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.build:static-x86_64-linux-binaries: - image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} - stage: build - tags: - - gcp - rules: - - when: always - needs: - - job: trigger - optional: true - dependencies: [] - before_script: - - ./scripts/ci/take_ownership.sh - - eval $(opam env) - script: - - ./scripts/ci/build_static_binaries.sh - variables: - ARCH: x86_64 - EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables - artifacts: - paths: - - octez-binaries/$ARCH/* diff --git a/.gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra.yml b/.gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra.yml index 01a9c940887600397f5c9804a515631d1ca4fb78..c6f4f4db5b28f2544a1ced9b10b6c6bb1bf23642 100644 --- a/.gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra.yml +++ b/.gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra.yml @@ -1,14 +1,41 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. -# The build_arm64 jobs are split in two to keep the artifact size -# under the 1GB hard limit set by GitLab. - -# 'oc.build_arm64-exp-dev-extra' builds the developer and experimental -# executables, as well as the tezt test suite and the TPS evaluation -# tool. oc.build_arm64-exp-dev-extra: - extends: - - .oc.build_arm64 + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS" + when: always + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--arm64(?:$|,)/ + when: on_success + - changes: + - src/**/* + - .gitlab/**/* + - .gitlab-ci.yml + when: manual + allow_failure: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/ci/build_full_unreleased.sh variables: - EXECUTABLE_FILES: "script-inputs/experimental-executables script-inputs/dev-executables" - BUILD_EXTRA: "src/bin_tps_evaluation/main_tps_evaluation.exe src/bin_octogram/octogram_main.exe tezt/tests/main.exe" + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/experimental-executables script-inputs/dev-executables + BUILD_EXTRA: src/bin_tps_evaluation/main_tps_evaluation.exe src/bin_octogram/octogram_main.exe + tezt/tests/main.exe + artifacts: + name: build-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-* + - src/proto_*/parameters/*.json + - _build/default/src/lib_protocol_compiler/bin/main_native.exe + - _build/default/tezt/tests/main.exe + - _build/default/contrib/octez_injector_server/octez_injector_server.exe + when: on_success diff --git a/.gitlab/ci/jobs/build/oc.build_arm64-released.yml b/.gitlab/ci/jobs/build/oc.build_arm64-released.yml index 4c947e6f3edcdc95351ef365d88287192ed1a7ca..287950d42cd9d18645553e1c7f882d2876b77dea 100644 --- a/.gitlab/ci/jobs/build/oc.build_arm64-released.yml +++ b/.gitlab/ci/jobs/build/oc.build_arm64-released.yml @@ -1,11 +1,39 @@ -include: .gitlab/ci/jobs/build/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. -# The build_arm64 jobs are split in two to keep the artifact size -# under the 1GB hard limit set by GitLab. - -# 'oc.build_arm64-released' builds the released executables. oc.build_arm64-released: - extends: - - .oc.build_arm64 + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS" + when: always + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--arm64(?:$|,)/ + when: on_success + - changes: + - src/**/* + - .gitlab/**/* + - .gitlab-ci.yml + when: manual + allow_failure: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/ci/build_full_unreleased.sh variables: - EXECUTABLE_FILES: "script-inputs/released-executables" + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + name: build-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-* + - src/proto_*/parameters/*.json + - _build/default/src/lib_protocol_compiler/bin/main_native.exe + - _build/default/tezt/tests/main.exe + - _build/default/contrib/octez_injector_server/octez_injector_server.exe + when: on_success diff --git a/.gitlab/ci/jobs/build/oc.docker:amd64-experimental.yml b/.gitlab/ci/jobs/build/oc.docker:amd64-experimental.yml deleted file mode 100644 index 4b1fc6f267026b6eea850b54c903fbcf9753ce43..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/build/oc.docker:amd64-experimental.yml +++ /dev/null @@ -1,44 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.docker:amd64: - image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 - stage: build - tags: - - gcp - rules: - - if: $CI_COMMIT_BRANCH == "master" - when: always - - changes: - - scripts/**/* - - script-inputs/**/* - - src/**/* - - tezt/**/* - - vendors/**/* - - dune - - dune-project - - dune-workspace - - opam - - Makefile - - kernels.mk - - build.Dockerfile - - Dockerfile - - .gitlab/**/* - - .gitlab-ci.yml - when: on_success - needs: - - oc.docker:rust-toolchain - dependencies: - - oc.docker:rust-toolchain - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - ./scripts/ci/docker_release.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - DOCKER_BUILD_TARGET: with-evm-artifacts - IMAGE_ARCH_PREFIX: amd64_ - CI_DOCKER_HUB: "true" - EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables diff --git a/.gitlab/ci/jobs/build/oc.docker:arm64-experimental.yml b/.gitlab/ci/jobs/build/oc.docker:arm64-experimental.yml deleted file mode 100644 index 31ae139c30ed1dcac67dadf5553b2673b83003f2..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/build/oc.docker:arm64-experimental.yml +++ /dev/null @@ -1,44 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.docker:arm64: - image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 - stage: build - tags: - - gcp_arm64 - rules: - - if: $CI_COMMIT_BRANCH == "master" - when: always - - changes: - - scripts/**/* - - script-inputs/**/* - - src/**/* - - tezt/**/* - - vendors/**/* - - dune - - dune-project - - dune-workspace - - opam - - Makefile - - kernels.mk - - build.Dockerfile - - Dockerfile - - .gitlab/**/* - - .gitlab-ci.yml - when: on_success - needs: - - oc.docker:rust-toolchain - dependencies: - - oc.docker:rust-toolchain - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - ./scripts/ci/docker_release.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - DOCKER_BUILD_TARGET: without-evm-artifacts - IMAGE_ARCH_PREFIX: arm64_ - CI_DOCKER_HUB: "true" - EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables diff --git a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-master.yml b/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-master.yml deleted file mode 100644 index 6bce59a27e949c97c62df38765d9a4649e0cf338..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/build/oc.docker:rust-toolchain-master.yml +++ /dev/null @@ -1,22 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -oc.docker:rust-toolchain: - image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 - stage: build - tags: - - gcp - rules: - - when: always - dependencies: [] - before_script: [] - script: - - ./scripts/ci/docker_rust_toolchain_build.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - CI_DOCKER_HUB: "false" - artifacts: - reports: - dotenv: rust_toolchain_image_tag.env diff --git a/.gitlab/ci/jobs/coverage/oc.unified_coverage-default.yml b/.gitlab/ci/jobs/coverage/oc.unified_coverage-default.yml deleted file mode 100644 index 9a57fa61af70cee7f94dbf1757d080e722ae25c5..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/coverage/oc.unified_coverage-default.yml +++ /dev/null @@ -1,29 +0,0 @@ -# This job fetches coverage files from the most recently merged MR on the default branch. -# It creates the html, summary and cobertura reports. It also provide a coverage % for the -# merge request. - -include: .gitlab/ci/jobs/coverage/common.yml - -oc.unified_coverage: - extends: - - .default_settings_template - - .image_template__runtime_build_test_dependencies - - .oc.template__coverage_report - dependencies: [] - variables: - # The GitLab project to use in the coverage downloader. - PROJECT: $CI_PROJECT_PATH - # The default branch where the coverage downloader will search for - # merge commits. - DEFAULT_BRANCH: $CI_COMMIT_SHA - before_script: - - . ./scripts/version.sh # sets COVERAGE_OUTPUT - script: - # On the project default branch, we fetch coverage from the last merged MR - - mkdir -p _coverage_report - - dune exec scripts/ci/download_coverage/download.exe -- -a from=last-merged-pipeline --info --log-file _coverage_report/download_coverage.log - - ./scripts/ci/report_coverage.sh - # This job will fail if coverage is not found, but that is not - # reason to stop the pipeline. - allow_failure: true - when: always diff --git a/.gitlab/ci/jobs/doc/publish:documentation.yml b/.gitlab/ci/jobs/doc/publish:documentation.yml deleted file mode 100644 index 0b92cdda552192fa11661c868900d9abc8c649f8..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/doc/publish:documentation.yml +++ /dev/null @@ -1,30 +0,0 @@ -include: .gitlab/ci/jobs/shared/templates.yml - -# here we use this hack to publish the tezos documentation on -# gitlab.io because we want to publish the doc for the project -# tezos under https://tezos.gitlab.io and not https://tezos.gitlab.io/tezos -# The latter follows the gitlab url convention of -# https://.gitlab.io// -# Notice that we push only if CI_COMMIT_REF_NAME is really master . -# This allows to test the release workflow -publish:documentation: - extends: - - .default_settings_template - - .image_template__runtime_build_test_dependencies - - .rules__octez_docs_changes - stage: doc - # Make the publish_documentation run in the beginning of the master - # pipeline to ensure it has time to run before the next merge. - needs: [] - before_script: - - eval $(opam env) - # Load the environment poetry previously created in the docker image. - # Give access to the Python dependencies/executables - - . $HOME/.venv/bin/activate - - echo "${CI_PK_GITLAB_DOC}" > ~/.ssh/id_ed25519 - - echo "${CI_KH}" > ~/.ssh/known_hosts - - chmod 400 ~/.ssh/id_ed25519 - script: - - ./scripts/ci/doc_publish.sh - # Publish jobs are uninterruptible to avoid publishing partial results. - interruptible: false diff --git a/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml b/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml deleted file mode 100644 index d1318ef9c8b4fd8a12e553284febd4805278e3bd..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml +++ /dev/null @@ -1,21 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -docker:merge_manifests: - image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 - stage: prepare_release - tags: - - gcp - needs: - - oc.docker:amd64 - - oc.docker:arm64 - dependencies: [] - before_script: - - ./scripts/ci/docker_initialize.sh - script: - - ./scripts/ci/docker_merge_manifests.sh - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.6 - CI_DOCKER_HUB: "true" diff --git a/.gitlab/ci/jobs/publish/publish_kernel_sdk.yml b/.gitlab/ci/jobs/publish/publish_kernel_sdk.yml deleted file mode 100644 index 32fd14baae48bb9ee559cf3a36ed1671d9a054d9..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/publish/publish_kernel_sdk.yml +++ /dev/null @@ -1,25 +0,0 @@ -# -# Smart Rollup: Kernel SDK -# -# See src/kernel_sdk/RELEASE.md for more information - -publish_kernel_sdk: - extends: - - .default_settings_template - - .image_template__rust_toolchain - - .needs__rust_toolchain - stage: manual - rules: - - when: manual - allow_failure: true - interruptible: false - script: - - make -f kernels.mk publish-sdk-deps - # Manually set SSL_CERT_DIR as default setting points to empty dir - - SSL_CERT_DIR=/etc/ssl/certs CC=clang make -f kernels.mk publish-sdk - variables: - CARGO_HOME: $CI_PROJECT_DIR/cargo - cache: - - key: kernels - paths: - - cargo/ diff --git a/.gitlab/ci/pipelines/master_branch.yml b/.gitlab/ci/pipelines/master_branch.yml index 4f28d8a75d5a148cb233407c8b9f72b8a00b9847..5ba80ed299115c0c6b7adf8831a5663b1c171ab5 100644 --- a/.gitlab/ci/pipelines/master_branch.yml +++ b/.gitlab/ci/pipelines/master_branch.yml @@ -1,21 +1,332 @@ -include: - # Stage: build - - .gitlab/ci/jobs/build/oc.docker:rust-toolchain-master.yml - - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-master.yml - - .gitlab/ci/jobs/build/oc.build:static-arm64-linux-binaries-master.yml - - .gitlab/ci/jobs/build/oc.build_arm64-exp-dev-extra.yml - - .gitlab/ci/jobs/build/oc.build_arm64-released.yml - - .gitlab/ci/jobs/build/oc.docker:amd64-experimental.yml - - .gitlab/ci/jobs/build/oc.docker:arm64-experimental.yml - - # Stage: test_coverage - - .gitlab/ci/jobs/coverage/oc.unified_coverage-default.yml - - # Stage: doc - - .gitlab/ci/jobs/doc/publish:documentation.yml - - # Stage: prepare_release - - .gitlab/ci/jobs/prepare_release/docker:merge_manifests-release.yml - - # Stage: manual - - .gitlab/ci/jobs/publish/publish_kernel_sdk.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.docker:rust-toolchain: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + rules: + - when: always + dependencies: [] + before_script: [] + script: + - ./scripts/ci/docker_rust_toolchain_build.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" + artifacts: + reports: + dotenv: rust_toolchain_image_tag.env + +oc.build:static-x86_64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp + rules: + - when: always + needs: + - job: trigger + optional: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + artifacts: + paths: + - octez-binaries/$ARCH/* + +oc.build:static-arm64-linux-binaries: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + rules: + - when: always + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + script: + - ./scripts/ci/build_static_binaries.sh + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + artifacts: + paths: + - octez-binaries/$ARCH/* + +oc.build_arm64-released: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS" + when: always + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--arm64(?:$|,)/ + when: on_success + - changes: + - src/**/* + - .gitlab/**/* + - .gitlab-ci.yml + when: manual + allow_failure: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/ci/build_full_unreleased.sh + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/released-executables + artifacts: + name: build-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-* + - src/proto_*/parameters/*.json + - _build/default/src/lib_protocol_compiler/bin/main_native.exe + - _build/default/tezt/tests/main.exe + - _build/default/contrib/octez_injector_server/octez_injector_server.exe + when: on_success + +oc.build_arm64-exp-dev-extra: + image: ${build_deps_image_name}:runtime-build-dependencies--${build_deps_image_version} + stage: build + tags: + - gcp_arm64 + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS" + when: always + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--arm64(?:$|,)/ + when: on_success + - changes: + - src/**/* + - .gitlab/**/* + - .gitlab-ci.yml + when: manual + allow_failure: true + dependencies: [] + before_script: + - ./scripts/ci/take_ownership.sh + - . ./scripts/version.sh + - eval $(opam env) + script: + - ./scripts/ci/build_full_unreleased.sh + variables: + ARCH: arm64 + EXECUTABLE_FILES: script-inputs/experimental-executables script-inputs/dev-executables + BUILD_EXTRA: src/bin_tps_evaluation/main_tps_evaluation.exe src/bin_octogram/octogram_main.exe + tezt/tests/main.exe + artifacts: + name: build-$ARCH-$CI_COMMIT_REF_SLUG + expire_in: 1 day + paths: + - octez-* + - src/proto_*/parameters/*.json + - _build/default/src/lib_protocol_compiler/bin/main_native.exe + - _build/default/tezt/tests/main.exe + - _build/default/contrib/octez_injector_server/octez_injector_server.exe + when: on_success + +oc.docker:amd64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp + rules: + - if: $CI_COMMIT_BRANCH == "master" + when: always + - changes: + - scripts/**/* + - script-inputs/**/* + - src/**/* + - tezt/**/* + - vendors/**/* + - dune + - dune-project + - dune-workspace + - opam + - Makefile + - kernels.mk + - build.Dockerfile + - Dockerfile + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: with-evm-artifacts + IMAGE_ARCH_PREFIX: amd64_ + CI_DOCKER_HUB: "true" + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + +oc.docker:arm64: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: build + tags: + - gcp_arm64 + rules: + - if: $CI_COMMIT_BRANCH == "master" + when: always + - changes: + - scripts/**/* + - script-inputs/**/* + - src/**/* + - tezt/**/* + - vendors/**/* + - dune + - dune-project + - dune-workspace + - opam + - Makefile + - kernels.mk + - build.Dockerfile + - Dockerfile + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_release.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + DOCKER_BUILD_TARGET: without-evm-artifacts + IMAGE_ARCH_PREFIX: arm64_ + CI_DOCKER_HUB: "true" + EXECUTABLE_FILES: script-inputs/released-executables script-inputs/experimental-executables + +oc.unified_coverage: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: test_coverage + tags: + - gcp + dependencies: [] + allow_failure: true + before_script: + - . ./scripts/version.sh + script: + - mkdir -p _coverage_report + - dune exec scripts/ci/download_coverage/download.exe -- -a from=last-merged-pipeline + --info --log-file _coverage_report/download_coverage.log + - ./scripts/ci/report_coverage.sh + variables: + PROJECT: $CI_PROJECT_PATH + DEFAULT_BRANCH: $CI_COMMIT_SHA + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + SLACK_COVERAGE_CHANNEL: C02PHBE7W73 + artifacts: + expire_in: 15 days + paths: + - _coverage_report/ + - $BISECT_FILE + reports: + coverage_report: + coverage_format: cobertura + path: _coverage_report/cobertura.xml + when: always + expose_as: Coverage report + when: always + coverage: '/Coverage: ([^%]+%)/' + +publish:documentation: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: doc + tags: + - gcp + rules: + - changes: + - scripts/**/*/ + - script-inputs/**/*/ + - src/**/* + - tezt/**/* + - vendors/**/* + - dune + - dune-project + - dune-workspace + - docs/**/* + - .gitlab/**/* + - .gitlab-ci.yml + when: on_success + needs: [] + dependencies: [] + interruptible: false + before_script: + - eval $(opam env) + - . $HOME/.venv/bin/activate + - echo "${CI_PK_GITLAB_DOC}" > ~/.ssh/id_ed25519 + - echo "${CI_KH}" > ~/.ssh/known_hosts + - chmod 400 ~/.ssh/id_ed25519 + script: + - ./scripts/ci/doc_publish.sh + +docker:merge_manifests: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: prepare_release + tags: + - gcp + needs: + - oc.docker:amd64 + - oc.docker:arm64 + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_merge_manifests.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "true" + +publish_kernel_sdk: + image: ${rust_toolchain_image_name}:${rust_toolchain_image_tag} + stage: manual + tags: + - gcp + rules: + - when: manual + allow_failure: false + needs: + - oc.docker:rust-toolchain + dependencies: + - oc.docker:rust-toolchain + allow_failure: true + cache: + key: kernels + paths: + - cargo/ + interruptible: false + script: + - make -f kernels.mk publish-sdk-deps + - SSL_CERT_DIR=/etc/ssl/certs CC=clang make -f kernels.mk publish-sdk + variables: + CARGO_HOME: $CI_PROJECT_DIR/cargo diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 524b8b0a973f99f5b17a389125a175d54b0c95d6..d2316aaf4d59fe7ba4ccf7c7018e5d79b7d37c3a 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -33,11 +33,11 @@ module Stages = struct let _test = Stage.register "test" - let _test_coverage = Stage.register "test_coverage" + let test_coverage = Stage.register "test_coverage" let _packaging = Stage.register "packaging" - let _doc = Stage.register "doc" + let doc = Stage.register "doc" let prepare_release = Stage.register "prepare_release" @@ -150,7 +150,7 @@ module Images = struct ~image_path: "${client_libs_dependencies_image_name}:${client_libs_dependencies_image_tag}" - let _rust_toolchain = + let rust_toolchain = (* Warning: we are relying on ill-specified behavior from GitLab that allows the expansion of dotenv variables (here: $rust_toolchain_image_tag) in the image field. @@ -207,6 +207,28 @@ let before_script ?(take_ownership = false) ?(source_version = false) @ toggle install_js_deps ". ./scripts/install_build_deps.js.sh" @ before_script +(** Add variable for bisect_ppx instrumentation. + + This template should be extended by jobs that build OCaml targets + that should be instrumented for coverage output. This set of job + includes build jobs (like [oc.build_x86_64_*]). It also includes + OCaml unit test jobs like [oc.unit:*-x86_64] as they build the test + runners before their execution. *) +let enable_coverage_instrumentation : tezos_job -> tezos_job = + Tezos_ci.append_variables + [("COVERAGE_OPTIONS", "--instrument-with bisect_ppx")] + +(** Add variable specifying coverage trace storage. + + This function should be applied to jobs that either produce (like + test jobs) or consume (like the [unified_coverage] job) coverage + traces. In addition to specifying the location of traces, setting + this variable also _enables_ coverage trace output for + instrumented binaries. *) +let enable_coverage_location : tezos_job -> tezos_job = + Tezos_ci.append_variables + [("BISECT_FILE", "$CI_PROJECT_DIR/_coverage_output/")] + let changeset_octez = [ "src/**/*"; @@ -222,6 +244,21 @@ let changeset_octez_or_kernels = ["images/**/*"; "scripts/ci/**/*"; "kernels.mk"; "etherlink.mk"] @ changeset_octez +let changeset_octez_docs = + [ + "scripts/**/*/"; + "script-inputs/**/*/"; + "src/**/*"; + "tezt/**/*"; + "vendors/**/*"; + "dune"; + "dune-project"; + "dune-workspace"; + "docs/**/*"; + ".gitlab/**/*"; + ".gitlab-ci.yml"; + ] + (* Dummy job. This fixes the "configuration must contain at least one @@ -243,33 +280,33 @@ let job_dummy : job = necessary and the decision to do so belongs to the developer ยง2: We also perform some fast sanity checks. *) -let trigger = - job_external - @@ job - ~__POS__ - ~image:Images.alpine - ~stage:Stages.trigger - ~allow_failure:No - ~rules: - [ - job_rule - ~if_:(If.not Rules.assigned_to_marge_bot) - ~allow_failure:No - ~when_:Manual - (); - job_rule ~when_:Always (); - ] - ~timeout:(Minutes 10) - ~name:"trigger" - [ - "echo 'Trigger pipeline!'"; - (* Check that [.gitlab-ci.yml]'s [build_deps_image_version] and - [scripts/version.sh]'s [opam_repository_tag] are the same. *) - "./scripts/ci/check_opam_repository_tag.sh"; - (* Check that the Alpine version of the trigger job's image - corresponds to the value in scripts/version.sh. *) - "./scripts/ci/check_alpine_version.sh"; - ] +let job_trigger = + job + ~__POS__ + ~image:Images.alpine + ~stage:Stages.trigger + ~allow_failure:No + ~rules: + [ + job_rule + ~if_:(If.not Rules.assigned_to_marge_bot) + ~allow_failure:No + ~when_:Manual + (); + job_rule ~when_:Always (); + ] + ~timeout:(Minutes 10) + ~name:"trigger" + [ + "echo 'Trigger pipeline!'"; + (* Check that [.gitlab-ci.yml]'s [build_deps_image_version] and + [scripts/version.sh]'s [opam_repository_tag] are the same. *) + "./scripts/ci/check_opam_repository_tag.sh"; + (* Check that the Alpine version of the trigger job's image + corresponds to the value in scripts/version.sh. *) + "./scripts/ci/check_alpine_version.sh"; + ] + |> job_external (** Helper to create jobs that uses the docker deamon. @@ -340,7 +377,7 @@ let job_build_static_binaries ~__POS__ ~arch ?(release = false) pipelines. So when requested through [needs_trigger] we start this job as early as possible, without waiting for sanity_ci. *) - if needs_trigger then Dependent [Optional trigger] else Staged [] + if needs_trigger then Dependent [Optional job_trigger] else Staged [] in job ?rules @@ -355,45 +392,24 @@ let job_build_static_binaries ~__POS__ ~arch ?(release = false) ~artifacts ["./scripts/ci/build_static_binaries.sh"] -let rules_static_build_master = [job_rule ~when_:Always ()] - let rules_static_build_other = [job_rule ~changes:changeset_octez ()] let _job_static_arm64_experimental = - job_external ~filename_suffix:"experimental" - @@ job_build_static_binaries - ~__POS__ - ~arch:Arm64 - ~rules:rules_static_build_other - () - -let _job_static_arm64_master = - job_external ~filename_suffix:"master" - @@ job_build_static_binaries - ~__POS__ - ~arch:Arm64 - ~rules:rules_static_build_master - () + job_build_static_binaries + ~__POS__ + ~arch:Arm64 + ~rules:rules_static_build_other + () + |> job_external ~filename_suffix:"experimental" let _job_static_x86_64_experimental = - job_external ~filename_suffix:"experimental" - @@ job_build_static_binaries - ~__POS__ - ~arch:Amd64 - ~needs_trigger:true - ~rules:rules_static_build_other - () - -let _job_static_x86_64_master = - job_external ~filename_suffix:"master" - @@ job_build_static_binaries - ~__POS__ - ~arch:Amd64 - (* TODO: this job doesn't actually need trigger and there is no - need to set it optional since we know this job is only on the master branch. *) - ~needs_trigger:true - ~rules:rules_static_build_master - () + job_build_static_binaries + ~__POS__ + ~arch:Amd64 + ~needs_trigger:true + ~rules:rules_static_build_other + () + |> job_external ~filename_suffix:"experimental" let job_docker_rust_toolchain ?rules ?dependencies ~__POS__ () = job_docker_authenticated @@ -411,23 +427,19 @@ let job_docker_rust_toolchain ?rules ?dependencies ~__POS__ () = ["./scripts/ci/docker_rust_toolchain_build.sh"] let _job_docker_rust_toolchain_before_merging = - job_external ~filename_suffix:"before_merging" - @@ job_docker_rust_toolchain - ~__POS__ - ~dependencies:(Dependent [Optional trigger]) - ~rules: - [ - job_rule ~changes:changeset_octez_or_kernels ~when_:On_success (); - job_rule ~when_:Manual (); - ] - () - -let _job_docker_rust_toolchain_master = - job_external ~filename_suffix:"master" - @@ job_docker_rust_toolchain ~__POS__ ~rules:[job_rule ~when_:Always ()] () + job_docker_rust_toolchain + ~__POS__ + ~dependencies:(Dependent [Optional job_trigger]) + ~rules: + [ + job_rule ~changes:changeset_octez_or_kernels ~when_:On_success (); + job_rule ~when_:Manual (); + ] + () + |> job_external ~filename_suffix:"before_merging" let _job_docker_rust_toolchain_other = - job_external ~filename_suffix:"other" @@ job_docker_rust_toolchain ~__POS__ () + job_docker_rust_toolchain ~__POS__ () |> job_external ~filename_suffix:"other" (** Type of Docker build jobs. @@ -534,17 +546,6 @@ let rules_octez_docker_changes_or_master = job_rule ~changes:changeset_octez_docker_changes_or_master (); ] -let job_docker_amd64_experimental : Tezos_ci.tezos_job = - job_docker_build - ~__POS__ - ~external_:true - (* TODO: when this is generated for a given pipeline, then the correct variant of [_job_docker_rust_toolchain_*] must be set. - For now we can set any variant to get the correct name of the need. *) - ~dependencies:(Dependent [Artifacts _job_docker_rust_toolchain_master]) - ~rules:rules_octez_docker_changes_or_master - ~arch:Amd64 - Experimental - let _job_docker_amd64_test_manual : Tezos_ci.tezos_job = job_docker_build ~__POS__ @@ -554,15 +555,6 @@ let _job_docker_amd64_test_manual : Tezos_ci.tezos_job = ~arch:Amd64 Test_manual -let job_docker_arm64_experimental : Tezos_ci.tezos_job = - job_docker_build - ~__POS__ - ~external_:true (* TODO: see above *) - ~dependencies:(Dependent [Artifacts _job_docker_rust_toolchain_master]) - ~rules:rules_octez_docker_changes_or_master - ~arch:Arm64 - Experimental - let _job_docker_arm64_test_manual : Tezos_ci.tezos_job = job_docker_build ~__POS__ @@ -589,21 +581,6 @@ let job_docker_merge_manifests ~__POS__ ~ci_docker_hub ~job_docker_amd64 ~variables:[("CI_DOCKER_HUB", Bool.to_string ci_docker_hub)] ["./scripts/ci/docker_merge_manifests.sh"] -let _job_docker_merge_manifests_release = - job_external ~filename_suffix:"release" - @@ job_docker_merge_manifests - ~__POS__ - ~ci_docker_hub:true - (* TODO: In theory, actually uses either release or - experimental variant of docker jobs depending on - pipeline. In practice, this does not matter as these jobs - have the same name in the generated files - ([oc.build:ARCH]). However, when the merge_manifest jobs - are created directly in the appropriate pipeline, the - correcty variant must be used. *) - ~job_docker_amd64:job_docker_amd64_experimental - ~job_docker_arm64:job_docker_arm64_experimental - type bin_package_target = Dpkg | Rpm let job_build_bin_package ?rules ~__POS__ ~name ?(stage = Stages.build) ~arch @@ -679,44 +656,44 @@ let job_build_bin_package ?rules ~__POS__ ~name ?(stage = Stages.build) ~arch ] let job_build_dpkg_amd64 = - job_external - @@ job_build_bin_package - ~__POS__ - ~name:"oc.build:dpkg:amd64" - ~target:Dpkg - ~arch:Tezos_ci.Amd64 - () + job_build_bin_package + ~__POS__ + ~name:"oc.build:dpkg:amd64" + ~target:Dpkg + ~arch:Tezos_ci.Amd64 + () + |> job_external let job_build_rpm_amd64 = - job_external - @@ job_build_bin_package - ~__POS__ - ~name:"oc.build:rpm:amd64" - ~target:Rpm - ~arch:Tezos_ci.Amd64 - () + job_build_bin_package + ~__POS__ + ~name:"oc.build:rpm:amd64" + ~target:Rpm + ~arch:Tezos_ci.Amd64 + () + |> job_external let _job_build_dpkg_amd64_manual = - job_external ~directory:"build" ~filename_suffix:"manual" - @@ job_build_bin_package - ~__POS__ - ~name:"oc.build:dpkg:amd64" - ~target:Dpkg - ~arch:Tezos_ci.Amd64 - ~rules:[job_rule ~when_:Manual ()] - ~stage:Stages.manual - () + job_build_bin_package + ~__POS__ + ~name:"oc.build:dpkg:amd64" + ~target:Dpkg + ~arch:Tezos_ci.Amd64 + ~rules:[job_rule ~when_:Manual ()] + ~stage:Stages.manual + () + |> job_external ~directory:"build" ~filename_suffix:"manual" let _job_build_rpm_amd64_manual = - job_external ~directory:"build" ~filename_suffix:"manual" - @@ job_build_bin_package - ~__POS__ - ~rules:[job_rule ~when_:Manual ()] - ~name:"oc.build:rpm:amd64" - ~target:Rpm - ~arch:Tezos_ci.Amd64 - ~stage:Stages.manual - () + job_build_bin_package + ~__POS__ + ~rules:[job_rule ~when_:Manual ()] + ~name:"oc.build:rpm:amd64" + ~target:Rpm + ~arch:Tezos_ci.Amd64 + ~stage:Stages.manual + () + |> job_external ~directory:"build" ~filename_suffix:"manual" (** Type of release tag pipelines. @@ -841,6 +818,141 @@ let release_tag_pipeline ?(test = false) release_tag_pipeline_type = | false, Release_tag -> [job_opam_release] | _ -> [] +let arm64_build_extra = + [ + "src/bin_tps_evaluation/main_tps_evaluation.exe"; + "src/bin_octogram/octogram_main.exe tezt/tests/main.exe"; + ] + +let amd64_build_extra = + [ + "src/bin_tps_evaluation/main_tps_evaluation.exe"; + "src/bin_octogram/octogram_main.exe"; + "tezt/tests/main.exe"; + "contrib/octez_injector_server/octez_injector_server.exe"; + ] + +let job_build_dynamic_binaries ?rules ~__POS__ ~arch ?(release = false) + ?(needs_trigger = false) () = + let arch_string = match arch with Amd64 -> "x86_64" | Arm64 -> "arm64" in + let name = + sf + "oc.build_%s-%s" + arch_string + (if release then "released" else "exp-dev-extra") + in + let executable_files = + if release then "script-inputs/released-executables" + else "script-inputs/experimental-executables script-inputs/dev-executables" + in + let build_extra = + match (release, arch) with + | true, _ -> None + | false, Amd64 -> Some amd64_build_extra + | false, Arm64 -> Some arm64_build_extra + in + let variables = + [("ARCH", arch_string); ("EXECUTABLE_FILES", executable_files)] + @ + match build_extra with + | Some build_extra -> [("BUILD_EXTRA", String.concat " " build_extra)] + | None -> [] + in + let artifacts = + artifacts + ~name:"build-$ARCH-$CI_COMMIT_REF_SLUG" + ~when_:On_success + ~expire_in:(Days 1) + (* TODO: [paths] can be refined based on [release] *) + [ + "octez-*"; + "src/proto_*/parameters/*.json"; + "_build/default/src/lib_protocol_compiler/bin/main_native.exe"; + "_build/default/tezt/tests/main.exe"; + "_build/default/contrib/octez_injector_server/octez_injector_server.exe"; + ] + in + let dependencies = + (* Even though not many tests depend on static executables, some + of those that do are limiting factors in the total duration of + pipelines. So when requested through [needs_trigger] we start + this job as early as possible, without waiting for + sanity_ci. *) + if needs_trigger then Dependent [Optional job_trigger] else Staged [] + in + let job = + job + ?rules + ~__POS__ + ~stage:Stages.build + ~arch + ~name + ~image:Images.runtime_build_dependencies + ~before_script: + (before_script + ~take_ownership:true + ~source_version:true + ~eval_opam:true + []) + ~variables + ~dependencies + ~artifacts + ["./scripts/ci/build_full_unreleased.sh"] + in + (* Disable coverage for arm64 *) + if arch = Amd64 then enable_coverage_instrumentation job else job + +let build_arm_rules = + [ + job_rule ~if_:Rules.schedule_extended_tests ~when_:Always (); + job_rule ~if_:Rules.(has_mr_label "ci--arm64") ~when_:On_success (); + job_rule + ~changes:["src/**/*"; ".gitlab/**/*"; ".gitlab-ci.yml"] + ~when_:Manual + ~allow_failure:Yes + (); + ] + +(* Write external files for build_arm64_jobs. + + Used in external pipelines [before_merging] and [schedule_extended_test]. *) +let job_build_arm64_release : tezos_job = + job_build_dynamic_binaries + ~__POS__ + ~arch:Arm64 + ~needs_trigger:false + ~release:true + ~rules:build_arm_rules + () + |> job_external + +let job_build_arm64_exp_dev_extra : tezos_job = + job_build_dynamic_binaries + ~__POS__ + ~arch:Arm64 + ~needs_trigger:false + ~release:false + ~rules:build_arm_rules + () + |> job_external + +let enable_coverage_report job : tezos_job = + job + |> Tezos_ci.add_artifacts + ~expose_as:"Coverage report" + ~reports: + (reports + ~coverage_report: + { + coverage_format = Cobertura; + path = "_coverage_report/cobertura.xml"; + } + ()) + ~expire_in:(Days 15) + ~when_:Always + ["_coverage_report/"; "$BISECT_FILE"] + |> Tezos_ci.append_variables [("SLACK_COVERAGE_CHANNEL", "C02PHBE7W73")] + (* Register pipelines types. Pipelines types are used to generate workflow rules and includes of the files where the jobs of the pipeline is defined. At the moment, all these pipelines are defined @@ -869,7 +981,161 @@ let () = "latest_release_test" If.(not_on_tezos_namespace && push && on_branch "latest-release-test") ~jobs:[job_docker_promote_to_latest ~ci_docker_hub:false] ; - register "master_branch" If.(on_tezos_namespace && push && on_branch "master") ; + register + "master_branch" + If.(on_tezos_namespace && push && on_branch "master") + ~jobs: + (let job_docker_rust_toolchain = + job_docker_rust_toolchain + ~__POS__ + ~rules:[job_rule ~when_:Always ()] + () + in + let job_docker_amd64_experimental : tezos_job = + job_docker_build + ~__POS__ + ~dependencies:(Dependent [Artifacts job_docker_rust_toolchain]) + ~rules:rules_octez_docker_changes_or_master + ~arch:Amd64 + Experimental + in + let job_docker_arm64_experimental : tezos_job = + job_docker_build + ~__POS__ + ~dependencies:(Dependent [Artifacts job_docker_rust_toolchain]) + ~rules:rules_octez_docker_changes_or_master + ~arch:Arm64 + Experimental + in + let job_docker_merge_manifests = + job_docker_merge_manifests + ~__POS__ + ~ci_docker_hub:true + (* TODO: In theory, actually uses either release or + experimental variant of docker jobs depending on + pipeline. In practice, this does not matter as these jobs + have the same name in the generated files + ([oc.build:ARCH]). However, when the merge_manifest jobs + are created directly in the appropriate pipeline, the + correcty variant must be used. *) + ~job_docker_amd64:job_docker_amd64_experimental + ~job_docker_arm64:job_docker_arm64_experimental + in + let job_static_arm64 = + job_build_static_binaries + ~__POS__ + ~arch:Arm64 + ~rules:[job_rule ~when_:Always ()] + () + in + let job_static_x86_64 = + job_build_static_binaries + ~__POS__ + ~arch:Amd64 + (* TODO: this job doesn't actually need trigger and there is no + need to set it optional since we know this job is only on the master branch. *) + ~needs_trigger:true + ~rules:[job_rule ~when_:Always ()] + () + in + let job_unified_coverage_default : tezos_job = + job + ~__POS__ + ~image:Images.runtime_build_test_dependencies + ~name:"oc.unified_coverage" + ~stage:Stages.test_coverage + ~variables: + [ + ("PROJECT", Predefined_vars.(show ci_project_path)); + ("DEFAULT_BRANCH", Predefined_vars.(show ci_commit_sha)); + ] + ~allow_failure:Yes + ~before_script: + ((* sets COVERAGE_OUTPUT *) + before_script ~source_version:true []) + ~when_:Always + ~coverage:"/Coverage: ([^%]+%)/" + [ + (* On the project default branch, we fetch coverage from the last merged MR *) + "mkdir -p _coverage_report"; + "dune exec scripts/ci/download_coverage/download.exe -- -a \ + from=last-merged-pipeline --info --log-file \ + _coverage_report/download_coverage.log"; + "./scripts/ci/report_coverage.sh"; + ] + |> enable_coverage_location |> enable_coverage_report + in + let job_publish_documentation : tezos_job = + job + ~__POS__ + ~name:"publish:documentation" + ~image:Images.runtime_build_test_dependencies + ~stage:Stages.doc + ~dependencies:(Dependent []) + ~before_script: + (before_script + ~eval_opam:true + (* Load the environment poetry previously created in the docker image. + Give access to the Python dependencies/executables. *) + ~init_python_venv:true + [ + {|echo "${CI_PK_GITLAB_DOC}" > ~/.ssh/id_ed25519|}; + {|echo "${CI_KH}" > ~/.ssh/known_hosts|}; + {|chmod 400 ~/.ssh/id_ed25519|}; + ]) + ~interruptible:false + ~rules:[job_rule ~changes:changeset_octez_docs ~when_:On_success ()] + ["./scripts/ci/doc_publish.sh"] + in + (* Smart Rollup: Kernel SDK + + See [src/kernel_sdk/RELEASE.md] for more information. *) + let job_publish_kernel_sdk : tezos_job = + job + ~__POS__ + ~name:"publish_kernel_sdk" + ~image:Images.rust_toolchain + ~stage:Stages.manual + ~rules: + [ + (* This job is in the last stage {!Stages.manual} so we + can disallow failure without blocking the pipeline. + Furthermore, unlike other manual jobs, this is not + an "optional" job for which failures are + tolerated. *) + job_rule ~when_:Manual ~allow_failure:No (); + ] + ~allow_failure:Yes + ~dependencies:(Dependent [Artifacts job_docker_rust_toolchain]) + ~interruptible:false + ~variables: + [("CARGO_HOME", Predefined_vars.(show ci_project_dir) // "cargo")] + ~cache:[{key = "kernels"; paths = ["cargo/"]}] + [ + "make -f kernels.mk publish-sdk-deps"; + (* Manually set SSL_CERT_DIR as default setting points to empty dir *) + "SSL_CERT_DIR=/etc/ssl/certs CC=clang make -f kernels.mk \ + publish-sdk"; + ] + in + [ + (* Stage: build *) + job_docker_rust_toolchain; + job_static_x86_64; + job_static_arm64; + job_build_arm64_release; + job_build_arm64_exp_dev_extra; + job_docker_amd64_experimental; + job_docker_arm64_experimental; + (* Stage: test_coverage *) + job_unified_coverage_default; + (* Stage: doc *) + job_publish_documentation; + (* Stage: prepare_release *) + job_docker_merge_manifests; + (* Stage: manual *) + job_publish_kernel_sdk; + ]) ; register "release_tag" If.(on_tezos_namespace && push && has_tag_match release_tag_re) diff --git a/ci/bin/tezos_ci.ml b/ci/bin/tezos_ci.ml index fbc33fab0daae8ccaa654d28fb5ad3c035dadd12..7b11ee7ffb82cb6ca946025e1dec4e4e58af6f5b 100644 --- a/ci/bin/tezos_ci.ml +++ b/ci/bin/tezos_ci.ml @@ -34,6 +34,10 @@ type tezos_job = { source_position : string * int * int * int; } +let map_job (tezos_job : tezos_job) + (f : Gitlab_ci.Types.job -> Gitlab_ci.Types.job) : tezos_job = + {tezos_job with job = f tezos_job.job} + let tezos_job_to_config_elements (j : tezos_job) = let source_comment = if Cli.config.inline_source_info then @@ -425,3 +429,103 @@ let job_external ?directory ?filename_suffix (tezos_job : tezos_job) : tezos_job filename ; Gitlab_ci.To_yaml.to_file ~header ~filename config ; tezos_job) + +let add_artifacts ?name ?expose_as ?reports ?expire_in ?when_ paths + (tezos_job : tezos_job) = + map_job tezos_job @@ fun (job : Gitlab_ci.Types.job) -> + match job.artifacts with + | None -> + { + job with + artifacts = Some (artifacts ?expose_as ?reports ?expire_in ?when_ paths); + } + | Some artifacts -> + let opt_merge_right o1 o2 = + match (o1, o2) with + | _, Some value -> Some value + | Some value, None -> Some value + | None, None -> None + in + let opt_combine o1 o2 f = + match (o1, o2) with + | None, opt | opt, None -> opt + | Some value1, Some value2 -> Some (f value1 value2) + in + let name = opt_merge_right artifacts.name name in + let expose_as = opt_merge_right artifacts.expose_as expose_as in + let expire_in = + opt_combine artifacts.expire_in expire_in @@ fun interval1 interval2 -> + (* This function gives a measure of the size of a duration in seconds. + It is only used to compare durations. *) + let interval_to_seconds = function + | Gitlab_ci.Types.Seconds n -> n + | Minutes n -> n * 60 + | Hours n -> n * 3600 + | Days n -> n * 24 * 3600 + | Weeks n -> n * 7 * 24 * 3600 + | Months n -> n * 31 * 7 * 24 * 3600 + | Years n -> n * 365 * 7 * 24 * 3600 + in + if interval_to_seconds interval1 > interval_to_seconds interval2 then + interval1 + else interval2 + in + let when_ = + opt_combine artifacts.when_ when_ @@ fun when1 when2 -> + if when1 = when2 then when1 else Always + in + let paths = + match paths with + | [] -> artifacts.paths + | _ -> Some (Option.value ~default:[] artifacts.paths @ paths) + in + let reports = + opt_combine artifacts.reports reports @@ fun reports1 reports2 -> + let opt_combine_fail field o1 o2 = + opt_combine o1 o2 @@ fun _v1 _v2 -> + failwith + "[add_artifacts] attempted to override existing %s report" + field + in + { + dotenv = opt_combine_fail "dotenv" reports1.dotenv reports2.dotenv; + junit = opt_combine_fail "junit" reports1.junit reports2.junit; + coverage_report = + opt_combine_fail + "coverage_report" + reports1.coverage_report + reports2.coverage_report; + } + in + { + job with + artifacts = Some {name; expose_as; expire_in; when_; paths; reports}; + } + +let append_variables ?(allow_overwrite = false) new_variables + (tezos_job : tezos_job) : tezos_job = + map_job tezos_job @@ fun job -> + let variables = + let old_variables, new_variables = + List.fold_left + (fun (old_variables, new_variables) (name, value) -> + let old_variables = + match List.assoc_opt name old_variables with + | Some old_value -> + if not allow_overwrite then + failwith + "[Tezos_ci.append_variables] attempted to overwrite the \ + variable '%s' (old value: '%s', new value: '%s')" + name + old_value + value ; + List.remove_assoc name old_variables + | None -> old_variables + in + (old_variables, (name, value) :: new_variables)) + (Option.value ~default:[] job.variables, []) + new_variables + in + old_variables @ List.rev new_variables + in + {job with variables = Some variables} diff --git a/ci/bin/tezos_ci.mli b/ci/bin/tezos_ci.mli index 222f59fa873812266b0e3edd35e1ce066e4f7659..3c47c00cbc85c14809adea43f8770af551e63a68 100644 --- a/ci/bin/tezos_ci.mli +++ b/ci/bin/tezos_ci.mli @@ -56,20 +56,20 @@ end (** A facility for registering pipelines. *) module Pipeline : sig - (* Register a pipeline. + (** Register a pipeline. - [register ?variables name rule] will register a pipeline [name] - that runs when [rule] is true. + [register ?variables name rule] will register a pipeline [name] + that runs when [rule] is true. - If [variables] is set, then these variables will be added to the - [workflow:] clause for this pipeline in the top-level [.gitlab-ci.yml]. + If [variables] is set, then these variables will be added to the + [workflow:] clause for this pipeline in the top-level [.gitlab-ci.yml]. - If [jobs] is not set, then the pipeline is a legacy, hand-written - .yml file, expected to be defined in - [.gitlab/ci/pipelines/NAME.yml]. If [jobs] is set, then the those - jobs will be generated to the same file when {!write} is - called. In both cases, this file will be included from the - top-level [.gitlab-ci.yml]. *) + If [jobs] is not set, then the pipeline is a legacy, hand-written + .yml file, expected to be defined in + [.gitlab/ci/pipelines/NAME.yml]. If [jobs] is set, then the those + jobs will be generated to the same file when {!write} is + called. In both cases, this file will be included from the + top-level [.gitlab-ci.yml]. *) val register : ?variables:Gitlab_ci.Types.variables -> ?jobs:tezos_job list -> @@ -225,3 +225,38 @@ val job : used to write another job. *) val job_external : ?directory:string -> ?filename_suffix:string -> tezos_job -> tezos_job + +(** Adds artifacts to a job without overriding, if possible, existing artifacts. + + - If the job already has an artifact with [old_name] and [name] is given, then + [name] is used. + - If the job already has an artifact exposed as [old_exposed_as] and + [exposed_as] is given, then [exposed_as] is used. + - If the job already has an artifact added [old_when] and + [when_] is given, the new artifact will be added: + - at [old_when] if [old_when = when_] + - at [Always] if [old_when <> when_] + - If the job already has an artifact that expires in [old_expires_in] and + [expires_in] is given, then the largest of the two durations is used. Note + that for the purpose of this comparison, we consider that a minute is 60 + seconds, that an hour is 60 minutes, that a week is 7 days, that a + month is 31 days, and that a year is 365 days. + - Individual fields of a {!report} cannot be combined: a run-time error is + raised if [reports] contains a report that is already set in the job. + - [paths] are appended to the set of paths in [job]. *) +val add_artifacts : + ?name:string -> + ?expose_as:string -> + ?reports:Gitlab_ci.Types.reports -> + ?expire_in:Gitlab_ci.Types.time_interval -> + ?when_:Gitlab_ci.Types.when_artifact -> + string list -> + tezos_job -> + tezos_job + +(** Append the variables [variables] to the variables of [job]. + + Raises [Failure] if any of the [variables] is already defined for + [job], unless [allow_overwrite] is true (default is [false]). *) +val append_variables : + ?allow_overwrite:bool -> Gitlab_ci.Types.variables -> tezos_job -> tezos_job