From 69029073b53156164734281426db50a6afea758b Mon Sep 17 00:00:00 2001 From: Romain Date: Thu, 31 Jul 2025 14:11:44 +0200 Subject: [PATCH 1/9] CI: migrate documentation:rst-check to Cacio --- .gitlab-ci.yml | 8 ++++ .gitlab/ci/pipelines/before_merging.yml | 46 +++++++++---------- .gitlab/ci/pipelines/documentation.daily.yml | 42 +++++++++++++++++ .gitlab/ci/pipelines/merge_train.yml | 46 +++++++++---------- .../ci/pipelines/schedule_extended_test.yml | 20 -------- ci/bin/code_verification.ml | 12 ----- ci/bin/documentation.ml | 29 ++++++++++++ ci/bin/main.ml | 2 + 8 files changed, 125 insertions(+), 80 deletions(-) create mode 100644 .gitlab/ci/pipelines/documentation.daily.yml create mode 100644 ci/bin/documentation.ml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ab36e8ba6d28..7e625463c8a5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,6 +37,10 @@ workflow: variables: PIPELINE_TYPE: teztale.test_release when: always + - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "documentation.daily" + variables: + PIPELINE_TYPE: documentation.daily + when: always - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_EVENT_TYPE != "merge_train" variables: @@ -254,6 +258,10 @@ include: - if: $CI_PROJECT_NAMESPACE != "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG =~ /^teztale-v\d+\.\d+$/ when: always +- local: .gitlab/ci/pipelines/documentation.daily.yml + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "documentation.daily" + when: always - local: .gitlab/ci/pipelines/before_merging.yml rules: - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "merge_request_event" diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 5356f8685028..79e066dfd459 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -375,30 +375,6 @@ check_rust_fmt: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' -documentation:rst-check: - image: ${ci_image_name_protected}/test:amd64--master - stage: sanity - tags: - - gcp - rules: - - changes: - - '**/*.rst' - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - when: on_success - needs: [] - dependencies: [] - timeout: 60 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . $HOME/.venv/bin/activate - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - make --silent -C docs sphinx-check - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - nix: image: nixos/nix:2.22.1 stage: sanity @@ -4678,6 +4654,28 @@ oc.script.docker_verify_image_amd64: DOCKER_VERSION: 24.0.7 IMAGE_ARCH_PREFIX: amd64_ +documentation.rst-check: + image: ${ci_image_name_protected}/test:amd64--master + stage: test + tags: + - gcp + rules: + - changes: + - '**/*.rst' + when: on_success + needs: [] + dependencies: [] + timeout: 60 minutes + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - . $HOME/.venv/bin/activate + - make --silent -C docs sphinx-check + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + teztale.build-arm64: image: ${ci_image_name}/build:${ci_image_tag} stage: test diff --git a/.gitlab/ci/pipelines/documentation.daily.yml b/.gitlab/ci/pipelines/documentation.daily.yml new file mode 100644 index 000000000000..f70f86d192a6 --- /dev/null +++ b/.gitlab/ci/pipelines/documentation.daily.yml @@ -0,0 +1,42 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +stages: +- start +- test + +datadog_pipeline_trace: + image: datadog/ci:v2.44.0 + stage: start + tags: + - gcp + dependencies: [] + allow_failure: true + timeout: 60 minutes + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - CI_MERGE_REQUEST_IID=${CI_MERGE_REQUEST_IID:-none} + - DATADOG_SITE=datadoghq.eu datadog-ci tag --level pipeline --tags pipeline_type:$PIPELINE_TYPE + --tags mr_number:$CI_MERGE_REQUEST_IID + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + +documentation.rst-check: + image: ${ci_image_name_protected}/test:amd64--master + stage: test + tags: + - gcp + needs: [] + dependencies: [] + timeout: 60 minutes + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - . $HOME/.venv/bin/activate + - make --silent -C docs sphinx-check + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index a6eea4e7b257..c5acad90d030 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -375,30 +375,6 @@ check_rust_fmt: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' -documentation:rst-check: - image: ${ci_image_name_protected}/test:amd64--master - stage: sanity - tags: - - gcp - rules: - - changes: - - '**/*.rst' - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - when: on_success - needs: [] - dependencies: [] - timeout: 60 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . $HOME/.venv/bin/activate - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - make --silent -C docs sphinx-check - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - nix: image: nixos/nix:2.22.1 stage: sanity @@ -4612,6 +4588,28 @@ oc.script.docker_verify_image_amd64: DOCKER_VERSION: 24.0.7 IMAGE_ARCH_PREFIX: amd64_ +documentation.rst-check: + image: ${ci_image_name_protected}/test:amd64--master + stage: test + tags: + - gcp + rules: + - changes: + - '**/*.rst' + when: on_success + needs: [] + dependencies: [] + timeout: 60 minutes + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - . $HOME/.venv/bin/activate + - make --silent -C docs sphinx-check + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + teztale.build-arm64: image: ${ci_image_name}/build:${ci_image_tag} stage: test diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 4fd634f6e96b..3c9c0fab7b5e 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -314,26 +314,6 @@ check_rust_fmt: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' -documentation:rst-check: - image: ${ci_image_name_protected}/test:amd64--master - stage: sanity - tags: - - gcp - rules: - - when: always - needs: [] - dependencies: [] - timeout: 60 minutes - interruptible: false - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . $HOME/.venv/bin/activate - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - make --silent -C docs sphinx-check - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - oc.build_arm64-released: image: ${ci_image_name}/build:${ci_image_tag} stage: build diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 3507dfe8de67..02342aa7ea3d 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -542,17 +542,6 @@ let jobs pipeline_type = ~rules:(make_rules ~dependent:true ~changes:changeset_rust_fmt_files ()) ["scripts/check-format-rust.sh"] in - let job_check_rst = - job - ~__POS__ - ~name:"documentation:rst-check" - ~image:Images.CI.test_master - ~dependencies - ~stage - ~rules:(make_rules ~changes:changeset_octez_docs_rst ()) - ~before_script:(before_script ~init_python_venv:true []) - ["make --silent -C docs sphinx-check"] - in let job_commit_titles : tezos_job = let allow_failure : allow_failure_job = match pipeline_type with Merge_train -> No | _ -> With_exit_codes [65] @@ -590,7 +579,6 @@ let jobs pipeline_type = job_oc_misc_checks; job_check_jsonnet; job_check_rust_fmt; - job_check_rst; ] @ mr_only_jobs in diff --git a/ci/bin/documentation.ml b/ci/bin/documentation.ml new file mode 100644 index 000000000000..b479c2bcefaa --- /dev/null +++ b/ci/bin/documentation.ml @@ -0,0 +1,29 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2025 Nomadic Labs. *) +(* *) +(*****************************************************************************) + +module CI = Cacio.Make (struct + let name = "documentation" + + let paths = ["**/*.rst"] +end) + +let job_rst_check = + CI.job + "rst-check" + ~__POS__ + ~image:Tezos_ci.Images.CI.test_master + ~stage:Test + ~description:"Check ReStructured Text files." + [". $HOME/.venv/bin/activate"; "make --silent -C docs sphinx-check"] + +let register () = + CI.register_before_merging_jobs [(Immediate, job_rst_check)] ; + CI.register_scheduled_pipeline + "daily" + ~description:"Daily tests to run for the documentation." + [(Auto, job_rst_check)] ; + () diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 573da6621b32..cd77e0bc50da 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -112,6 +112,8 @@ let () = Grafazos.register () let () = Teztale.register () +let () = Documentation.register () + (** {3 General pipelines} *) let () = -- GitLab From a77e9a1bc287ffc9b6fe7917c136b3b8bc1ae2b4 Mon Sep 17 00:00:00 2001 From: Romain Date: Thu, 31 Jul 2025 15:11:25 +0200 Subject: [PATCH 2/9] CI: migrate python installation script test to Cacio --- .gitlab/ci/pipelines/before_merging.yml | 55 ++++++++-------- .gitlab/ci/pipelines/documentation.daily.yml | 51 +++++++++++++++ .gitlab/ci/pipelines/merge_train.yml | 55 ++++++++-------- .../ci/pipelines/schedule_extended_test.yml | 54 --------------- ci/bin/code_verification.ml | 65 +------------------ ci/bin/documentation.ml | 57 +++++++++++++++- 6 files changed, 160 insertions(+), 177 deletions(-) diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 79e066dfd459..97f835ac5213 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -4131,34 +4131,6 @@ oc.unified_coverage: expose_as: Coverage report coverage: '/Coverage: ([^%]+%)/' -documentation:install_python_bookworm: - image: debian:bookworm - stage: test - tags: - - gcp - rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - - changes: - - docs/developer/install-python-debian-ubuntu.sh - - poetry.lock - - pyproject.toml - when: on_success - - when: manual - allow_failure: true - needs: - - trigger - dependencies: [] - timeout: 60 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - ./docs/developer/install-python-debian-ubuntu.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} - ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - documentation:odoc: image: ${ci_image_name}/test:${ci_image_tag} stage: build @@ -4676,6 +4648,33 @@ documentation.rst-check: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' +documentation.install_python_bookworm: + image: debian:bookworm + stage: test + tags: + - gcp + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success + - changes: + - docs/developer/install-python-debian-ubuntu.sh + - poetry.lock + - pyproject.toml + when: on_success + needs: + - trigger + dependencies: [] + timeout: 60 minutes + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - ./docs/developer/install-python-debian-ubuntu.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} + ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + teztale.build-arm64: image: ${ci_image_name}/build:${ci_image_tag} stage: test diff --git a/.gitlab/ci/pipelines/documentation.daily.yml b/.gitlab/ci/pipelines/documentation.daily.yml index f70f86d192a6..b9449efde04c 100644 --- a/.gitlab/ci/pipelines/documentation.daily.yml +++ b/.gitlab/ci/pipelines/documentation.daily.yml @@ -40,3 +40,54 @@ documentation.rst-check: - make --silent -C docs sphinx-check after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + +documentation.install_python_bookworm: + image: debian:bookworm + stage: test + tags: + - gcp + needs: [] + dependencies: [] + timeout: 60 minutes + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - ./docs/developer/install-python-debian-ubuntu.sh tezos/tezos master + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + +documentation.install_python_jammy: + image: public.ecr.aws/lts/ubuntu:22.04_stable + stage: test + tags: + - gcp + needs: [] + dependencies: [] + timeout: 60 minutes + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - ./docs/developer/install-python-debian-ubuntu.sh tezos/tezos master + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + +documentation.install_python_noble: + image: public.ecr.aws/lts/ubuntu:24.04_stable + stage: test + tags: + - gcp + needs: [] + dependencies: [] + timeout: 60 minutes + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - ./docs/developer/install-python-debian-ubuntu.sh tezos/tezos master + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index c5acad90d030..6a1e47371860 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -4130,34 +4130,6 @@ oc.unified_coverage: expose_as: Coverage report coverage: '/Coverage: ([^%]+%)/' -documentation:install_python_bookworm: - image: debian:bookworm - stage: test - tags: - - gcp - rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - - changes: - - docs/developer/install-python-debian-ubuntu.sh - - poetry.lock - - pyproject.toml - when: on_success - - when: manual - allow_failure: true - needs: - - trigger - dependencies: [] - timeout: 60 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - ./docs/developer/install-python-debian-ubuntu.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} - ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - documentation:odoc: image: ${ci_image_name}/test:${ci_image_tag} stage: build @@ -4610,6 +4582,33 @@ documentation.rst-check: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' +documentation.install_python_bookworm: + image: debian:bookworm + stage: test + tags: + - gcp + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success + - changes: + - docs/developer/install-python-debian-ubuntu.sh + - poetry.lock + - pyproject.toml + when: on_success + needs: + - trigger + dependencies: [] + timeout: 60 minutes + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - ./docs/developer/install-python-debian-ubuntu.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} + ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + teztale.build-arm64: image: ${ci_image_name}/build:${ci_image_tag} stage: test diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 3c9c0fab7b5e..72c279e08ce0 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -3649,60 +3649,6 @@ tezt-riscv-slow-sequential: when: always retry: 2 -documentation:install_python_noble: - image: public.ecr.aws/lts/ubuntu:24.04_stable - stage: test - tags: - - gcp - rules: - - when: always - dependencies: [] - timeout: 60 minutes - interruptible: false - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - ./docs/developer/install-python-debian-ubuntu.sh tezos/tezos master - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - -documentation:install_python_jammy: - image: public.ecr.aws/lts/ubuntu:22.04_stable - stage: test - tags: - - gcp - rules: - - when: always - dependencies: [] - timeout: 60 minutes - interruptible: false - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - ./docs/developer/install-python-debian-ubuntu.sh tezos/tezos master - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - -documentation:install_python_bookworm: - image: debian:bookworm - stage: test - tags: - - gcp - rules: - - when: always - dependencies: [] - timeout: 60 minutes - interruptible: false - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - ./docs/developer/install-python-debian-ubuntu.sh tezos/tezos master - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - documentation:odoc: image: ${ci_image_name}/test:${ci_image_tag} stage: build diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 02342aa7ea3d..512685248674 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -1958,69 +1958,6 @@ let jobs pipeline_type = (* Doc jobs *) let doc = - let jobs_install_python = - (* Creates a job that tests installation of the python environment in [image] *) - let job_install_python ~__POS__ ~name ~image ~project ~branch = - job - ~__POS__ - ~name - ~image - ~stage:Stages.test - ~dependencies:dependencies_needs_start - ~rules: - (make_rules - ~changes: - (Changeset.make - [ - "docs/developer/install-python-debian-ubuntu.sh"; - "pyproject.toml"; - "poetry.lock"; - ]) - ~manual:Yes - ~label:"ci--docs" - ()) - [ - sf - "./docs/developer/install-python-debian-ubuntu.sh %s %s" - project - branch; - ] - in - (* The set of python installation test jobs. Since python is - today less used, we do the bulk of the tests in scheduled pipelines - and we only test debian_bookworm in a merge pipeline *) - match pipeline_type with - | Schedule_extended_test -> - [ - job_install_python - ~__POS__ - ~name:"documentation:install_python_noble" - ~image:Images.ubuntu_noble - ~project:"tezos/tezos" - ~branch:"master"; - job_install_python - ~__POS__ - ~name:"documentation:install_python_jammy" - ~image:Images.ubuntu_jammy - ~project:"tezos/tezos" - ~branch:"master"; - job_install_python - ~__POS__ - ~name:"documentation:install_python_bookworm" - ~image:Images.debian_bookworm - ~project:"tezos/tezos" - ~branch:"master"; - ] - | Before_merging | Merge_train -> - [ - job_install_python - ~__POS__ - ~name:"documentation:install_python_bookworm" - ~image:Images.debian_bookworm - ~project:"${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos}" - ~branch:"${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master}"; - ] - in let jobs_documentation : tezos_job list = let rules = make_rules ~changes:changeset_octez_docs ~label:"ci--docs" () @@ -2079,7 +2016,7 @@ let jobs pipeline_type = job_documentation_linkcheck; ] in - jobs_install_python @ jobs_documentation + jobs_documentation in (* Manual jobs *) diff --git a/ci/bin/documentation.ml b/ci/bin/documentation.ml index b479c2bcefaa..42f01f30b421 100644 --- a/ci/bin/documentation.ml +++ b/ci/bin/documentation.ml @@ -5,10 +5,23 @@ (* *) (*****************************************************************************) +module Files = struct + let rst = ["**/*.rst"] + + let python_install_script = + [ + "docs/developer/install-python-debian-ubuntu.sh"; + "pyproject.toml"; + "poetry.lock"; + ] + + let all = rst @ python_install_script +end + module CI = Cacio.Make (struct let name = "documentation" - let paths = ["**/*.rst"] + let paths = Files.all end) let job_rst_check = @@ -18,12 +31,50 @@ let job_rst_check = ~image:Tezos_ci.Images.CI.test_master ~stage:Test ~description:"Check ReStructured Text files." + ~only_if_changed:Files.rst [". $HOME/.venv/bin/activate"; "make --silent -C docs sphinx-check"] +let job_install_python = + Cacio.parameterize @@ fun os_distribution -> + Cacio.parameterize @@ fun branch -> + let os_distribution_name, image = + match os_distribution with + | `ubuntu_noble -> ("noble", Tezos_ci.Images.ubuntu_noble) + | `ubuntu_jammy -> ("jammy", Tezos_ci.Images.ubuntu_jammy) + | `debian_bookworm -> ("bookworm", Tezos_ci.Images.debian_bookworm) + in + let project, branch = + match branch with + | `master -> ("tezos/tezos", "master") + | `current_branch -> + ( "${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos}", + "${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master}" ) + in + CI.job + ("install_python_" ^ os_distribution_name) + ~__POS__ + ~image + ~stage:Test + ~description: + "Check that the Python installation script from the documentation \ + actually work." + ~only_if_changed:Files.python_install_script + ~force_if_label:["ci--docs"] + [sf "./docs/developer/install-python-debian-ubuntu.sh %s %s" project branch] + let register () = - CI.register_before_merging_jobs [(Immediate, job_rst_check)] ; + CI.register_before_merging_jobs + [ + (Immediate, job_rst_check); + (Auto, job_install_python `debian_bookworm `current_branch); + ] ; CI.register_scheduled_pipeline "daily" ~description:"Daily tests to run for the documentation." - [(Auto, job_rst_check)] ; + [ + (Auto, job_rst_check); + (Auto, job_install_python `ubuntu_noble `master); + (Auto, job_install_python `ubuntu_jammy `master); + (Auto, job_install_python `debian_bookworm `master); + ] ; () -- GitLab From 7b8b4d78fbd84175723e9ba3b3b239b1fad9eff9 Mon Sep 17 00:00:00 2001 From: Romain Date: Wed, 27 Aug 2025 15:17:35 +0200 Subject: [PATCH 3/9] CI: migrate documentation:odoc to Cacio --- .gitlab-ci.yml | 8 ++ .gitlab/ci/pipelines/before_merging.yml | 130 ++++++++++-------- .gitlab/ci/pipelines/documentation.daily.yml | 72 ++++++++++ .gitlab/ci/pipelines/documentation.update.yml | 95 +++++++++++++ .gitlab/ci/pipelines/merge_train.yml | 130 ++++++++++-------- .../ci/pipelines/schedule_documentation.yml | 37 +---- .../ci/pipelines/schedule_extended_test.yml | 37 +---- ci/bin/code_verification.ml | 8 +- ci/bin/common.ml | 34 ----- ci/bin/documentation.ml | 67 ++++++++- ci/bin/master_branch.ml | 10 +- 11 files changed, 409 insertions(+), 219 deletions(-) create mode 100644 .gitlab/ci/pipelines/documentation.update.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7e625463c8a5..168e7b8019af 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,6 +41,10 @@ workflow: variables: PIPELINE_TYPE: documentation.daily when: always + - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "documentation.update" + variables: + PIPELINE_TYPE: documentation.update + when: always - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_EVENT_TYPE != "merge_train" variables: @@ -262,6 +266,10 @@ include: rules: - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "documentation.daily" when: always +- local: .gitlab/ci/pipelines/documentation.update.yml + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "documentation.update" + when: always - local: .gitlab/ci/pipelines/before_merging.yml rules: - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "merge_request_event" diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 97f835ac5213..a2b2ed398c35 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -4132,74 +4132,20 @@ oc.unified_coverage: coverage: '/Coverage: ([^%]+%)/' documentation:odoc: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build + stage: start tags: - gcp - rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - - changes: - - '**/*.rst' - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - - brassaia/**/*.ml* - - client-libs/**/*.ml* - - data-encoding/**/*.ml* - - docs/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/*.ml* - - grafazos/doc/**/* - - irmin/**/*.ml* - - script-inputs/**/*/ - - scripts/**/*/ - - src/**/*.ml* - - tezt/**/*.ml* - - vendors/**/*.ml* - when: on_success - needs: - - oc.docker:ci:amd64 - - trigger - dependencies: - - oc.docker:ci:amd64 + dependencies: [] timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push before_script: - . ./scripts/ci/datadog_send_job_info.sh - - eval $(opam env) - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh - script: - - export OPAMFETCH='wget' - - opam remote add default https://opam.ocaml.org/ - - opam repo add archive git+https://github.com/ocaml/opam-repository-archive - - opam update - - opam install --yes odoc.2.4.4 - - make -C docs odoc-lite + script: [] after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G artifacts: - expire_in: 4 hours paths: - - docs/_build/api/odoc/ - - docs/odoc.log - when: always + - dummy documentation:manuals: image: ${ci_image_name}/test:${ci_image_tag} @@ -4648,6 +4594,74 @@ documentation.rst-check: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' +documentation.odoc: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build + tags: + - gcp + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success + - changes: + - '**/*.rst' + - brassaia/**/*.ml* + - client-libs/**/*.ml* + - data-encoding/**/*.ml* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/*.ml* + - grafazos/doc/**/* + - irmin/**/*.ml* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/*.ml* + - tezt/**/*.ml* + - vendors/**/*.ml* + when: on_success + needs: + - oc.docker:ci:amd64 + - trigger + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh + script: + - eval $(opam env) + - export OPAMFETCH='wget' + - opam remote add default https://opam.ocaml.org/ + - opam repo add archive git+https://github.com/ocaml/opam-repository-archive + - opam update + - opam install --yes odoc.2.4.4 + - make -C docs odoc-lite + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh + variables: + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 5G + artifacts: + expire_in: 4 hours + paths: + - docs/_build/api/odoc/ + - docs/odoc.log + when: always + documentation.install_python_bookworm: image: debian:bookworm stage: test diff --git a/.gitlab/ci/pipelines/documentation.daily.yml b/.gitlab/ci/pipelines/documentation.daily.yml index b9449efde04c..8d675394c529 100644 --- a/.gitlab/ci/pipelines/documentation.daily.yml +++ b/.gitlab/ci/pipelines/documentation.daily.yml @@ -3,8 +3,34 @@ stages: - start +- images +- build - test +oc.docker:ci:amd64: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + dependencies: [] + timeout: 90 minutes + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - ./images/ci_create_ci_images.sh + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + CI_DOCKER_HUB: "false" + ARCH: amd64 + artifacts: + reports: + dotenv: ci_image_tag.env + datadog_pipeline_trace: image: datadog/ci:v2.44.0 stage: start @@ -41,6 +67,52 @@ documentation.rst-check: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' +documentation.odoc: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build + tags: + - gcp + needs: + - oc.docker:ci:amd64 + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh + script: + - eval $(opam env) + - export OPAMFETCH='wget' + - opam remote add default https://opam.ocaml.org/ + - opam repo add archive git+https://github.com/ocaml/opam-repository-archive + - opam update + - opam install --yes odoc.2.4.4 + - make -C docs odoc-lite + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh + variables: + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 5G + artifacts: + expire_in: 4 hours + paths: + - docs/_build/api/odoc/ + - docs/odoc.log + when: always + documentation.install_python_bookworm: image: debian:bookworm stage: test diff --git a/.gitlab/ci/pipelines/documentation.update.yml b/.gitlab/ci/pipelines/documentation.update.yml new file mode 100644 index 000000000000..d18dd3525ceb --- /dev/null +++ b/.gitlab/ci/pipelines/documentation.update.yml @@ -0,0 +1,95 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +stages: +- start +- images +- build + +oc.docker:ci:amd64: + image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 + stage: images + tags: + - gcp + dependencies: [] + timeout: 90 minutes + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - ./images/ci_create_ci_images.sh + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.7 + CI_DOCKER_HUB: "false" + ARCH: amd64 + artifacts: + reports: + dotenv: ci_image_tag.env + +datadog_pipeline_trace: + image: datadog/ci:v2.44.0 + stage: start + tags: + - gcp + dependencies: [] + allow_failure: true + timeout: 60 minutes + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - CI_MERGE_REQUEST_IID=${CI_MERGE_REQUEST_IID:-none} + - DATADOG_SITE=datadoghq.eu datadog-ci tag --level pipeline --tags pipeline_type:$PIPELINE_TYPE + --tags mr_number:$CI_MERGE_REQUEST_IID + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + +documentation.odoc: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build + tags: + - gcp + needs: + - oc.docker:ci:amd64 + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh + script: + - eval $(opam env) + - export OPAMFETCH='wget' + - opam remote add default https://opam.ocaml.org/ + - opam repo add archive git+https://github.com/ocaml/opam-repository-archive + - opam update + - opam install --yes odoc.2.4.4 + - make -C docs odoc + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh + variables: + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 5G + artifacts: + expire_in: 4 hours + paths: + - docs/_build/api/odoc/ + - docs/odoc.log + when: always diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index 6a1e47371860..4942c981fbfa 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -4131,74 +4131,20 @@ oc.unified_coverage: coverage: '/Coverage: ([^%]+%)/' documentation:odoc: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build + stage: start tags: - gcp - rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - - changes: - - '**/*.rst' - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - - brassaia/**/*.ml* - - client-libs/**/*.ml* - - data-encoding/**/*.ml* - - docs/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/*.ml* - - grafazos/doc/**/* - - irmin/**/*.ml* - - script-inputs/**/*/ - - scripts/**/*/ - - src/**/*.ml* - - tezt/**/*.ml* - - vendors/**/*.ml* - when: on_success - needs: - - oc.docker:ci:amd64 - - trigger - dependencies: - - oc.docker:ci:amd64 + dependencies: [] timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push before_script: - . ./scripts/ci/datadog_send_job_info.sh - - eval $(opam env) - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh - script: - - export OPAMFETCH='wget' - - opam remote add default https://opam.ocaml.org/ - - opam repo add archive git+https://github.com/ocaml/opam-repository-archive - - opam update - - opam install --yes odoc.2.4.4 - - make -C docs odoc-lite + script: [] after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G artifacts: - expire_in: 4 hours paths: - - docs/_build/api/odoc/ - - docs/odoc.log - when: always + - dummy documentation:manuals: image: ${ci_image_name}/test:${ci_image_tag} @@ -4582,6 +4528,74 @@ documentation.rst-check: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' +documentation.odoc: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build + tags: + - gcp + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success + - changes: + - '**/*.rst' + - brassaia/**/*.ml* + - client-libs/**/*.ml* + - data-encoding/**/*.ml* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/*.ml* + - grafazos/doc/**/* + - irmin/**/*.ml* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/*.ml* + - tezt/**/*.ml* + - vendors/**/*.ml* + when: on_success + needs: + - oc.docker:ci:amd64 + - trigger + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh + script: + - eval $(opam env) + - export OPAMFETCH='wget' + - opam remote add default https://opam.ocaml.org/ + - opam repo add archive git+https://github.com/ocaml/opam-repository-archive + - opam update + - opam install --yes odoc.2.4.4 + - make -C docs odoc-lite + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh + variables: + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 5G + artifacts: + expire_in: 4 hours + paths: + - docs/_build/api/odoc/ + - docs/odoc.log + when: always + documentation.install_python_bookworm: image: debian:bookworm stage: test diff --git a/.gitlab/ci/pipelines/schedule_documentation.yml b/.gitlab/ci/pipelines/schedule_documentation.yml index ebcd200a4b42..2483d360a1db 100644 --- a/.gitlab/ci/pipelines/schedule_documentation.yml +++ b/.gitlab/ci/pipelines/schedule_documentation.yml @@ -99,50 +99,21 @@ oc.build:static-x86_64-linux-binaries: - octez-binaries/$ARCH/* documentation:odoc: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build + stage: start tags: - gcp - needs: - - oc.docker:ci:amd64 - dependencies: - - oc.docker:ci:amd64 + dependencies: [] timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push interruptible: false before_script: - . ./scripts/ci/datadog_send_job_info.sh - - eval $(opam env) - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh - script: - - export OPAMFETCH='wget' - - opam remote add default https://opam.ocaml.org/ - - opam repo add archive git+https://github.com/ocaml/opam-repository-archive - - opam update - - opam install --yes odoc.2.4.4 - - make -C docs odoc + script: [] after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G artifacts: - expire_in: 4 hours paths: - - docs/_build/api/odoc/ - - docs/odoc.log - when: always + - dummy documentation:manuals: image: ${ci_image_name}/test:${ci_image_tag} diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 72c279e08ce0..9b973cd739fd 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -3650,50 +3650,21 @@ tezt-riscv-slow-sequential: retry: 2 documentation:odoc: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build + stage: start tags: - gcp - rules: - - when: always - dependencies: - - oc.docker:ci:amd64 + dependencies: [] timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push interruptible: false before_script: - . ./scripts/ci/datadog_send_job_info.sh - - eval $(opam env) - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh - script: - - export OPAMFETCH='wget' - - opam remote add default https://opam.ocaml.org/ - - opam repo add archive git+https://github.com/ocaml/opam-repository-archive - - opam update - - opam install --yes odoc.2.4.4 - - make -C docs odoc-lite + script: [] after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G artifacts: - expire_in: 4 hours paths: - - docs/_build/api/odoc/ - - docs/odoc.log - when: always + - dummy documentation:manuals: image: ${ci_image_name}/test:${ci_image_tag} diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 512685248674..56028869c367 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -1964,7 +1964,13 @@ let jobs pipeline_type = in let dependencies = dependencies_needs_start in let job_odoc = - Documentation.job_odoc ~rules ~dependencies ~lite:true () + (* Dummy job to remove in a commit later in this MR. *) + Tezos_ci.job + ~__POS__ + ~stage:Tezos_ci.Stages.start + ~name:"documentation:odoc" + ~artifacts:(Gitlab_ci.Util.artifacts ["dummy"]) + [] in let job_manuals = Documentation.job_manuals diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 213cb7bf64cb..b16ccbe20891 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -1232,40 +1232,6 @@ module Documentation = struct jobs dependencies - (** Create an odoc job. - - The job will build the target [odoc] (resp. [odoc-lite]) in the - directory [docs] if lite is [false] (resp. [true]), which is the - default. - - This job is one of the prerequisites to {!job_build_all}. *) - let job_odoc ?(lite = false) ?rules ?dependencies () : tezos_job = - let target = if lite then "odoc-lite" else "odoc" in - job - ~__POS__ - ~name:"documentation:odoc" - ~image:Images.CI.test - ~stage:Stages.build - ?dependencies - ?rules - ~before_script:(before_script ~eval_opam:true []) - ~artifacts: - (artifacts - ~when_:Always - ~expire_in:(Duration (Hours 4)) - (* Path must be terminated with / to expose artifact (gitlab-org/gitlab#/36706) *) - ["docs/_build/api/odoc/"; "docs/odoc.log"]) - [ - "export OPAMFETCH='wget'"; - "opam remote add default https://opam.ocaml.org/"; - "opam repo add archive \ - git+https://github.com/ocaml/opam-repository-archive"; - "opam update"; - "opam install --yes odoc.2.4.4"; - "make -C docs " ^ target; - ] - |> enable_cargo_cache |> enable_sccache - (** Create the manuals job. This job builds the command-line interface manuals (i.e [man]) diff --git a/ci/bin/documentation.ml b/ci/bin/documentation.ml index 42f01f30b421..621cf3b1d6f5 100644 --- a/ci/bin/documentation.ml +++ b/ci/bin/documentation.ml @@ -15,7 +15,32 @@ module Files = struct "poetry.lock"; ] - let all = rst @ python_install_script + (* Copied from changeset_octez_docs in common.ml, + which will be removed once all documentation jobs are migrated to Cacio. + In particular, the TODO was copy-pasted as-is, it does not come from this MR. *) + let odoc = + [ + (* TODO: refine scripts. *) + "scripts/**/*/"; + "script-inputs/**/*/"; + "src/**/*.ml*"; + "tezt/**/*.ml*"; + "brassaia/**/*.ml*"; + "irmin/**/*.ml*"; + "client-libs/**/*.ml*"; + "etherlink/**/*.ml*"; + "data-encoding/**/*.ml*"; + "vendors/**/*.ml*"; + "dune"; + "dune-project"; + "dune-workspace"; + "**/*.rst"; + (* Nota: stays as it is, many non-rst files in this folder *) + "docs/**/*"; + "grafazos/doc/**/*"; + ] + + let all = rst @ python_install_script @ odoc end module CI = Cacio.Make (struct @@ -62,11 +87,44 @@ let job_install_python = ~force_if_label:["ci--docs"] [sf "./docs/developer/install-python-debian-ubuntu.sh %s %s" project branch] +let job_odoc = + Cacio.parameterize @@ fun mode -> + let target = match mode with `lite -> "odoc-lite" | `full -> "odoc" in + CI.job + "odoc" + ~__POS__ + ~image:Tezos_ci.Images.CI.test + ~stage:Build + ~description: + ("Build the documentation of our OCaml libraries (make -C docs " ^ target + ^ ").") + ~only_if_changed:Files.odoc + ~force_if_label:["ci--docs"] + ~artifacts: + (Gitlab_ci.Util.artifacts + ~when_:Always + ~expire_in:(Duration (Hours 4)) + (* Path must be terminated with / to expose artifact (gitlab-org/gitlab#/36706) *) + ["docs/_build/api/odoc/"; "docs/odoc.log"]) + ~cargo_cache:true + ~sccache:(Cacio.sccache ()) + [ + "eval $(opam env)"; + "export OPAMFETCH='wget'"; + "opam remote add default https://opam.ocaml.org/"; + "opam repo add archive \ + git+https://github.com/ocaml/opam-repository-archive"; + "opam update"; + "opam install --yes odoc.2.4.4"; + "make -C docs " ^ target; + ] + let register () = CI.register_before_merging_jobs [ (Immediate, job_rst_check); (Auto, job_install_python `debian_bookworm `current_branch); + (Auto, job_odoc `lite); ] ; CI.register_scheduled_pipeline "daily" @@ -76,5 +134,12 @@ let register () = (Auto, job_install_python `ubuntu_noble `master); (Auto, job_install_python `ubuntu_jammy `master); (Auto, job_install_python `debian_bookworm `master); + (Auto, job_odoc `lite); ] ; + CI.register_scheduled_pipeline + "update" + ~description: + "Generate and push the documentation to octez.com/docs without being \ + interrupted." + [(Auto, job_odoc `full)] ; () diff --git a/ci/bin/master_branch.ml b/ci/bin/master_branch.ml index 7fe32dadeaa2..0a42c4dc4e62 100644 --- a/ci/bin/master_branch.ml +++ b/ci/bin/master_branch.ml @@ -43,7 +43,15 @@ let job_static_x86_64 = let jobs_documentation : tezos_job list = let dependencies = Dependent [] in - let job_odoc = Documentation.job_odoc ~dependencies () in + let job_odoc = + (* Dummy job to remove in a commit later in this MR. *) + Tezos_ci.job + ~__POS__ + ~stage:Tezos_ci.Stages.start + ~name:"documentation:odoc" + ~artifacts:(Gitlab_ci.Util.artifacts ["dummy"]) + [] + in let job_manuals = Documentation.job_manuals ~dependencies:(Dependent [Artifacts job_static_x86_64]) -- GitLab From bb0710131cd8048b0156f1e009acbeb2888afadd Mon Sep 17 00:00:00 2001 From: Romain Date: Wed, 27 Aug 2025 16:38:54 +0200 Subject: [PATCH 4/9] CI: migrate documentation:manuals to Cacio --- .gitlab/ci/pipelines/before_merging.yml | 113 ++++++++++-------- .gitlab/ci/pipelines/documentation.daily.yml | 77 ++++++++++++ .gitlab/ci/pipelines/documentation.update.yml | 77 ++++++++++++ .gitlab/ci/pipelines/merge_train.yml | 113 ++++++++++-------- .../ci/pipelines/schedule_documentation.yml | 21 +--- .../ci/pipelines/schedule_extended_test.yml | 31 +---- ci/bin/code_verification.ml | 20 ++-- ci/bin/common.ml | 29 ----- ci/bin/documentation.ml | 70 ++++++++++- ci/bin/master_branch.ml | 15 ++- 10 files changed, 377 insertions(+), 189 deletions(-) diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index a2b2ed398c35..bc2e6ae17a3c 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -4148,65 +4148,20 @@ documentation:odoc: - dummy documentation:manuals: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build + stage: start tags: - gcp - rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - - changes: - - '**/*.rst' - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - - brassaia/**/*.ml* - - client-libs/**/*.ml* - - data-encoding/**/*.ml* - - docs/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/*.ml* - - grafazos/doc/**/* - - irmin/**/*.ml* - - script-inputs/**/*/ - - scripts/**/*/ - - src/**/*.ml* - - tezt/**/*.ml* - - vendors/**/*.ml* - when: on_success - needs: - - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_amd64-extra-dev - - oc.build_amd64-extra-exp - - oc.build_kernels - - oc.build_dsn_node - dependencies: - - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_amd64-extra-dev - - oc.build_amd64-extra-exp - - oc.build_kernels - - oc.build_dsn_node + dependencies: [] timeout: 60 minutes before_script: - . ./scripts/ci/datadog_send_job_info.sh - - eval $(opam env) - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - make -C docs -j octez-gen + script: [] after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' artifacts: - expire_in: 1 week paths: - - docs/*/octez-*.html - - docs/api/octez-*.txt - - docs/developer/metrics.csv - - docs/developer/rollup_metrics.csv - - docs/user/node-config.json + - dummy documentation:docgen: image: ${ci_image_name}/test:${ci_image_tag} @@ -4594,6 +4549,66 @@ documentation.rst-check: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' +documentation.manuals: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build + tags: + - gcp + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success + - changes: + - '**/*.rst' + - brassaia/**/*.ml* + - client-libs/**/*.ml* + - data-encoding/**/*.ml* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/*.ml* + - grafazos/doc/**/* + - irmin/**/*.ml* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/*.ml* + - tezt/**/*.ml* + - vendors/**/*.ml* + when: on_success + needs: + - oc.docker:ci:amd64 + - trigger + - oc.build_x86_64-released + - oc.build_amd64-extra-dev + - oc.build_amd64-extra-exp + - oc.build_kernels + - oc.build_dsn_node + dependencies: + - oc.docker:ci:amd64 + - oc.build_x86_64-released + - oc.build_amd64-extra-dev + - oc.build_amd64-extra-exp + - oc.build_kernels + - oc.build_dsn_node + timeout: 60 minutes + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - eval $(opam env) + - make -C docs -j octez-gen + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + artifacts: + expire_in: 1 week + paths: + - docs/*/octez-*.html + - docs/api/octez-*.txt + - docs/developer/metrics.csv + - docs/developer/rollup_metrics.csv + - docs/user/node-config.json + documentation.odoc: image: ${ci_image_name}/test:${ci_image_tag} stage: build diff --git a/.gitlab/ci/pipelines/documentation.daily.yml b/.gitlab/ci/pipelines/documentation.daily.yml index 8d675394c529..44434b06bdbb 100644 --- a/.gitlab/ci/pipelines/documentation.daily.yml +++ b/.gitlab/ci/pipelines/documentation.daily.yml @@ -49,6 +49,53 @@ datadog_pipeline_trace: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' +oc.build:static-x86_64-linux-binaries: + image: ${ci_image_name}/build:${ci_image_tag} + stage: build + tags: + - gcp_very_high_cpu_ramfs + rules: + - when: always + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + - key: rust-targets-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_target + policy: pull-push + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh + script: + - ./scripts/ci/build_static_binaries.sh + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh + variables: + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/octez-released-executables script-inputs/octez-experimental-executables + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 2G + OCTEZ_RUST_DEPS_TARGET_DIR: $CI_PROJECT_DIR/_target/rust_deps + OCTEZ_RUSTZCASH_DEPS_TARGET_DIR: $CI_PROJECT_DIR/_target/rustzcash_deps + OCTEZ_ETHERLINK_WASM_RUNTIME_TARGET_DIR: $CI_PROJECT_DIR/_target/etherlink_wasm_runtime + artifacts: + paths: + - octez-binaries/$ARCH/* + documentation.rst-check: image: ${ci_image_name_protected}/test:amd64--master stage: test @@ -113,6 +160,36 @@ documentation.odoc: - docs/odoc.log when: always +documentation.manuals: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build + tags: + - gcp + needs: + - oc.docker:ci:amd64 + - oc.build:static-x86_64-linux-binaries + dependencies: + - oc.docker:ci:amd64 + - oc.build:static-x86_64-linux-binaries + timeout: 60 minutes + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - eval $(opam env) + - scripts/ci/documentation:manuals_static.sh + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + artifacts: + expire_in: 1 week + paths: + - docs/*/octez-*.html + - docs/api/octez-*.txt + - docs/developer/metrics.csv + - docs/developer/rollup_metrics.csv + - docs/user/node-config.json + documentation.install_python_bookworm: image: debian:bookworm stage: test diff --git a/.gitlab/ci/pipelines/documentation.update.yml b/.gitlab/ci/pipelines/documentation.update.yml index d18dd3525ceb..b7df2a5ccb57 100644 --- a/.gitlab/ci/pipelines/documentation.update.yml +++ b/.gitlab/ci/pipelines/documentation.update.yml @@ -48,6 +48,83 @@ datadog_pipeline_trace: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' +oc.build:static-x86_64-linux-binaries: + image: ${ci_image_name}/build:${ci_image_tag} + stage: build + tags: + - gcp_very_high_cpu_ramfs + rules: + - when: always + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + - key: rust-targets-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_target + policy: pull-push + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - ./scripts/ci/take_ownership.sh + - eval $(opam env) + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh + script: + - ./scripts/ci/build_static_binaries.sh + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh + variables: + ARCH: x86_64 + EXECUTABLE_FILES: script-inputs/octez-released-executables script-inputs/octez-experimental-executables + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 2G + OCTEZ_RUST_DEPS_TARGET_DIR: $CI_PROJECT_DIR/_target/rust_deps + OCTEZ_RUSTZCASH_DEPS_TARGET_DIR: $CI_PROJECT_DIR/_target/rustzcash_deps + OCTEZ_ETHERLINK_WASM_RUNTIME_TARGET_DIR: $CI_PROJECT_DIR/_target/etherlink_wasm_runtime + artifacts: + paths: + - octez-binaries/$ARCH/* + +documentation.manuals: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build + tags: + - gcp + needs: + - oc.docker:ci:amd64 + - oc.build:static-x86_64-linux-binaries + dependencies: + - oc.docker:ci:amd64 + - oc.build:static-x86_64-linux-binaries + timeout: 60 minutes + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - eval $(opam env) + - scripts/ci/documentation:manuals_static.sh + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + artifacts: + expire_in: 1 week + paths: + - docs/*/octez-*.html + - docs/api/octez-*.txt + - docs/developer/metrics.csv + - docs/developer/rollup_metrics.csv + - docs/user/node-config.json + documentation.odoc: image: ${ci_image_name}/test:${ci_image_tag} stage: build diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index 4942c981fbfa..d85bd8dd8d0e 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -4147,65 +4147,20 @@ documentation:odoc: - dummy documentation:manuals: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build + stage: start tags: - gcp - rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - - changes: - - '**/*.rst' - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - - brassaia/**/*.ml* - - client-libs/**/*.ml* - - data-encoding/**/*.ml* - - docs/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/*.ml* - - grafazos/doc/**/* - - irmin/**/*.ml* - - script-inputs/**/*/ - - scripts/**/*/ - - src/**/*.ml* - - tezt/**/*.ml* - - vendors/**/*.ml* - when: on_success - needs: - - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_amd64-extra-dev - - oc.build_amd64-extra-exp - - oc.build_kernels - - oc.build_dsn_node - dependencies: - - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_amd64-extra-dev - - oc.build_amd64-extra-exp - - oc.build_kernels - - oc.build_dsn_node + dependencies: [] timeout: 60 minutes before_script: - . ./scripts/ci/datadog_send_job_info.sh - - eval $(opam env) - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - make -C docs -j octez-gen + script: [] after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' artifacts: - expire_in: 1 week paths: - - docs/*/octez-*.html - - docs/api/octez-*.txt - - docs/developer/metrics.csv - - docs/developer/rollup_metrics.csv - - docs/user/node-config.json + - dummy documentation:docgen: image: ${ci_image_name}/test:${ci_image_tag} @@ -4528,6 +4483,66 @@ documentation.rst-check: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' +documentation.manuals: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build + tags: + - gcp + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success + - changes: + - '**/*.rst' + - brassaia/**/*.ml* + - client-libs/**/*.ml* + - data-encoding/**/*.ml* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/*.ml* + - grafazos/doc/**/* + - irmin/**/*.ml* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/*.ml* + - tezt/**/*.ml* + - vendors/**/*.ml* + when: on_success + needs: + - oc.docker:ci:amd64 + - trigger + - oc.build_x86_64-released + - oc.build_amd64-extra-dev + - oc.build_amd64-extra-exp + - oc.build_kernels + - oc.build_dsn_node + dependencies: + - oc.docker:ci:amd64 + - oc.build_x86_64-released + - oc.build_amd64-extra-dev + - oc.build_amd64-extra-exp + - oc.build_kernels + - oc.build_dsn_node + timeout: 60 minutes + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - eval $(opam env) + - make -C docs -j octez-gen + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + artifacts: + expire_in: 1 week + paths: + - docs/*/octez-*.html + - docs/api/octez-*.txt + - docs/developer/metrics.csv + - docs/developer/rollup_metrics.csv + - docs/user/node-config.json + documentation.odoc: image: ${ci_image_name}/test:${ci_image_tag} stage: build diff --git a/.gitlab/ci/pipelines/schedule_documentation.yml b/.gitlab/ci/pipelines/schedule_documentation.yml index 2483d360a1db..c2dd3abb8e1e 100644 --- a/.gitlab/ci/pipelines/schedule_documentation.yml +++ b/.gitlab/ci/pipelines/schedule_documentation.yml @@ -116,34 +116,21 @@ documentation:odoc: - dummy documentation:manuals: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build + stage: start tags: - gcp - needs: - - oc.docker:ci:amd64 - - oc.build:static-x86_64-linux-binaries - dependencies: - - oc.docker:ci:amd64 - - oc.build:static-x86_64-linux-binaries + dependencies: [] timeout: 60 minutes interruptible: false before_script: - . ./scripts/ci/datadog_send_job_info.sh - - eval $(opam env) - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - scripts/ci/documentation:manuals_static.sh + script: [] after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' artifacts: - expire_in: 1 week paths: - - docs/*/octez-*.html - - docs/api/octez-*.txt - - docs/developer/metrics.csv - - docs/developer/rollup_metrics.csv - - docs/user/node-config.json + - dummy documentation:docgen: image: ${ci_image_name}/test:${ci_image_tag} diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 9b973cd739fd..0a2dc80f0c8e 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -3667,44 +3667,21 @@ documentation:odoc: - dummy documentation:manuals: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build + stage: start tags: - gcp - rules: - - when: always - needs: - - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_amd64-extra-dev - - oc.build_amd64-extra-exp - - oc.build_kernels - - oc.build_dsn_node - dependencies: - - oc.docker:ci:amd64 - - oc.build_x86_64-released - - oc.build_amd64-extra-dev - - oc.build_amd64-extra-exp - - oc.build_kernels - - oc.build_dsn_node + dependencies: [] timeout: 60 minutes interruptible: false before_script: - . ./scripts/ci/datadog_send_job_info.sh - - eval $(opam env) - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - make -C docs -j octez-gen + script: [] after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' artifacts: - expire_in: 1 week paths: - - docs/*/octez-*.html - - docs/api/octez-*.txt - - docs/developer/metrics.csv - - docs/developer/rollup_metrics.csv - - docs/user/node-config.json + - dummy documentation:docgen: image: ${ci_image_name}/test:${ci_image_tag} diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 56028869c367..e5d9fca9ee5b 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -1973,19 +1973,13 @@ let jobs pipeline_type = [] in let job_manuals = - Documentation.job_manuals - ~rules - ~use_static_executables:false - ~dependencies: - (Dependent - [ - Artifacts job_build_x86_64_release; - Artifacts job_build_x86_64_extra_dev; - Artifacts job_build_x86_64_extra_exp; - Artifacts job_build_kernels; - Artifacts job_build_dsn_node; - ]) - () + (* Dummy job to remove in a commit later in this MR. *) + Tezos_ci.job + ~__POS__ + ~stage:Tezos_ci.Stages.start + ~name:"documentation:manuals" + ~artifacts:(Gitlab_ci.Util.artifacts ["dummy"]) + [] in let job_docgen = Documentation.job_docgen ~rules ~dependencies () in let job_build_all = diff --git a/ci/bin/common.ml b/ci/bin/common.ml index b16ccbe20891..69c2e2990fed 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -1232,35 +1232,6 @@ module Documentation = struct jobs dependencies - (** Create the manuals job. - - This job builds the command-line interface manuals (i.e [man]) - of octez binaries, for inclusion in the documentation. - - This job is one of the prerequisites to {!job_build_all}. *) - let job_manuals ?rules ?dependencies ~use_static_executables () : tezos_job = - job - ~__POS__ - ~name:"documentation:manuals" - ~image:Images.CI.test - ~stage:Stages.build - ?dependencies - ?rules - ~before_script:(before_script ~eval_opam:true []) - ~artifacts: - (artifacts - ~expire_in:(Duration (Weeks 1)) - [ - "docs/*/octez-*.html"; - "docs/api/octez-*.txt"; - "docs/developer/metrics.csv"; - "docs/developer/rollup_metrics.csv"; - "docs/user/node-config.json"; - ]) - (if use_static_executables then - ["scripts/ci/documentation:manuals_static.sh"] - else ["make -C docs -j octez-gen"]) - (** Create the docgen job. This job builds various generated reference material, for diff --git a/ci/bin/documentation.ml b/ci/bin/documentation.ml index 621cf3b1d6f5..ba3b5bb4e2c1 100644 --- a/ci/bin/documentation.ml +++ b/ci/bin/documentation.ml @@ -119,27 +119,95 @@ let job_odoc = "make -C docs " ^ target; ] +let job_manuals = + Cacio.parameterize @@ fun executables_to_use -> + CI.job + "manuals" + ~__POS__ + ~image:Tezos_ci.Images.CI.test + ~stage:Build + ~description: + "Build the command-line interface manuals (man pages) of Octez \ + executables." + ~needs_legacy: + (match executables_to_use with + | `dynamic -> + (* The legacy build jobs are defined deep in code_verification.ml + and it is not easy to extract them. + For now, we thus define a dummy job with the same name, + since CIAO only cares about the name of the dependency. + Note that CIAO will still check that those dependencies are in the pipeline. + - + This trick only works for global pipelines that already have those jobs. + Scheduled pipelines registered by Cacio + (such as documentation.daily and documentation.update) + would need to include them with [~legacy_jobs]. + Which would bring us back to the original issue which is that those jobs + are defined deep in code_verification.ml. + - + Fortunately, for this job (documentation.manuals), we can depend on + the job that builds static executables instead. + This job is already defined at toplevel as [Master_branch.job_static_x86_64]. + So it is easy to include it with [~legacy_jobs]. + - + However, the static build job is slower than the regular build jobs. + This is not an issue in scheduled pipelines, but for merge request pipelines, + being fast matters. Hence the dummy job trick, which works because + merge request pipelines already include the regular build jobs. *) + let dummy name = + Tezos_ci.job ~name ~stage:Tezos_ci.Stages.build ~__POS__ [] + in + [ + (Artifacts, dummy "oc.build_x86_64-released"); + (Artifacts, dummy "oc.build_amd64-extra-dev"); + (Artifacts, dummy "oc.build_amd64-extra-exp"); + (Artifacts, dummy "oc.build_kernels"); + (Artifacts, dummy "oc.build_dsn_node"); + ] + | `static -> [(Artifacts, Master_branch.job_static_x86_64)]) + ~only_if_changed:Files.odoc + ~force_if_label:["ci--docs"] + ~artifacts: + (Gitlab_ci.Util.artifacts + ~expire_in:(Duration (Weeks 1)) + [ + "docs/*/octez-*.html"; + "docs/api/octez-*.txt"; + "docs/developer/metrics.csv"; + "docs/developer/rollup_metrics.csv"; + "docs/user/node-config.json"; + ]) + ("eval $(opam env)" + :: + (match executables_to_use with + | `dynamic -> ["make -C docs -j octez-gen"] + | `static -> ["scripts/ci/documentation:manuals_static.sh"])) + let register () = CI.register_before_merging_jobs [ (Immediate, job_rst_check); (Auto, job_install_python `debian_bookworm `current_branch); (Auto, job_odoc `lite); + (Auto, job_manuals `dynamic); ] ; CI.register_scheduled_pipeline "daily" ~description:"Daily tests to run for the documentation." + ~legacy_jobs:[Master_branch.job_static_x86_64] [ (Auto, job_rst_check); (Auto, job_install_python `ubuntu_noble `master); (Auto, job_install_python `ubuntu_jammy `master); (Auto, job_install_python `debian_bookworm `master); (Auto, job_odoc `lite); + (Auto, job_manuals `static); ] ; CI.register_scheduled_pipeline "update" ~description: "Generate and push the documentation to octez.com/docs without being \ interrupted." - [(Auto, job_odoc `full)] ; + ~legacy_jobs:[Master_branch.job_static_x86_64] + [(Auto, job_odoc `full); (Auto, job_manuals `static)] ; () diff --git a/ci/bin/master_branch.ml b/ci/bin/master_branch.ml index 0a42c4dc4e62..f901dae70de5 100644 --- a/ci/bin/master_branch.ml +++ b/ci/bin/master_branch.ml @@ -21,6 +21,10 @@ open Gitlab_ci.Util open Tezos_ci open Tezos_ci.Cache +(* Without this, Dune's dependency analysis is wrong: + it thinks Documentation is the new module that uses Cacio. *) +module Documentation = Common.Documentation + let rules_always = [job_rule ~when_:Always ()] (* static binaries *) @@ -53,10 +57,13 @@ let jobs_documentation : tezos_job list = [] in let job_manuals = - Documentation.job_manuals - ~dependencies:(Dependent [Artifacts job_static_x86_64]) - ~use_static_executables:true - () + (* Dummy job to remove in a commit later in this MR. *) + Tezos_ci.job + ~__POS__ + ~stage:Tezos_ci.Stages.start + ~name:"documentation:manuals" + ~artifacts:(Gitlab_ci.Util.artifacts ["dummy"]) + [] in let job_docgen = Documentation.job_docgen ~dependencies () in let job_build_all = -- GitLab From 59a37b53ae9463b85cf7fbaf9e9688162eb3e436 Mon Sep 17 00:00:00 2001 From: Romain Date: Wed, 27 Aug 2025 16:19:08 +0200 Subject: [PATCH 5/9] CI: migrate documentation:docgen to Cacio --- .gitlab/ci/pipelines/before_merging.yml | 124 ++++++++++-------- .gitlab/ci/pipelines/documentation.daily.yml | 43 ++++++ .gitlab/ci/pipelines/documentation.update.yml | 43 ++++++ .gitlab/ci/pipelines/merge_train.yml | 124 ++++++++++-------- .../ci/pipelines/schedule_documentation.yml | 34 +---- .../ci/pipelines/schedule_extended_test.yml | 34 +---- ci/bin/code_verification.ml | 14 +- ci/bin/common.ml | 29 ---- ci/bin/documentation.ml | 29 +++- ci/bin/master_branch.ml | 11 +- 10 files changed, 278 insertions(+), 207 deletions(-) diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index bc2e6ae17a3c..0c41fc45277e 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -4164,71 +4164,20 @@ documentation:manuals: - dummy documentation:docgen: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build + stage: start tags: - gcp - rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - - changes: - - '**/*.rst' - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - - brassaia/**/*.ml* - - client-libs/**/*.ml* - - data-encoding/**/*.ml* - - docs/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/*.ml* - - grafazos/doc/**/* - - irmin/**/*.ml* - - script-inputs/**/*/ - - scripts/**/*/ - - src/**/*.ml* - - tezt/**/*.ml* - - vendors/**/*.ml* - when: on_success - needs: - - oc.docker:ci:amd64 - - trigger - dependencies: - - oc.docker:ci:amd64 + dependencies: [] timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push before_script: - . ./scripts/ci/datadog_send_job_info.sh - - eval $(opam env) - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh - script: - - make -C docs -j docexes-gen + script: [] after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G artifacts: - expire_in: 1 week paths: - - docs/alpha/rpc.rst - - docs/shell/rpc.rst - - docs/user/default-acl.json - - docs/api/errors.rst - - docs/shell/p2p_api.rst + - dummy documentation:build_all: image: ${ci_image_name}/test:${ci_image_tag} @@ -4549,6 +4498,71 @@ documentation.rst-check: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' +documentation.docgen: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build + tags: + - gcp + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success + - changes: + - '**/*.rst' + - brassaia/**/*.ml* + - client-libs/**/*.ml* + - data-encoding/**/*.ml* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/*.ml* + - grafazos/doc/**/* + - irmin/**/*.ml* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/*.ml* + - tezt/**/*.ml* + - vendors/**/*.ml* + when: on_success + needs: + - oc.docker:ci:amd64 + - trigger + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh + script: + - eval $(opam env) + - make -C docs -j docexes-gen + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh + variables: + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 5G + artifacts: + expire_in: 1 week + paths: + - docs/alpha/rpc.rst + - docs/shell/rpc.rst + - docs/user/default-acl.json + - docs/api/errors.rst + - docs/shell/p2p_api.rst + documentation.manuals: image: ${ci_image_name}/test:${ci_image_tag} stage: build diff --git a/.gitlab/ci/pipelines/documentation.daily.yml b/.gitlab/ci/pipelines/documentation.daily.yml index 44434b06bdbb..4f489e7824c5 100644 --- a/.gitlab/ci/pipelines/documentation.daily.yml +++ b/.gitlab/ci/pipelines/documentation.daily.yml @@ -114,6 +114,49 @@ documentation.rst-check: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' +documentation.docgen: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build + tags: + - gcp + needs: + - oc.docker:ci:amd64 + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh + script: + - eval $(opam env) + - make -C docs -j docexes-gen + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh + variables: + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 5G + artifacts: + expire_in: 1 week + paths: + - docs/alpha/rpc.rst + - docs/shell/rpc.rst + - docs/user/default-acl.json + - docs/api/errors.rst + - docs/shell/p2p_api.rst + documentation.odoc: image: ${ci_image_name}/test:${ci_image_tag} stage: build diff --git a/.gitlab/ci/pipelines/documentation.update.yml b/.gitlab/ci/pipelines/documentation.update.yml index b7df2a5ccb57..90f983f52ca1 100644 --- a/.gitlab/ci/pipelines/documentation.update.yml +++ b/.gitlab/ci/pipelines/documentation.update.yml @@ -95,6 +95,49 @@ oc.build:static-x86_64-linux-binaries: paths: - octez-binaries/$ARCH/* +documentation.docgen: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build + tags: + - gcp + needs: + - oc.docker:ci:amd64 + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh + script: + - eval $(opam env) + - make -C docs -j docexes-gen + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh + variables: + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 5G + artifacts: + expire_in: 1 week + paths: + - docs/alpha/rpc.rst + - docs/shell/rpc.rst + - docs/user/default-acl.json + - docs/api/errors.rst + - docs/shell/p2p_api.rst + documentation.manuals: image: ${ci_image_name}/test:${ci_image_tag} stage: build diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index d85bd8dd8d0e..56f38f701030 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -4163,71 +4163,20 @@ documentation:manuals: - dummy documentation:docgen: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build + stage: start tags: - gcp - rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - - changes: - - '**/*.rst' - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - - brassaia/**/*.ml* - - client-libs/**/*.ml* - - data-encoding/**/*.ml* - - docs/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/*.ml* - - grafazos/doc/**/* - - irmin/**/*.ml* - - script-inputs/**/*/ - - scripts/**/*/ - - src/**/*.ml* - - tezt/**/*.ml* - - vendors/**/*.ml* - when: on_success - needs: - - oc.docker:ci:amd64 - - trigger - dependencies: - - oc.docker:ci:amd64 + dependencies: [] timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push before_script: - . ./scripts/ci/datadog_send_job_info.sh - - eval $(opam env) - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh - script: - - make -C docs -j docexes-gen + script: [] after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G artifacts: - expire_in: 1 week paths: - - docs/alpha/rpc.rst - - docs/shell/rpc.rst - - docs/user/default-acl.json - - docs/api/errors.rst - - docs/shell/p2p_api.rst + - dummy documentation:build_all: image: ${ci_image_name}/test:${ci_image_tag} @@ -4483,6 +4432,71 @@ documentation.rst-check: after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' +documentation.docgen: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build + tags: + - gcp + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success + - changes: + - '**/*.rst' + - brassaia/**/*.ml* + - client-libs/**/*.ml* + - data-encoding/**/*.ml* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/*.ml* + - grafazos/doc/**/* + - irmin/**/*.ml* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/*.ml* + - tezt/**/*.ml* + - vendors/**/*.ml* + when: on_success + needs: + - oc.docker:ci:amd64 + - trigger + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh + script: + - eval $(opam env) + - make -C docs -j docexes-gen + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh + variables: + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 5G + artifacts: + expire_in: 1 week + paths: + - docs/alpha/rpc.rst + - docs/shell/rpc.rst + - docs/user/default-acl.json + - docs/api/errors.rst + - docs/shell/p2p_api.rst + documentation.manuals: image: ${ci_image_name}/test:${ci_image_tag} stage: build diff --git a/.gitlab/ci/pipelines/schedule_documentation.yml b/.gitlab/ci/pipelines/schedule_documentation.yml index c2dd3abb8e1e..1c3763a9a312 100644 --- a/.gitlab/ci/pipelines/schedule_documentation.yml +++ b/.gitlab/ci/pipelines/schedule_documentation.yml @@ -133,47 +133,21 @@ documentation:manuals: - dummy documentation:docgen: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build + stage: start tags: - gcp - needs: - - oc.docker:ci:amd64 - dependencies: - - oc.docker:ci:amd64 + dependencies: [] timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push interruptible: false before_script: - . ./scripts/ci/datadog_send_job_info.sh - - eval $(opam env) - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh - script: - - make -C docs -j docexes-gen + script: [] after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G artifacts: - expire_in: 1 week paths: - - docs/alpha/rpc.rst - - docs/shell/rpc.rst - - docs/user/default-acl.json - - docs/api/errors.rst - - docs/shell/p2p_api.rst + - dummy documentation:build_all: image: ${ci_image_name}/test:${ci_image_tag} diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 0a2dc80f0c8e..ccc32eac1d74 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -3684,47 +3684,21 @@ documentation:manuals: - dummy documentation:docgen: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build + stage: start tags: - gcp - rules: - - when: always - dependencies: - - oc.docker:ci:amd64 + dependencies: [] timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push interruptible: false before_script: - . ./scripts/ci/datadog_send_job_info.sh - - eval $(opam env) - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh - script: - - make -C docs -j docexes-gen + script: [] after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G artifacts: - expire_in: 1 week paths: - - docs/alpha/rpc.rst - - docs/shell/rpc.rst - - docs/user/default-acl.json - - docs/api/errors.rst - - docs/shell/p2p_api.rst + - dummy documentation:build_all: image: ${ci_image_name}/test:${ci_image_tag} diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index e5d9fca9ee5b..6ef12debf8df 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -1959,10 +1959,6 @@ let jobs pipeline_type = (* Doc jobs *) let doc = let jobs_documentation : tezos_job list = - let rules = - make_rules ~changes:changeset_octez_docs ~label:"ci--docs" () - in - let dependencies = dependencies_needs_start in let job_odoc = (* Dummy job to remove in a commit later in this MR. *) Tezos_ci.job @@ -1981,7 +1977,15 @@ let jobs pipeline_type = ~artifacts:(Gitlab_ci.Util.artifacts ["dummy"]) [] in - let job_docgen = Documentation.job_docgen ~rules ~dependencies () in + let job_docgen = + (* Dummy job to remove in a commit later in this MR. *) + Tezos_ci.job + ~__POS__ + ~stage:Tezos_ci.Stages.start + ~name:"documentation:docgen" + ~artifacts:(Gitlab_ci.Util.artifacts ["dummy"]) + [] + in let job_build_all = Documentation.job_build_all ~job_odoc diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 69c2e2990fed..ee18a467bf0d 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -1232,35 +1232,6 @@ module Documentation = struct jobs dependencies - (** Create the docgen job. - - This job builds various generated reference material, for - inclusion in the documentation. This includes the RPC, P2p and - error reference. - - This job is one of the prerequisites to {!job_build_all}. *) - let job_docgen ?rules ?dependencies () : tezos_job = - job - ~__POS__ - ~name:"documentation:docgen" - ~image:Images.CI.test - ~stage:Stages.build - ?dependencies - ?rules - ~before_script:(before_script ~eval_opam:true []) - ~artifacts: - (artifacts - ~expire_in:(Duration (Weeks 1)) - [ - "docs/alpha/rpc.rst"; - "docs/shell/rpc.rst"; - "docs/user/default-acl.json"; - "docs/api/errors.rst"; - "docs/shell/p2p_api.rst"; - ]) - ["make -C docs -j docexes-gen"] - |> enable_cargo_cache |> enable_sccache - (** Create the [documentation:build_all] job. This jobs builds the RST sources in docs, which will include the diff --git a/ci/bin/documentation.ml b/ci/bin/documentation.ml index ba3b5bb4e2c1..532830f26173 100644 --- a/ci/bin/documentation.ml +++ b/ci/bin/documentation.ml @@ -183,6 +183,31 @@ let job_manuals = | `dynamic -> ["make -C docs -j octez-gen"] | `static -> ["scripts/ci/documentation:manuals_static.sh"])) +let job_docgen = + CI.job + "docgen" + ~__POS__ + ~image:Tezos_ci.Images.CI.test + ~stage:Build + ~description: + "Build various generated reference material. This includes the RPC, P2P \ + and error reference." + ~only_if_changed:Files.odoc + ~force_if_label:["ci--docs"] + ~artifacts: + (Gitlab_ci.Util.artifacts + ~expire_in:(Duration (Weeks 1)) + [ + "docs/alpha/rpc.rst"; + "docs/shell/rpc.rst"; + "docs/user/default-acl.json"; + "docs/api/errors.rst"; + "docs/shell/p2p_api.rst"; + ]) + ~cargo_cache:true + ~sccache:(Cacio.sccache ()) + ["eval $(opam env)"; "make -C docs -j docexes-gen"] + let register () = CI.register_before_merging_jobs [ @@ -190,6 +215,7 @@ let register () = (Auto, job_install_python `debian_bookworm `current_branch); (Auto, job_odoc `lite); (Auto, job_manuals `dynamic); + (Auto, job_docgen); ] ; CI.register_scheduled_pipeline "daily" @@ -202,6 +228,7 @@ let register () = (Auto, job_install_python `debian_bookworm `master); (Auto, job_odoc `lite); (Auto, job_manuals `static); + (Auto, job_docgen); ] ; CI.register_scheduled_pipeline "update" @@ -209,5 +236,5 @@ let register () = "Generate and push the documentation to octez.com/docs without being \ interrupted." ~legacy_jobs:[Master_branch.job_static_x86_64] - [(Auto, job_odoc `full); (Auto, job_manuals `static)] ; + [(Auto, job_odoc `full); (Auto, job_manuals `static); (Auto, job_docgen)] ; () diff --git a/ci/bin/master_branch.ml b/ci/bin/master_branch.ml index f901dae70de5..482b27bbb789 100644 --- a/ci/bin/master_branch.ml +++ b/ci/bin/master_branch.ml @@ -46,7 +46,6 @@ let job_static_x86_64 = () let jobs_documentation : tezos_job list = - let dependencies = Dependent [] in let job_odoc = (* Dummy job to remove in a commit later in this MR. *) Tezos_ci.job @@ -65,7 +64,15 @@ let jobs_documentation : tezos_job list = ~artifacts:(Gitlab_ci.Util.artifacts ["dummy"]) [] in - let job_docgen = Documentation.job_docgen ~dependencies () in + let job_docgen = + (* Dummy job to remove in a commit later in this MR. *) + Tezos_ci.job + ~__POS__ + ~stage:Tezos_ci.Stages.start + ~name:"documentation:docgen" + ~artifacts:(Gitlab_ci.Util.artifacts ["dummy"]) + [] + in let job_build_all = Documentation.job_build_all ~job_odoc ~job_manuals ~job_docgen () in -- GitLab From 464514711af1ce863aaa3b5261ec82dc6afd3109 Mon Sep 17 00:00:00 2001 From: Romain Date: Wed, 27 Aug 2025 16:35:02 +0200 Subject: [PATCH 6/9] CI: migrate documentation:build_all to Cacio --- .gitlab/ci/pipelines/before_merging.yml | 106 ++++++++++-------- .gitlab/ci/pipelines/documentation.daily.yml | 33 ++++++ .gitlab/ci/pipelines/documentation.update.yml | 33 ++++++ .gitlab/ci/pipelines/merge_train.yml | 106 ++++++++++-------- .../ci/pipelines/schedule_documentation.yml | 24 +--- .../ci/pipelines/schedule_extended_test.yml | 26 +---- ci/bin/code_verification.ml | 18 ++- ci/bin/common.ml | 27 ----- ci/bin/documentation.ml | 40 +++++-- ci/bin/master_branch.ml | 8 +- 10 files changed, 239 insertions(+), 182 deletions(-) diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 0c41fc45277e..d20677dc008e 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -4180,60 +4180,20 @@ documentation:docgen: - dummy documentation:build_all: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build + stage: start tags: - gcp - rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - - changes: - - '**/*.rst' - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - - brassaia/**/*.ml* - - client-libs/**/*.ml* - - data-encoding/**/*.ml* - - docs/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/*.ml* - - grafazos/doc/**/* - - irmin/**/*.ml* - - script-inputs/**/*/ - - scripts/**/*/ - - src/**/*.ml* - - tezt/**/*.ml* - - vendors/**/*.ml* - when: on_success - needs: - - oc.docker:ci:amd64 - - documentation:odoc - - documentation:manuals - - documentation:docgen - dependencies: - - oc.docker:ci:amd64 - - documentation:odoc - - documentation:manuals - - documentation:docgen + dependencies: [] timeout: 60 minutes before_script: - . ./scripts/ci/datadog_send_job_info.sh - - eval $(opam env) - - . $HOME/.venv/bin/activate - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - make -C docs -j sphinx - - make -C docs -j _build/octezdoc.txt + script: [] after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' artifacts: - expire_in: 1 week paths: - - docs/_build/ - expose_as: Documentation - excluding old protocols + - dummy documentation:linkcheck: image: ${ci_image_name}/test:${ci_image_tag} @@ -4526,7 +4486,6 @@ documentation.docgen: when: on_success needs: - oc.docker:ci:amd64 - - trigger dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -4591,7 +4550,6 @@ documentation.manuals: when: on_success needs: - oc.docker:ci:amd64 - - trigger - oc.build_x86_64-released - oc.build_amd64-extra-dev - oc.build_amd64-extra-exp @@ -4651,7 +4609,6 @@ documentation.odoc: when: on_success needs: - oc.docker:ci:amd64 - - trigger dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -4691,6 +4648,61 @@ documentation.odoc: - docs/odoc.log when: always +documentation.build_all: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build + tags: + - gcp + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success + - changes: + - '**/*.rst' + - brassaia/**/*.ml* + - client-libs/**/*.ml* + - data-encoding/**/*.ml* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/*.ml* + - grafazos/doc/**/* + - irmin/**/*.ml* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/*.ml* + - tezt/**/*.ml* + - vendors/**/*.ml* + when: on_success + needs: + - oc.docker:ci:amd64 + - trigger + - documentation.odoc + - documentation.manuals + - documentation.docgen + dependencies: + - oc.docker:ci:amd64 + - documentation.odoc + - documentation.manuals + - documentation.docgen + timeout: 60 minutes + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - eval $(opam env) + - . $HOME/.venv/bin/activate + - make -C docs -j sphinx + - make -C docs -j _build/octezdoc.txt + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + artifacts: + expire_in: 1 week + paths: + - docs/_build/ + expose_as: Documentation - excluding old protocols + documentation.install_python_bookworm: image: debian:bookworm stage: test diff --git a/.gitlab/ci/pipelines/documentation.daily.yml b/.gitlab/ci/pipelines/documentation.daily.yml index 4f489e7824c5..39e199c4c334 100644 --- a/.gitlab/ci/pipelines/documentation.daily.yml +++ b/.gitlab/ci/pipelines/documentation.daily.yml @@ -233,6 +233,39 @@ documentation.manuals: - docs/developer/rollup_metrics.csv - docs/user/node-config.json +documentation.build_all: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build + tags: + - gcp + needs: + - oc.docker:ci:amd64 + - documentation.odoc + - documentation.manuals + - documentation.docgen + dependencies: + - oc.docker:ci:amd64 + - documentation.odoc + - documentation.manuals + - documentation.docgen + timeout: 60 minutes + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - eval $(opam env) + - . $HOME/.venv/bin/activate + - make -C docs -j sphinx + - make -C docs -j _build/octezdoc.txt + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + artifacts: + expire_in: 1 week + paths: + - docs/_build/ + expose_as: Documentation - excluding old protocols + documentation.install_python_bookworm: image: debian:bookworm stage: test diff --git a/.gitlab/ci/pipelines/documentation.update.yml b/.gitlab/ci/pipelines/documentation.update.yml index 90f983f52ca1..8f179ccbb71b 100644 --- a/.gitlab/ci/pipelines/documentation.update.yml +++ b/.gitlab/ci/pipelines/documentation.update.yml @@ -213,3 +213,36 @@ documentation.odoc: - docs/_build/api/odoc/ - docs/odoc.log when: always + +documentation.build_all: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build + tags: + - gcp + needs: + - oc.docker:ci:amd64 + - documentation.odoc + - documentation.manuals + - documentation.docgen + dependencies: + - oc.docker:ci:amd64 + - documentation.odoc + - documentation.manuals + - documentation.docgen + timeout: 60 minutes + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - eval $(opam env) + - . $HOME/.venv/bin/activate + - make -C docs -j sphinx + - make -C docs -j _build/octezdoc.txt + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + artifacts: + expire_in: 1 week + paths: + - docs/_build/ + expose_as: Documentation - excluding old protocols diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index 56f38f701030..4da63d7e8ff7 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -4179,60 +4179,20 @@ documentation:docgen: - dummy documentation:build_all: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build + stage: start tags: - gcp - rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - - changes: - - '**/*.rst' - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - - brassaia/**/*.ml* - - client-libs/**/*.ml* - - data-encoding/**/*.ml* - - docs/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/*.ml* - - grafazos/doc/**/* - - irmin/**/*.ml* - - script-inputs/**/*/ - - scripts/**/*/ - - src/**/*.ml* - - tezt/**/*.ml* - - vendors/**/*.ml* - when: on_success - needs: - - oc.docker:ci:amd64 - - documentation:odoc - - documentation:manuals - - documentation:docgen - dependencies: - - oc.docker:ci:amd64 - - documentation:odoc - - documentation:manuals - - documentation:docgen + dependencies: [] timeout: 60 minutes before_script: - . ./scripts/ci/datadog_send_job_info.sh - - eval $(opam env) - - . $HOME/.venv/bin/activate - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - make -C docs -j sphinx - - make -C docs -j _build/octezdoc.txt + script: [] after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' artifacts: - expire_in: 1 week paths: - - docs/_build/ - expose_as: Documentation - excluding old protocols + - dummy documentation:linkcheck: image: ${ci_image_name}/test:${ci_image_tag} @@ -4460,7 +4420,6 @@ documentation.docgen: when: on_success needs: - oc.docker:ci:amd64 - - trigger dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -4525,7 +4484,6 @@ documentation.manuals: when: on_success needs: - oc.docker:ci:amd64 - - trigger - oc.build_x86_64-released - oc.build_amd64-extra-dev - oc.build_amd64-extra-exp @@ -4585,7 +4543,6 @@ documentation.odoc: when: on_success needs: - oc.docker:ci:amd64 - - trigger dependencies: - oc.docker:ci:amd64 timeout: 60 minutes @@ -4625,6 +4582,61 @@ documentation.odoc: - docs/odoc.log when: always +documentation.build_all: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build + tags: + - gcp + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success + - changes: + - '**/*.rst' + - brassaia/**/*.ml* + - client-libs/**/*.ml* + - data-encoding/**/*.ml* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/*.ml* + - grafazos/doc/**/* + - irmin/**/*.ml* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/*.ml* + - tezt/**/*.ml* + - vendors/**/*.ml* + when: on_success + needs: + - oc.docker:ci:amd64 + - trigger + - documentation.odoc + - documentation.manuals + - documentation.docgen + dependencies: + - oc.docker:ci:amd64 + - documentation.odoc + - documentation.manuals + - documentation.docgen + timeout: 60 minutes + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - eval $(opam env) + - . $HOME/.venv/bin/activate + - make -C docs -j sphinx + - make -C docs -j _build/octezdoc.txt + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + artifacts: + expire_in: 1 week + paths: + - docs/_build/ + expose_as: Documentation - excluding old protocols + documentation.install_python_bookworm: image: debian:bookworm stage: test diff --git a/.gitlab/ci/pipelines/schedule_documentation.yml b/.gitlab/ci/pipelines/schedule_documentation.yml index 1c3763a9a312..32b1aa314a51 100644 --- a/.gitlab/ci/pipelines/schedule_documentation.yml +++ b/.gitlab/ci/pipelines/schedule_documentation.yml @@ -150,37 +150,21 @@ documentation:docgen: - dummy documentation:build_all: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build + stage: start tags: - gcp - needs: - - oc.docker:ci:amd64 - - documentation:odoc - - documentation:manuals - - documentation:docgen - dependencies: - - oc.docker:ci:amd64 - - documentation:odoc - - documentation:manuals - - documentation:docgen + dependencies: [] timeout: 60 minutes interruptible: false before_script: - . ./scripts/ci/datadog_send_job_info.sh - - eval $(opam env) - - . $HOME/.venv/bin/activate - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - make -C docs -j sphinx - - make -C docs -j _build/octezdoc.txt + script: [] after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' artifacts: - expire_in: 1 week paths: - - docs/_build/ - expose_as: Documentation - excluding old protocols + - dummy documentation:publish: image: ${ci_image_name}/test:${ci_image_tag} diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index ccc32eac1d74..fc42a0466e3e 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -3701,39 +3701,21 @@ documentation:docgen: - dummy documentation:build_all: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build + stage: start tags: - gcp - rules: - - when: on_success - needs: - - oc.docker:ci:amd64 - - documentation:odoc - - documentation:manuals - - documentation:docgen - dependencies: - - oc.docker:ci:amd64 - - documentation:odoc - - documentation:manuals - - documentation:docgen + dependencies: [] timeout: 60 minutes interruptible: false before_script: - . ./scripts/ci/datadog_send_job_info.sh - - eval $(opam env) - - . $HOME/.venv/bin/activate - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - make -C docs -j sphinx - - make -C docs -j _build/octezdoc.txt + script: [] after_script: - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' artifacts: - expire_in: 1 week paths: - - docs/_build/ - expose_as: Documentation - excluding old protocols + - dummy documentation:linkcheck: image: ${ci_image_name}/test:${ci_image_tag} diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 6ef12debf8df..0f651f81f6e9 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -1987,17 +1987,13 @@ let jobs pipeline_type = [] in let job_build_all = - Documentation.job_build_all - ~job_odoc - ~job_manuals - ~job_docgen - ~rules: - (make_rules - ~dependent:true - ~changes:changeset_octez_docs - ~label:"ci--docs" - ()) - () + (* Dummy job to remove in a commit later in this MR. *) + Tezos_ci.job + ~__POS__ + ~stage:Tezos_ci.Stages.start + ~name:"documentation:build_all" + ~artifacts:(Gitlab_ci.Util.artifacts ["dummy"]) + [] in let job_documentation_linkcheck : tezos_job = Documentation.job_linkcheck diff --git a/ci/bin/common.ml b/ci/bin/common.ml index ee18a467bf0d..73a520378d1e 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -1232,33 +1232,6 @@ module Documentation = struct jobs dependencies - (** Create the [documentation:build_all] job. - - This jobs builds the RST sources in docs, which will include the - the material from the prerequisite jobs: {!job_odoc}, - {!job_manuals}, {!job_docgen}, on whose artifacts the - [documentation:build_all] job will depend. *) - let job_build_all ~job_odoc ~job_manuals ~job_docgen ?dependencies ?rules () : - tezos_job = - let dependencies = - mk_artifact_dependencies ?dependencies [job_odoc; job_manuals; job_docgen] - in - job - ~__POS__ - ~name:"documentation:build_all" - ~image:Images.CI.test - ~stage:Stages.build - ~dependencies - ?rules - ~before_script:(before_script ~eval_opam:true ~init_python_venv:true []) - ~artifacts: - (artifacts - ~expose_as:"Documentation - excluding old protocols" - ~expire_in:(Duration (Weeks 1)) - (* Path must be terminated with / to expose artifact (gitlab-org/gitlab#/36706) *) - ["docs/_build/"]) - ["make -C docs -j sphinx"; "make -C docs -j _build/octezdoc.txt"] - (** Create a [documentation:linkcheck] job. *) let job_linkcheck ~job_manuals ~job_docgen ~job_build_all ?dependencies ?rules () : tezos_job = diff --git a/ci/bin/documentation.ml b/ci/bin/documentation.ml index 532830f26173..92057d705275 100644 --- a/ci/bin/documentation.ml +++ b/ci/bin/documentation.ml @@ -208,14 +208,42 @@ let job_docgen = ~sccache:(Cacio.sccache ()) ["eval $(opam env)"; "make -C docs -j docexes-gen"] +let job_build_all = + Cacio.parameterize @@ fun mode -> + Cacio.parameterize @@ fun executables_to_use -> + CI.job + "build_all" + ~__POS__ + ~image:Tezos_ci.Images.CI.test + ~stage:Build + ~description:"Build the RSTs. Include material from previous build jobs." + ~only_if_changed:Files.odoc + ~force_if_label:["ci--docs"] + ~needs: + [ + (Artifacts, job_odoc mode); + (Artifacts, job_manuals executables_to_use); + (Artifacts, job_docgen); + ] + ~artifacts: + (Gitlab_ci.Util.artifacts + ~expose_as:"Documentation - excluding old protocols" + ~expire_in:(Duration (Weeks 1)) + (* Path must be terminated with / to expose artifact (gitlab-org/gitlab#/36706) *) + ["docs/_build/"]) + [ + "eval $(opam env)"; + ". $HOME/.venv/bin/activate"; + "make -C docs -j sphinx"; + "make -C docs -j _build/octezdoc.txt"; + ] + let register () = CI.register_before_merging_jobs [ (Immediate, job_rst_check); (Auto, job_install_python `debian_bookworm `current_branch); - (Auto, job_odoc `lite); - (Auto, job_manuals `dynamic); - (Auto, job_docgen); + (Auto, job_build_all `lite `dynamic); ] ; CI.register_scheduled_pipeline "daily" @@ -226,9 +254,7 @@ let register () = (Auto, job_install_python `ubuntu_noble `master); (Auto, job_install_python `ubuntu_jammy `master); (Auto, job_install_python `debian_bookworm `master); - (Auto, job_odoc `lite); - (Auto, job_manuals `static); - (Auto, job_docgen); + (Auto, job_build_all `lite `static); ] ; CI.register_scheduled_pipeline "update" @@ -236,5 +262,5 @@ let register () = "Generate and push the documentation to octez.com/docs without being \ interrupted." ~legacy_jobs:[Master_branch.job_static_x86_64] - [(Auto, job_odoc `full); (Auto, job_manuals `static); (Auto, job_docgen)] ; + [(Auto, job_build_all `full `static)] ; () diff --git a/ci/bin/master_branch.ml b/ci/bin/master_branch.ml index 482b27bbb789..b22758fe3c7e 100644 --- a/ci/bin/master_branch.ml +++ b/ci/bin/master_branch.ml @@ -74,7 +74,13 @@ let jobs_documentation : tezos_job list = [] in let job_build_all = - Documentation.job_build_all ~job_odoc ~job_manuals ~job_docgen () + (* Dummy job to remove in a commit later in this MR. *) + Tezos_ci.job + ~__POS__ + ~stage:Tezos_ci.Stages.start + ~name:"documentation:build_all" + ~artifacts:(Gitlab_ci.Util.artifacts ["dummy"]) + [] in let job_publish_documentation : tezos_job = Documentation.job_publish_documentation ~job_build_all () -- GitLab From e22aac3ea7d30d57cc75d5bb83a8f1aa19784287 Mon Sep 17 00:00:00 2001 From: Romain Date: Wed, 27 Aug 2025 16:53:39 +0200 Subject: [PATCH 7/9] CI: migrate documentation:linkcheck to Cacio --- .gitlab/ci/pipelines/before_merging.yml | 172 ++++++------------ .gitlab/ci/pipelines/documentation.daily.yml | 30 +++ .gitlab/ci/pipelines/merge_train.yml | 172 ++++++------------ .../ci/pipelines/schedule_extended_test.yml | 100 ---------- ci/bin/code_verification.ml | 65 +------ ci/bin/common.ml | 28 --- ci/bin/documentation.ml | 27 +++ 7 files changed, 166 insertions(+), 428 deletions(-) diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index d20677dc008e..dbb92e5650f3 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -4131,124 +4131,6 @@ oc.unified_coverage: expose_as: Coverage report coverage: '/Coverage: ([^%]+%)/' -documentation:odoc: - stage: start - tags: - - gcp - dependencies: [] - timeout: 60 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: [] - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - artifacts: - paths: - - dummy - -documentation:manuals: - stage: start - tags: - - gcp - dependencies: [] - timeout: 60 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: [] - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - artifacts: - paths: - - dummy - -documentation:docgen: - stage: start - tags: - - gcp - dependencies: [] - timeout: 60 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: [] - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - artifacts: - paths: - - dummy - -documentation:build_all: - stage: start - tags: - - gcp - dependencies: [] - timeout: 60 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: [] - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - artifacts: - paths: - - dummy - -documentation:linkcheck: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - - changes: - - '**/*.rst' - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - - brassaia/**/*.ml* - - client-libs/**/*.ml* - - data-encoding/**/*.ml* - - docs/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/*.ml* - - grafazos/doc/**/* - - irmin/**/*.ml* - - script-inputs/**/*/ - - scripts/**/*/ - - src/**/*.ml* - - tezt/**/*.ml* - - vendors/**/*.ml* - when: manual - allow_failure: true - needs: - - oc.docker:ci:amd64 - - documentation:manuals - - documentation:docgen - - documentation:build_all - dependencies: - - oc.docker:ci:amd64 - - documentation:manuals - - documentation:docgen - - documentation:build_all - allow_failure: true - timeout: 60 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/version.sh - - eval $(opam env) - - . $HOME/.venv/bin/activate - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - make -C docs redirectcheck - - make -C docs linkcheck - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - trigger:homebrew: stage: manual variables: @@ -4703,6 +4585,60 @@ documentation.build_all: - docs/_build/ expose_as: Documentation - excluding old protocols +documentation.linkcheck: + image: ${ci_image_name}/test:${ci_image_tag} + stage: test + tags: + - gcp + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: manual + allow_failure: true + - changes: + - '**/*.rst' + - brassaia/**/*.ml* + - client-libs/**/*.ml* + - data-encoding/**/*.ml* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/*.ml* + - grafazos/doc/**/* + - irmin/**/*.ml* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/*.ml* + - tezt/**/*.ml* + - vendors/**/*.ml* + when: manual + allow_failure: true + needs: + - oc.docker:ci:amd64 + - trigger + - documentation.manuals + - documentation.docgen + - documentation.build_all + dependencies: + - oc.docker:ci:amd64 + - documentation.manuals + - documentation.docgen + - documentation.build_all + allow_failure: true + timeout: 60 minutes + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - . ./scripts/version.sh + - eval $(opam env) + - . $HOME/.venv/bin/activate + - make -C docs redirectcheck + - make -C docs linkcheck + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + documentation.install_python_bookworm: image: debian:bookworm stage: test diff --git a/.gitlab/ci/pipelines/documentation.daily.yml b/.gitlab/ci/pipelines/documentation.daily.yml index 39e199c4c334..56a9824c9b50 100644 --- a/.gitlab/ci/pipelines/documentation.daily.yml +++ b/.gitlab/ci/pipelines/documentation.daily.yml @@ -266,6 +266,36 @@ documentation.build_all: - docs/_build/ expose_as: Documentation - excluding old protocols +documentation.linkcheck: + image: ${ci_image_name}/test:${ci_image_tag} + stage: test + tags: + - gcp + needs: + - oc.docker:ci:amd64 + - documentation.manuals + - documentation.docgen + - documentation.build_all + dependencies: + - oc.docker:ci:amd64 + - documentation.manuals + - documentation.docgen + - documentation.build_all + allow_failure: true + timeout: 60 minutes + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - . ./scripts/version.sh + - eval $(opam env) + - . $HOME/.venv/bin/activate + - make -C docs redirectcheck + - make -C docs linkcheck + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + documentation.install_python_bookworm: image: debian:bookworm stage: test diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index 4da63d7e8ff7..e76f8040a6e6 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -4130,124 +4130,6 @@ oc.unified_coverage: expose_as: Coverage report coverage: '/Coverage: ([^%]+%)/' -documentation:odoc: - stage: start - tags: - - gcp - dependencies: [] - timeout: 60 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: [] - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - artifacts: - paths: - - dummy - -documentation:manuals: - stage: start - tags: - - gcp - dependencies: [] - timeout: 60 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: [] - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - artifacts: - paths: - - dummy - -documentation:docgen: - stage: start - tags: - - gcp - dependencies: [] - timeout: 60 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: [] - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - artifacts: - paths: - - dummy - -documentation:build_all: - stage: start - tags: - - gcp - dependencies: [] - timeout: 60 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: [] - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - artifacts: - paths: - - dummy - -documentation:linkcheck: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ - when: on_success - - changes: - - '**/*.rst' - - .gitlab-ci.yml - - .gitlab/ci/pipelines/before_merging.yml - - .gitlab/ci/pipelines/merge_train.yml - - brassaia/**/*.ml* - - client-libs/**/*.ml* - - data-encoding/**/*.ml* - - docs/**/* - - dune - - dune-project - - dune-workspace - - etherlink/**/*.ml* - - grafazos/doc/**/* - - irmin/**/*.ml* - - script-inputs/**/*/ - - scripts/**/*/ - - src/**/*.ml* - - tezt/**/*.ml* - - vendors/**/*.ml* - when: manual - allow_failure: true - needs: - - oc.docker:ci:amd64 - - documentation:manuals - - documentation:docgen - - documentation:build_all - dependencies: - - oc.docker:ci:amd64 - - documentation:manuals - - documentation:docgen - - documentation:build_all - allow_failure: true - timeout: 60 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/version.sh - - eval $(opam env) - - . $HOME/.venv/bin/activate - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - make -C docs redirectcheck - - make -C docs linkcheck - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - oc.docker:amd64: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 stage: manual @@ -4637,6 +4519,60 @@ documentation.build_all: - docs/_build/ expose_as: Documentation - excluding old protocols +documentation.linkcheck: + image: ${ci_image_name}/test:${ci_image_tag} + stage: test + tags: + - gcp + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: manual + allow_failure: true + - changes: + - '**/*.rst' + - brassaia/**/*.ml* + - client-libs/**/*.ml* + - data-encoding/**/*.ml* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/*.ml* + - grafazos/doc/**/* + - irmin/**/*.ml* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/*.ml* + - tezt/**/*.ml* + - vendors/**/*.ml* + when: manual + allow_failure: true + needs: + - oc.docker:ci:amd64 + - trigger + - documentation.manuals + - documentation.docgen + - documentation.build_all + dependencies: + - oc.docker:ci:amd64 + - documentation.manuals + - documentation.docgen + - documentation.build_all + allow_failure: true + timeout: 60 minutes + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - . ./scripts/version.sh + - eval $(opam env) + - . $HOME/.venv/bin/activate + - make -C docs redirectcheck + - make -C docs linkcheck + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + documentation.install_python_bookworm: image: debian:bookworm stage: test diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index fc42a0466e3e..99c7b8b1fc3e 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -3648,103 +3648,3 @@ tezt-riscv-slow-sequential: junit: $JUNIT when: always retry: 2 - -documentation:odoc: - stage: start - tags: - - gcp - dependencies: [] - timeout: 60 minutes - interruptible: false - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: [] - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - artifacts: - paths: - - dummy - -documentation:manuals: - stage: start - tags: - - gcp - dependencies: [] - timeout: 60 minutes - interruptible: false - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: [] - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - artifacts: - paths: - - dummy - -documentation:docgen: - stage: start - tags: - - gcp - dependencies: [] - timeout: 60 minutes - interruptible: false - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: [] - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - artifacts: - paths: - - dummy - -documentation:build_all: - stage: start - tags: - - gcp - dependencies: [] - timeout: 60 minutes - interruptible: false - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: [] - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - artifacts: - paths: - - dummy - -documentation:linkcheck: - image: ${ci_image_name}/test:${ci_image_tag} - stage: test - tags: - - gcp - rules: - - when: on_success - needs: - - oc.docker:ci:amd64 - - documentation:manuals - - documentation:docgen - - documentation:build_all - dependencies: - - oc.docker:ci:amd64 - - documentation:manuals - - documentation:docgen - - documentation:build_all - allow_failure: true - timeout: 60 minutes - interruptible: false - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/version.sh - - eval $(opam env) - - . $HOME/.venv/bin/activate - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - make -C docs redirectcheck - - make -C docs linkcheck - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 0f651f81f6e9..d2ab1133d66b 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -1956,69 +1956,6 @@ let jobs pipeline_type = | Schedule_extended_test -> [] in - (* Doc jobs *) - let doc = - let jobs_documentation : tezos_job list = - let job_odoc = - (* Dummy job to remove in a commit later in this MR. *) - Tezos_ci.job - ~__POS__ - ~stage:Tezos_ci.Stages.start - ~name:"documentation:odoc" - ~artifacts:(Gitlab_ci.Util.artifacts ["dummy"]) - [] - in - let job_manuals = - (* Dummy job to remove in a commit later in this MR. *) - Tezos_ci.job - ~__POS__ - ~stage:Tezos_ci.Stages.start - ~name:"documentation:manuals" - ~artifacts:(Gitlab_ci.Util.artifacts ["dummy"]) - [] - in - let job_docgen = - (* Dummy job to remove in a commit later in this MR. *) - Tezos_ci.job - ~__POS__ - ~stage:Tezos_ci.Stages.start - ~name:"documentation:docgen" - ~artifacts:(Gitlab_ci.Util.artifacts ["dummy"]) - [] - in - let job_build_all = - (* Dummy job to remove in a commit later in this MR. *) - Tezos_ci.job - ~__POS__ - ~stage:Tezos_ci.Stages.start - ~name:"documentation:build_all" - ~artifacts:(Gitlab_ci.Util.artifacts ["dummy"]) - [] - in - let job_documentation_linkcheck : tezos_job = - Documentation.job_linkcheck - ~job_manuals - ~job_docgen - ~job_build_all - ~rules: - (make_rules - ~dependent:true - ~label:"ci--docs" - ~manual:(On_changes changeset_octez_docs) - ()) - () - in - [ - job_odoc; - job_manuals; - job_docgen; - job_build_all; - job_documentation_linkcheck; - ] - in - jobs_documentation - in - (* Manual jobs *) let manual = (* On scheduled pipelines we build and test the full packages test matrix. @@ -2144,4 +2081,4 @@ let jobs pipeline_type = (* No manual jobs on the scheduled pipeline *) | Schedule_extended_test -> [] in - start_stage @ sanity @ build @ packaging @ test @ coverage @ doc @ manual + start_stage @ sanity @ build @ packaging @ test @ coverage @ manual diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 73a520378d1e..604add1bfc6e 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -1232,34 +1232,6 @@ module Documentation = struct jobs dependencies - (** Create a [documentation:linkcheck] job. *) - let job_linkcheck ~job_manuals ~job_docgen ~job_build_all ?dependencies ?rules - () : tezos_job = - let dependencies = - mk_artifact_dependencies - ?dependencies - [job_manuals; job_docgen; job_build_all] - in - job - ~__POS__ - ~name:"documentation:linkcheck" - ~image:Images.CI.test - ~stage:Stages.test - ~dependencies - (* Warning: the [documentation:linkcheck] job must have at least the same - restrictions in the rules as [documentation:build_all], otherwise the CI - may complain that [documentation:linkcheck] depends on [documentation:build_all] - which does not exist. *) - ?rules - ~allow_failure:Yes - ~before_script: - (before_script - ~source_version:true - ~eval_opam:true - ~init_python_venv:true - []) - ["make -C docs redirectcheck"; "make -C docs linkcheck"] - (** Create a [publish:documentation] job. This job, for inclusion on [master_branch] pipelines, publishes diff --git a/ci/bin/documentation.ml b/ci/bin/documentation.ml index 92057d705275..b4d769c8edfb 100644 --- a/ci/bin/documentation.ml +++ b/ci/bin/documentation.ml @@ -238,12 +238,38 @@ let job_build_all = "make -C docs -j _build/octezdoc.txt"; ] +let job_linkcheck = + Cacio.parameterize @@ fun mode executables_to_use -> + CI.job + "linkcheck" + ~__POS__ + ~image:Tezos_ci.Images.CI.test + ~stage:Test + ~description:"Check links in the documentation." + ~only_if_changed:Files.odoc + ~force_if_label:["ci--docs"] + ~needs: + [ + (Artifacts, job_manuals executables_to_use); + (Artifacts, job_docgen); + (Artifacts, job_build_all mode executables_to_use); + ] + ~allow_failure:Yes + [ + ". ./scripts/version.sh"; + "eval $(opam env)"; + ". $HOME/.venv/bin/activate"; + "make -C docs redirectcheck"; + "make -C docs linkcheck"; + ] + let register () = CI.register_before_merging_jobs [ (Immediate, job_rst_check); (Auto, job_install_python `debian_bookworm `current_branch); (Auto, job_build_all `lite `dynamic); + (Manual, job_linkcheck `lite `dynamic); ] ; CI.register_scheduled_pipeline "daily" @@ -255,6 +281,7 @@ let register () = (Auto, job_install_python `ubuntu_jammy `master); (Auto, job_install_python `debian_bookworm `master); (Auto, job_build_all `lite `static); + (Auto, job_linkcheck `lite `static); ] ; CI.register_scheduled_pipeline "update" -- GitLab From a43b53a7e5610b3076229a05f959e5ca5fc3dfe8 Mon Sep 17 00:00:00 2001 From: Romain Date: Wed, 27 Aug 2025 16:59:58 +0200 Subject: [PATCH 8/9] CI: migrate documentation:publish to Cacio --- .gitlab-ci.yml | 8 - .gitlab/ci/pipelines/documentation.daily.yml | 60 ++--- .gitlab/ci/pipelines/documentation.update.yml | 43 ++++ .../ci/pipelines/schedule_documentation.yml | 209 ------------------ ci/bin/common.ml | 48 ---- ci/bin/documentation.ml | 21 +- ci/bin/main.ml | 10 - ci/bin/master_branch.ml | 46 ---- 8 files changed, 93 insertions(+), 352 deletions(-) delete mode 100644 .gitlab/ci/pipelines/schedule_documentation.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 168e7b8019af..6ca7bb8f4888 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -170,10 +170,6 @@ workflow: variables: PIPELINE_TYPE: schedule_security_scans when: always - - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "DOCUMENTATION" - variables: - PIPELINE_TYPE: schedule_documentation - when: always - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "DOCKER_FRESH_IMAGE_BUILD" variables: PIPELINE_TYPE: schedule_docker_build_pipeline @@ -393,10 +389,6 @@ include: rules: - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "SECURITY_SCANS" when: always -- local: .gitlab/ci/pipelines/schedule_documentation.yml - rules: - - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "DOCUMENTATION" - when: always - local: .gitlab/ci/pipelines/schedule_docker_build_pipeline.yml rules: - if: $CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "DOCKER_FRESH_IMAGE_BUILD" diff --git a/.gitlab/ci/pipelines/documentation.daily.yml b/.gitlab/ci/pipelines/documentation.daily.yml index 56a9824c9b50..22adfe92e1b8 100644 --- a/.gitlab/ci/pipelines/documentation.daily.yml +++ b/.gitlab/ci/pipelines/documentation.daily.yml @@ -157,6 +157,36 @@ documentation.docgen: - docs/api/errors.rst - docs/shell/p2p_api.rst +documentation.manuals: + image: ${ci_image_name}/test:${ci_image_tag} + stage: build + tags: + - gcp + needs: + - oc.docker:ci:amd64 + - oc.build:static-x86_64-linux-binaries + dependencies: + - oc.docker:ci:amd64 + - oc.build:static-x86_64-linux-binaries + timeout: 60 minutes + interruptible: true + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + script: + - eval $(opam env) + - scripts/ci/documentation:manuals_static.sh + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + artifacts: + expire_in: 1 week + paths: + - docs/*/octez-*.html + - docs/api/octez-*.txt + - docs/developer/metrics.csv + - docs/developer/rollup_metrics.csv + - docs/user/node-config.json + documentation.odoc: image: ${ci_image_name}/test:${ci_image_tag} stage: build @@ -203,36 +233,6 @@ documentation.odoc: - docs/odoc.log when: always -documentation.manuals: - image: ${ci_image_name}/test:${ci_image_tag} - stage: build - tags: - - gcp - needs: - - oc.docker:ci:amd64 - - oc.build:static-x86_64-linux-binaries - dependencies: - - oc.docker:ci:amd64 - - oc.build:static-x86_64-linux-binaries - timeout: 60 minutes - interruptible: true - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - eval $(opam env) - - scripts/ci/documentation:manuals_static.sh - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - artifacts: - expire_in: 1 week - paths: - - docs/*/octez-*.html - - docs/api/octez-*.txt - - docs/developer/metrics.csv - - docs/developer/rollup_metrics.csv - - docs/user/node-config.json - documentation.build_all: image: ${ci_image_name}/test:${ci_image_tag} stage: build diff --git a/.gitlab/ci/pipelines/documentation.update.yml b/.gitlab/ci/pipelines/documentation.update.yml index 8f179ccbb71b..74fc7974c0ac 100644 --- a/.gitlab/ci/pipelines/documentation.update.yml +++ b/.gitlab/ci/pipelines/documentation.update.yml @@ -5,6 +5,7 @@ stages: - start - images - build +- publish oc.docker:ci:amd64: image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 @@ -246,3 +247,45 @@ documentation.build_all: paths: - docs/_build/ expose_as: Documentation - excluding old protocols + +documentation.publish: + image: ${ci_image_name}/test:${ci_image_tag} + stage: publish + tags: + - gcp_not_interruptible + needs: + - oc.docker:ci:amd64 + - documentation.build_all + dependencies: + - oc.docker:ci:amd64 + - documentation.build_all + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + interruptible: false + before_script: + - . ./scripts/ci/datadog_send_job_info.sh + - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' + - . ./scripts/ci/sccache-start.sh + script: + - eval $(opam env) + - . $HOME/.venv/bin/activate + - echo "${CI_PK_GITLAB_DOC}" > ~/.ssh/id_ed25519 + - echo "${CI_KH}" > ~/.ssh/known_hosts + - chmod 400 ~/.ssh/id_ed25519 + - ./scripts/ci/doc_publish.sh + after_script: + - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' + - ./scripts/ci/sccache-stop.sh + variables: + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + SCCACHE_CACHE_SIZE: 5G + retry: 0 diff --git a/.gitlab/ci/pipelines/schedule_documentation.yml b/.gitlab/ci/pipelines/schedule_documentation.yml deleted file mode 100644 index 32b1aa314a51..000000000000 --- a/.gitlab/ci/pipelines/schedule_documentation.yml +++ /dev/null @@ -1,209 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file ci/bin/main.ml instead. - -stages: -- start -- images -- build -- publish - -oc.docker:ci:amd64: - image: ${GCP_REGISTRY}/tezos/docker-images/ci-docker:v1.13.0 - stage: images - tags: - - gcp - dependencies: [] - timeout: 90 minutes - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - ./images/ci_create_ci_images.sh - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - services: - - docker:${DOCKER_VERSION}-dind - variables: - DOCKER_VERSION: 24.0.7 - CI_DOCKER_HUB: "false" - ARCH: amd64 - artifacts: - reports: - dotenv: ci_image_tag.env - -datadog_pipeline_trace: - image: datadog/ci:v2.44.0 - stage: start - tags: - - gcp - dependencies: [] - allow_failure: true - timeout: 60 minutes - interruptible: false - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: - - CI_MERGE_REQUEST_IID=${CI_MERGE_REQUEST_IID:-none} - - DATADOG_SITE=datadoghq.eu datadog-ci tag --level pipeline --tags pipeline_type:$PIPELINE_TYPE - --tags mr_number:$CI_MERGE_REQUEST_IID - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - -oc.build:static-x86_64-linux-binaries: - image: ${ci_image_name}/build:${ci_image_tag} - stage: build - tags: - - gcp_very_high_cpu_ramfs - rules: - - when: always - dependencies: - - oc.docker:ci:amd64 - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - - key: rust-targets-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_target - policy: pull-push - interruptible: false - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - ./scripts/ci/take_ownership.sh - - eval $(opam env) - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh - script: - - ./scripts/ci/build_static_binaries.sh - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh - variables: - ARCH: x86_64 - EXECUTABLE_FILES: script-inputs/octez-released-executables script-inputs/octez-experimental-executables - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 2G - OCTEZ_RUST_DEPS_TARGET_DIR: $CI_PROJECT_DIR/_target/rust_deps - OCTEZ_RUSTZCASH_DEPS_TARGET_DIR: $CI_PROJECT_DIR/_target/rustzcash_deps - OCTEZ_ETHERLINK_WASM_RUNTIME_TARGET_DIR: $CI_PROJECT_DIR/_target/etherlink_wasm_runtime - artifacts: - paths: - - octez-binaries/$ARCH/* - -documentation:odoc: - stage: start - tags: - - gcp - dependencies: [] - timeout: 60 minutes - interruptible: false - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: [] - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - artifacts: - paths: - - dummy - -documentation:manuals: - stage: start - tags: - - gcp - dependencies: [] - timeout: 60 minutes - interruptible: false - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: [] - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - artifacts: - paths: - - dummy - -documentation:docgen: - stage: start - tags: - - gcp - dependencies: [] - timeout: 60 minutes - interruptible: false - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: [] - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - artifacts: - paths: - - dummy - -documentation:build_all: - stage: start - tags: - - gcp - dependencies: [] - timeout: 60 minutes - interruptible: false - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - script: [] - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - artifacts: - paths: - - dummy - -documentation:publish: - image: ${ci_image_name}/test:${ci_image_tag} - stage: publish - tags: - - gcp_not_interruptible - needs: - - oc.docker:ci:amd64 - - documentation:build_all - dependencies: - - oc.docker:ci:amd64 - - documentation:build_all - timeout: 60 minutes - cache: - - key: cargo-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/.cargo/registry/cache - policy: pull-push - - key: sccache-$CI_JOB_NAME_SLUG - paths: - - $CI_PROJECT_DIR/_sccache - policy: pull-push - interruptible: false - before_script: - - . ./scripts/ci/datadog_send_job_info.sh - - eval $(opam env) - - . $HOME/.venv/bin/activate - - echo "${CI_PK_GITLAB_DOC}" > ~/.ssh/id_ed25519 - - echo "${CI_KH}" > ~/.ssh/known_hosts - - chmod 400 ~/.ssh/id_ed25519 - - . ./scripts/ci/datadog_send_job_cache_info.sh 'before' - - . ./scripts/ci/sccache-start.sh - script: - - ./scripts/ci/doc_publish.sh - after_script: - - . ./scripts/ci/datadog_send_job_cache_info.sh 'after' - - ./scripts/ci/sccache-stop.sh - variables: - CARGO_NET_OFFLINE: "false" - SCCACHE_DIR: $CI_PROJECT_DIR/_sccache - SCCACHE_CACHE_SIZE: 5G - retry: 0 diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 604add1bfc6e..f7664c0d7c6c 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -1223,51 +1223,3 @@ module Tezt = struct "tezt/records/*.json.broken"; ]) end - -module Documentation = struct - let mk_artifact_dependencies ?(dependencies = Dependent []) jobs = - List.fold_right - (fun job dependencies -> - dependencies_add_artifact_dependency dependencies job) - jobs - dependencies - - (** Create a [publish:documentation] job. - - This job, for inclusion on [master_branch] pipelines, publishes - built documentation received as artifact from a {!job_build_all} - to [tezos.gitlab.io]. *) - let job_publish_documentation ~job_build_all ?dependencies ?rules () : - tezos_job = - let dependencies = mk_artifact_dependencies ?dependencies [job_build_all] in - job - ~__POS__ - ~name:"documentation:publish" - ~image:Images.CI.test - ~stage:Stages.publish - ~dependencies - ~before_script: - (before_script - ~eval_opam:true - (* Load the environment poetry previously created in the docker image. - Give access to the Python dependencies/executables. *) - ~init_python_venv:true - (* [CI_PK_GITLAB_DOC] and [CI_KH] are set in the projects - GitLab setting and are only available on protected - refs. [CI_PK_GITLAB_DOC] contains the private key used - as deploy key for the Octez documentation on - [tezos.gitlab.io]. [CI_KH] contains the host key of - [gitlab.com], which is not really a secret (it's - public!). *) - [ - {|echo "${CI_PK_GITLAB_DOC}" > ~/.ssh/id_ed25519|}; - {|echo "${CI_KH}" > ~/.ssh/known_hosts|}; - {|chmod 400 ~/.ssh/id_ed25519|}; - ]) - ~interruptible:false - ?rules - ["./scripts/ci/doc_publish.sh"] - ~retry:Gitlab_ci.Types.{max = 0; when_ = []} - ~tag:Gcp_not_interruptible - |> enable_cargo_cache |> enable_sccache -end diff --git a/ci/bin/documentation.ml b/ci/bin/documentation.ml index b4d769c8edfb..b167f38c3094 100644 --- a/ci/bin/documentation.ml +++ b/ci/bin/documentation.ml @@ -263,6 +263,25 @@ let job_linkcheck = "make -C docs linkcheck"; ] +let job_publish = + CI.job + "publish" + ~__POS__ + ~image:Tezos_ci.Images.CI.test + ~stage:Publish + ~description:"Publish the documentation to octez.com/docs." + ~needs:[(Artifacts, job_build_all `full `static)] + ~cargo_cache:true + ~sccache:(Cacio.sccache ()) + [ + "eval $(opam env)"; + ". $HOME/.venv/bin/activate"; + {|echo "${CI_PK_GITLAB_DOC}" > ~/.ssh/id_ed25519|}; + {|echo "${CI_KH}" > ~/.ssh/known_hosts|}; + "chmod 400 ~/.ssh/id_ed25519"; + "./scripts/ci/doc_publish.sh"; + ] + let register () = CI.register_before_merging_jobs [ @@ -289,5 +308,5 @@ let register () = "Generate and push the documentation to octez.com/docs without being \ interrupted." ~legacy_jobs:[Master_branch.job_static_x86_64] - [(Auto, job_build_all `full `static)] ; + [(Auto, job_publish)] ; () diff --git a/ci/bin/main.ml b/ci/bin/main.ml index cd77e0bc50da..4ccac5fc5a72 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -474,16 +474,6 @@ let () = ~description: "Scheduled pipeline for various security scans. Currently scanning for \ vulnerabilities in Docker images" ; - register - "schedule_documentation" - schedule_documentation - ~jobs: - (Tezos_ci.job_datadog_pipeline_trace :: Master_branch.job_static_x86_64 - :: Master_branch.jobs_documentation - |> List.map (with_interruptible false)) - ~description: - "Scheduled pipeline that updates the octez.com/docs documentation \ - without being interrupted." ; register "schedule_docker_build_pipeline" schedule_docker_build diff --git a/ci/bin/master_branch.ml b/ci/bin/master_branch.ml index b22758fe3c7e..044337bff23d 100644 --- a/ci/bin/master_branch.ml +++ b/ci/bin/master_branch.ml @@ -21,10 +21,6 @@ open Gitlab_ci.Util open Tezos_ci open Tezos_ci.Cache -(* Without this, Dune's dependency analysis is wrong: - it thinks Documentation is the new module that uses Cacio. *) -module Documentation = Common.Documentation - let rules_always = [job_rule ~when_:Always ()] (* static binaries *) @@ -45,48 +41,6 @@ let job_static_x86_64 = ~rules:rules_always () -let jobs_documentation : tezos_job list = - let job_odoc = - (* Dummy job to remove in a commit later in this MR. *) - Tezos_ci.job - ~__POS__ - ~stage:Tezos_ci.Stages.start - ~name:"documentation:odoc" - ~artifacts:(Gitlab_ci.Util.artifacts ["dummy"]) - [] - in - let job_manuals = - (* Dummy job to remove in a commit later in this MR. *) - Tezos_ci.job - ~__POS__ - ~stage:Tezos_ci.Stages.start - ~name:"documentation:manuals" - ~artifacts:(Gitlab_ci.Util.artifacts ["dummy"]) - [] - in - let job_docgen = - (* Dummy job to remove in a commit later in this MR. *) - Tezos_ci.job - ~__POS__ - ~stage:Tezos_ci.Stages.start - ~name:"documentation:docgen" - ~artifacts:(Gitlab_ci.Util.artifacts ["dummy"]) - [] - in - let job_build_all = - (* Dummy job to remove in a commit later in this MR. *) - Tezos_ci.job - ~__POS__ - ~stage:Tezos_ci.Stages.start - ~name:"documentation:build_all" - ~artifacts:(Gitlab_ci.Util.artifacts ["dummy"]) - [] - in - let job_publish_documentation : tezos_job = - Documentation.job_publish_documentation ~job_build_all () - in - [job_odoc; job_manuals; job_docgen; job_build_all; job_publish_documentation] - (* Defines the jobs of the [schedule_docker_build_pipeline] pipeline. This pipeline runs scheduled on the [master] branch. The goal -- GitLab From 8010ac4e67fcc87408c94b66f09c835b8f688500 Mon Sep 17 00:00:00 2001 From: Romain Date: Mon, 1 Sep 2025 16:02:16 +0200 Subject: [PATCH 9/9] Cacio: immediate test => sanity --- .gitlab/ci/pipelines/before_merging.yml | 2 +- .gitlab/ci/pipelines/merge_train.yml | 2 +- ci/lib_cacio/cacio.ml | 17 +++++++++++------ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index dbb92e5650f3..45d0573a9a58 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -4320,7 +4320,7 @@ oc.script.docker_verify_image_amd64: documentation.rst-check: image: ${ci_image_name_protected}/test:amd64--master - stage: test + stage: sanity tags: - gcp rules: diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index e76f8040a6e6..10eeb1b83c51 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -4254,7 +4254,7 @@ oc.script.docker_verify_image_amd64: documentation.rst-check: image: ${ci_image_name_protected}/test:amd64--master - stage: test + stage: sanity tags: - gcp rules: diff --git a/ci/lib_cacio/cacio.ml b/ci/lib_cacio/cacio.ml index 597ca124c49d..ce98eadcc9c2 100644 --- a/ci/lib_cacio/cacio.ml +++ b/ci/lib_cacio/cacio.ml @@ -284,11 +284,16 @@ let fix_graph (graph : job_graph) : fixed_job_graph = graph ; !result -let convert_stage (stage : stage) : Tezos_ci.Stage.t = - match stage with - | Build -> Tezos_ci.Stages.build - | Test -> Tezos_ci.Stages.test - | Publish -> Tezos_ci.Stages.publish +let convert_stage (trigger : trigger) (stage : stage) : Tezos_ci.Stage.t = + match (stage, trigger) with + | Build, _ -> Tezos_ci.Stages.build + | Test, Immediate -> + (* In the future, we plan to remove the sanity stage. + Hence why we do not expose a [Sanity] constructor in type [stage]. + But for now, immediate test jobs are still supposed to be in the sanity stage. *) + Tezos_ci.Stages.sanity + | Test, _ -> Tezos_ci.Stages.test + | Publish, _ -> Tezos_ci.Stages.publish type tezos_job_graph = Tezos_ci.tezos_job UID_map.t @@ -414,7 +419,7 @@ let convert_graph ~with_condition (graph : fixed_job_graph) : tezos_job_graph = Tezos_ci.job ~__POS__:source_location ~name - ~stage:(convert_stage stage) + ~stage:(convert_stage trigger stage) ~description ~dev_infra ?arch -- GitLab