From 7b3599053207881c0b54e97d3cd4de688f7ea9f6 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 13 Sep 2024 13:11:22 +0200 Subject: [PATCH 1/6] CI: Move doc jobs to [Common] --- ci/bin/code_verification.ml | 100 +++-------------------------- ci/bin/common.ml | 124 ++++++++++++++++++++++++++++++++++++ ci/bin/master_branch.ml | 22 +------ 3 files changed, 135 insertions(+), 111 deletions(-) diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index ae018a330407..e1901971a185 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -1562,104 +1562,27 @@ let jobs pipeline_type = let rules = make_rules ~changes:changeset_octez_docs ~label:"ci--docs" () in - let job_odoc = - job - ~__POS__ - ~name:"documentation:odoc" - ~image:Images.CI.test - ~stage:Stages.doc - ~dependencies:dependencies_needs_start - ~rules - ~before_script:(before_script ~eval_opam:true []) - ~artifacts: - (artifacts - ~when_:Always - ~expire_in:(Duration (Hours 1)) - (* Path must be terminated with / to expose artifact (gitlab-org/gitlab#/36706) *) - ["docs/_build/api/odoc/"; "docs/odoc.log"]) - ["make -C docs odoc-lite"] - |> enable_cargo_cache |> enable_sccache - in - let job_manuals = - job - ~__POS__ - ~name:"documentation:manuals" - ~image:Images.CI.test - ~stage:Stages.doc - ~dependencies:dependencies_needs_start - ~rules - ~before_script:(before_script ~eval_opam:true []) - ~artifacts: - (artifacts - ~expire_in:(Duration (Weeks 1)) - [ - "docs/*/octez-*.html"; - "docs/api/octez-*.txt"; - "docs/developer/metrics.csv"; - "docs/user/node-config.json"; - ]) - ["./scripts/ci/documentation:manuals.sh"] - |> enable_cargo_cache |> enable_sccache - in - let job_docgen = - job - ~__POS__ - ~name:"documentation:docgen" - ~image:Images.CI.test - ~stage:Stages.doc - ~dependencies:dependencies_needs_start - ~rules - ~before_script:(before_script ~eval_opam:true []) - ~artifacts: - (artifacts - ~expire_in:(Duration (Weeks 1)) - [ - "docs/alpha/rpc.rst"; - "docs/shell/rpc.rst"; - "docs/user/default-acl.json"; - "docs/api/errors.rst"; - "docs/shell/p2p_api.rst"; - ]) - ["make -C docs -j docexes-gen"] - |> enable_cargo_cache |> enable_sccache - in + let dependencies = dependencies_needs_start in + let job_odoc = Documentation.job_odoc ~rules ~dependencies () in + let job_manuals = Documentation.job_manuals ~rules ~dependencies () in + let job_docgen = Documentation.job_docgen ~rules ~dependencies () in let doc_build_dependencies = Dependent [Artifacts job_odoc; Artifacts job_manuals; Artifacts job_docgen] in let job_build_all = - job - ~__POS__ - ~name:"documentation:build_all" - ~image:Images.CI.test - ~stage:Stages.doc + Documentation.job_build_all ~dependencies:doc_build_dependencies - (* Warning: the [documentation:linkcheck] job must have at least the same - restrictions in the rules as [documentation:build_all], otherwise the CI - may complain that [documentation:linkcheck] depends on [documentation:build_all] - which does not exist. *) ~rules: (make_rules ~dependent:true ~changes:changeset_octez_docs ~label:"ci--docs" ()) - ~before_script: - (before_script ~eval_opam:true ~init_python_venv:true []) - ~artifacts: - (artifacts - ~expose_as:"Documentation - excluding old protocols" - ~expire_in:(Duration (Weeks 1)) - (* Path must be terminated with / to expose artifact (gitlab-org/gitlab#/36706) *) - ["docs/_build/"]) - ["make -C docs -j sphinx"] + () in let job_documentation_linkcheck : tezos_job = - job - ~__POS__ - ~name:"documentation:linkcheck" - ~image:Images.CI.test - ~stage:Stages.doc + Documentation.job_linkcheck ~dependencies: (Dependent [ @@ -1673,14 +1596,7 @@ let jobs pipeline_type = ~label:"ci--docs" ~manual:(On_changes changeset_octez_docs) ()) - ~allow_failure:Yes - ~before_script: - (before_script - ~source_version:true - ~eval_opam:true - ~init_python_venv:true - []) - ["make -C docs redirectcheck"; "make -C docs linkcheck"] + () in [ job_odoc; diff --git a/ci/bin/common.ml b/ci/bin/common.ml index c92719214916..eb8402506945 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -1363,3 +1363,127 @@ module Tezt = struct "tezt/records/*.json.broken"; ]) end + +module Documentation = struct + let job_odoc ?rules ?dependencies () : tezos_job = + job + ~__POS__ + ~name:"documentation:odoc" + ~image:Images.CI.test + ~stage:Stages.doc + ?dependencies + ?rules + ~before_script:(before_script ~eval_opam:true []) + ~artifacts: + (artifacts + ~when_:Always + ~expire_in:(Duration (Hours 1)) + (* Path must be terminated with / to expose artifact (gitlab-org/gitlab#/36706) *) + ["docs/_build/api/odoc/"; "docs/odoc.log"]) + ["make -C docs odoc-lite"] + |> enable_cargo_cache |> enable_sccache + + let job_manuals ?rules ?dependencies () : tezos_job = + job + ~__POS__ + ~name:"documentation:manuals" + ~image:Images.CI.test + ~stage:Stages.doc + ?dependencies + ?rules + ~before_script:(before_script ~eval_opam:true []) + ~artifacts: + (artifacts + ~expire_in:(Duration (Weeks 1)) + [ + "docs/*/octez-*.html"; + "docs/api/octez-*.txt"; + "docs/developer/metrics.csv"; + "docs/user/node-config.json"; + ]) + ["./scripts/ci/documentation:manuals.sh"] + |> enable_cargo_cache |> enable_sccache + + let job_docgen ?rules ?dependencies () : tezos_job = + job + ~__POS__ + ~name:"documentation:docgen" + ~image:Images.CI.test + ~stage:Stages.doc + ?dependencies + ?rules + ~before_script:(before_script ~eval_opam:true []) + ~artifacts: + (artifacts + ~expire_in:(Duration (Weeks 1)) + [ + "docs/alpha/rpc.rst"; + "docs/shell/rpc.rst"; + "docs/user/default-acl.json"; + "docs/api/errors.rst"; + "docs/shell/p2p_api.rst"; + ]) + ["make -C docs -j docexes-gen"] + |> enable_cargo_cache |> enable_sccache + + let job_build_all ?rules ?dependencies () : tezos_job = + job + ~__POS__ + ~name:"documentation:build_all" + ~image:Images.CI.test + ~stage:Stages.doc + ?dependencies + (* Warning: the [documentation:linkcheck] job must have at least the same + restrictions in the rules as [documentation:build_all], otherwise the CI + may complain that [documentation:linkcheck] depends on [documentation:build_all] + which does not exist. *) + ?rules + ~before_script:(before_script ~eval_opam:true ~init_python_venv:true []) + ~artifacts: + (artifacts + ~expose_as:"Documentation - excluding old protocols" + ~expire_in:(Duration (Weeks 1)) + (* Path must be terminated with / to expose artifact (gitlab-org/gitlab#/36706) *) + ["docs/_build/"]) + ["make -C docs -j sphinx"] + + let job_linkcheck ?dependencies ?rules () : tezos_job = + job + ~__POS__ + ~name:"documentation:linkcheck" + ~image:Images.CI.test + ~stage:Stages.doc + ?dependencies + ?rules + ~allow_failure:Yes + ~before_script: + (before_script + ~source_version:true + ~eval_opam:true + ~init_python_venv:true + []) + ["make -C docs redirectcheck"; "make -C docs linkcheck"] + + let job_publish_documentation ?dependencies ?rules () : tezos_job = + job + ~__POS__ + ~name:"publish:documentation" + ~image:Images.CI.test + ~stage:Stages.doc + ?dependencies + ~before_script: + (before_script + ~eval_opam:true + (* Load the environment poetry previously created in the docker image. + Give access to the Python dependencies/executables. *) + ~init_python_venv:true + [ + {|echo "${CI_PK_GITLAB_DOC}" > ~/.ssh/id_ed25519|}; + {|echo "${CI_KH}" > ~/.ssh/known_hosts|}; + {|chmod 400 ~/.ssh/id_ed25519|}; + ]) + ~interruptible:false + ?rules + ["./scripts/ci/doc_publish.sh"] + |> enable_cargo_cache |> enable_sccache +end diff --git a/ci/bin/master_branch.ml b/ci/bin/master_branch.ml index e6954b330a2b..3eb74921839c 100644 --- a/ci/bin/master_branch.ml +++ b/ci/bin/master_branch.ml @@ -86,24 +86,8 @@ let jobs = |> enable_coverage_location |> enable_coverage_report in let job_publish_documentation : tezos_job = - job - ~__POS__ - ~name:"publish:documentation" - ~image:Images.CI.test - ~stage:Stages.doc + Documentation.job_publish_documentation ~dependencies:(Dependent []) - ~before_script: - (before_script - ~eval_opam:true - (* Load the environment poetry previously created in the docker image. - Give access to the Python dependencies/executables. *) - ~init_python_venv:true - [ - {|echo "${CI_PK_GITLAB_DOC}" > ~/.ssh/id_ed25519|}; - {|echo "${CI_KH}" > ~/.ssh/known_hosts|}; - {|chmod 400 ~/.ssh/id_ed25519|}; - ]) - ~interruptible:false (* We publish documentation [Always] -- the job has no dependencies and so [On_success] is actually equivalent to [Always], but the latter is more explicit. *) @@ -114,7 +98,7 @@ let jobs = ~when_:Always (); ] - ["./scripts/ci/doc_publish.sh"] + () in (* Smart Rollup: Kernel SDK @@ -153,7 +137,7 @@ let jobs = (* Stage: test_coverage *) job_unified_coverage_default; (* Stage: doc *) - job_publish_documentation |> enable_cargo_cache |> enable_sccache; + job_publish_documentation; (* Stage: prepare_release *) job_docker_merge_manifests; (* Stage: manual *) -- GitLab From 5eaac5595dab32fca0def8a71e647821db75ce02 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 13 Sep 2024 13:19:55 +0200 Subject: [PATCH 2/6] CI: split [publish:documentation] in [master_branch] --- .gitlab/ci/pipelines/master_branch.yml | 218 ++++++++++++++++++++++++- ci/bin/code_verification.ml | 4 +- ci/bin/common.ml | 5 +- ci/bin/master_branch.ml | 52 +++--- 4 files changed, 255 insertions(+), 24 deletions(-) diff --git a/.gitlab/ci/pipelines/master_branch.yml b/.gitlab/ci/pipelines/master_branch.yml index 110ac4fc399d..c00bbaf563aa 100644 --- a/.gitlab/ci/pipelines/master_branch.yml +++ b/.gitlab/ci/pipelines/master_branch.yml @@ -322,7 +322,7 @@ oc.unified_coverage: expose_as: Coverage report coverage: '/Coverage: ([^%]+%)/' -publish:documentation: +documentation:odoc: image: ${ci_image_name}/test:${ci_image_tag} stage: doc tags: @@ -345,11 +345,227 @@ publish:documentation: - src/**/* - tezt/**/* - vendors/**/* + when: on_success + needs: + - oc.docker:ci:amd64 + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + before_script: + - eval $(opam env) + - . ./scripts/ci/sccache-start.sh + script: + - make -C docs odoc + after_script: + - ./scripts/ci/sccache-stop.sh + variables: + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + artifacts: + expire_in: 1 hour + paths: + - docs/_build/api/odoc/ + - docs/odoc.log when: always + +documentation:manuals: + image: ${ci_image_name}/test:${ci_image_tag} + stage: doc + tags: + - gcp + rules: + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - brassaia/**/* + - client-libs/**/* + - data-encoding/**/* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/* + - irmin/**/* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/* + - tezt/**/* + - vendors/**/* + when: on_success + needs: + - oc.docker:ci:amd64 + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + before_script: + - eval $(opam env) + - . ./scripts/ci/sccache-start.sh + script: + - ./scripts/ci/documentation:manuals.sh + after_script: + - ./scripts/ci/sccache-stop.sh + variables: + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + artifacts: + expire_in: 1 week + paths: + - docs/*/octez-*.html + - docs/api/octez-*.txt + - docs/developer/metrics.csv + - docs/user/node-config.json + +documentation:docgen: + image: ${ci_image_name}/test:${ci_image_tag} + stage: doc + tags: + - gcp + rules: + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - brassaia/**/* + - client-libs/**/* + - data-encoding/**/* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/* + - irmin/**/* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/* + - tezt/**/* + - vendors/**/* + when: on_success + needs: + - oc.docker:ci:amd64 + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + before_script: + - eval $(opam env) + - . ./scripts/ci/sccache-start.sh + script: + - make -C docs -j docexes-gen + after_script: + - ./scripts/ci/sccache-stop.sh + variables: + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + artifacts: + expire_in: 1 week + paths: + - docs/alpha/rpc.rst + - docs/shell/rpc.rst + - docs/user/default-acl.json + - docs/api/errors.rst + - docs/shell/p2p_api.rst + +documentation:build_all: + image: ${ci_image_name}/test:${ci_image_tag} + stage: doc + tags: + - gcp + rules: + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - brassaia/**/* + - client-libs/**/* + - data-encoding/**/* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/* + - irmin/**/* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/* + - tezt/**/* + - vendors/**/* + when: on_success + needs: + - oc.docker:ci:amd64 + - documentation:odoc + - documentation:manuals + - documentation:docgen + dependencies: + - oc.docker:ci:amd64 + - documentation:odoc + - documentation:manuals + - documentation:docgen + timeout: 60 minutes + before_script: + - eval $(opam env) + - . $HOME/.venv/bin/activate + script: + - make -C docs -j sphinx + artifacts: + expire_in: 1 week + paths: + - docs/_build/ + expose_as: Documentation - excluding old protocols + +publish:documentation: + image: ${ci_image_name}/test:${ci_image_tag} + stage: doc + tags: + - gcp + rules: + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - brassaia/**/* + - client-libs/**/* + - data-encoding/**/* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/* + - irmin/**/* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/* + - tezt/**/* + - vendors/**/* + when: on_success needs: - oc.docker:ci:amd64 + - documentation:build_all dependencies: - oc.docker:ci:amd64 + - documentation:build_all timeout: 60 minutes cache: - key: cargo-$CI_JOB_NAME_SLUG diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index e1901971a185..8f8daf19f25d 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -1563,7 +1563,9 @@ let jobs pipeline_type = make_rules ~changes:changeset_octez_docs ~label:"ci--docs" () in let dependencies = dependencies_needs_start in - let job_odoc = Documentation.job_odoc ~rules ~dependencies () in + let job_odoc = + Documentation.job_odoc ~rules ~dependencies ~lite:true () + in let job_manuals = Documentation.job_manuals ~rules ~dependencies () in let job_docgen = Documentation.job_docgen ~rules ~dependencies () in let doc_build_dependencies = diff --git a/ci/bin/common.ml b/ci/bin/common.ml index eb8402506945..186a230960ce 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -1365,7 +1365,8 @@ module Tezt = struct end module Documentation = struct - let job_odoc ?rules ?dependencies () : tezos_job = + let job_odoc ?(lite = false) ?rules ?dependencies () : tezos_job = + let target = if lite then "odoc-lite" else "odoc" in job ~__POS__ ~name:"documentation:odoc" @@ -1380,7 +1381,7 @@ module Documentation = struct ~expire_in:(Duration (Hours 1)) (* Path must be terminated with / to expose artifact (gitlab-org/gitlab#/36706) *) ["docs/_build/api/odoc/"; "docs/odoc.log"]) - ["make -C docs odoc-lite"] + ["make -C docs " ^ target] |> enable_cargo_cache |> enable_sccache let job_manuals ?rules ?dependencies () : tezos_job = diff --git a/ci/bin/master_branch.ml b/ci/bin/master_branch.ml index 3eb74921839c..4ccdb8c91998 100644 --- a/ci/bin/master_branch.ml +++ b/ci/bin/master_branch.ml @@ -85,20 +85,30 @@ let jobs = ] |> enable_coverage_location |> enable_coverage_report in - let job_publish_documentation : tezos_job = - Documentation.job_publish_documentation - ~dependencies:(Dependent []) - (* We publish documentation [Always] -- the job has no - dependencies and so [On_success] is actually equivalent to - [Always], but the latter is more explicit. *) - ~rules: - [ - job_rule - ~changes:(Changeset.encode changeset_octez_docs) - ~when_:Always - (); - ] - () + let jobs_documentation : tezos_job list = + let rules = + [job_rule ~changes:(Changeset.encode changeset_octez_docs) ()] + in + let dependencies = Dependent [] in + let job_odoc = Documentation.job_odoc ~rules ~dependencies () in + let job_manuals = Documentation.job_manuals ~rules ~dependencies () in + let job_docgen = Documentation.job_docgen ~rules ~dependencies () in + let doc_build_dependencies = + Dependent + [Artifacts job_odoc; Artifacts job_manuals; Artifacts job_docgen] + in + let job_build_all = + Documentation.job_build_all ~dependencies:doc_build_dependencies ~rules () + in + let job_publish_documentation : tezos_job = + Documentation.job_publish_documentation + ~dependencies:(Dependent [Artifacts job_build_all]) + ~rules + () + in + [ + job_odoc; job_manuals; job_docgen; job_build_all; job_publish_documentation; + ] in (* Smart Rollup: Kernel SDK @@ -136,10 +146,12 @@ let jobs = job_docker_arm64_experimental; (* Stage: test_coverage *) job_unified_coverage_default; - (* Stage: doc *) - job_publish_documentation; - (* Stage: prepare_release *) - job_docker_merge_manifests; - (* Stage: manual *) - job_publish_kernel_sdk; ] + (* Stage: doc *) + @ jobs_documentation + @ [ + (* Stage: prepare_release *) + job_docker_merge_manifests; + (* Stage: manual *) + job_publish_kernel_sdk; + ] -- GitLab From 2ca522fa5084c2e7fef51b4961b8e3ac32ccd37a Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 13 Sep 2024 13:21:00 +0200 Subject: [PATCH 3/6] scripts/ci: no need to build in [doc_publish.sh] --- scripts/ci/doc_publish.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/ci/doc_publish.sh b/scripts/ci/doc_publish.sh index 19093526b07b..d5e56f19ad51 100755 --- a/scripts/ci/doc_publish.sh +++ b/scripts/ci/doc_publish.sh @@ -4,9 +4,6 @@ set -e cd "${CI_PROJECT_DIR}" || exit 1 -make all -make -C docs -j full - if [ "${CI_COMMIT_REF_NAME}" == "master" ]; then git clone --depth 5 git@gitlab.com:"${CI_PROJECT_NAMESPACE}"/"${CI_PROJECT_NAMESPACE}".gitlab.io gitlab.io rsync --recursive --links --perms --delete --exclude=.doctrees --exclude={{main,alpha,zero}net,master}/index.html docs/_build/ gitlab.io/public/ -- GitLab From a4de82023f2d1eebd4e4f1cc5f0a83c1f4dbdc41 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 13 Sep 2024 15:26:02 +0200 Subject: [PATCH 4/6] CI/docs: make dependencies more explicit --- ci/bin/code_verification.ml | 8 +++----- ci/bin/common.ml | 21 +++++++++++++++++---- ci/bin/master_branch.ml | 11 ++--------- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 8f8daf19f25d..a393d336144e 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -1568,13 +1568,11 @@ let jobs pipeline_type = in let job_manuals = Documentation.job_manuals ~rules ~dependencies () in let job_docgen = Documentation.job_docgen ~rules ~dependencies () in - let doc_build_dependencies = - Dependent - [Artifacts job_odoc; Artifacts job_manuals; Artifacts job_docgen] - in let job_build_all = Documentation.job_build_all - ~dependencies:doc_build_dependencies + ~job_odoc + ~job_manuals + ~job_docgen ~rules: (make_rules ~dependent:true diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 186a230960ce..5ce6fe83c084 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -1365,6 +1365,13 @@ module Tezt = struct end module Documentation = struct + let mk_artifact_dependencies ?(dependencies = Dependent []) jobs = + List.fold_right + (fun job dependencies -> + dependencies_add_artifact_dependency dependencies job) + jobs + dependencies + let job_odoc ?(lite = false) ?rules ?dependencies () : tezos_job = let target = if lite then "odoc-lite" else "odoc" in job @@ -1427,13 +1434,17 @@ module Documentation = struct ["make -C docs -j docexes-gen"] |> enable_cargo_cache |> enable_sccache - let job_build_all ?rules ?dependencies () : tezos_job = + let job_build_all ~job_odoc ~job_manuals ~job_docgen ?dependencies ?rules () : + tezos_job = + let dependencies = + mk_artifact_dependencies ?dependencies [job_odoc; job_manuals; job_docgen] + in job ~__POS__ ~name:"documentation:build_all" ~image:Images.CI.test ~stage:Stages.doc - ?dependencies + ~dependencies (* Warning: the [documentation:linkcheck] job must have at least the same restrictions in the rules as [documentation:build_all], otherwise the CI may complain that [documentation:linkcheck] depends on [documentation:build_all] @@ -1465,13 +1476,15 @@ module Documentation = struct []) ["make -C docs redirectcheck"; "make -C docs linkcheck"] - let job_publish_documentation ?dependencies ?rules () : tezos_job = + let job_publish_documentation ~job_build_all ?dependencies ?rules () : + tezos_job = + let dependencies = mk_artifact_dependencies ?dependencies [job_build_all] in job ~__POS__ ~name:"publish:documentation" ~image:Images.CI.test ~stage:Stages.doc - ?dependencies + ~dependencies ~before_script: (before_script ~eval_opam:true diff --git a/ci/bin/master_branch.ml b/ci/bin/master_branch.ml index 4ccdb8c91998..4eed35425cd4 100644 --- a/ci/bin/master_branch.ml +++ b/ci/bin/master_branch.ml @@ -93,18 +93,11 @@ let jobs = let job_odoc = Documentation.job_odoc ~rules ~dependencies () in let job_manuals = Documentation.job_manuals ~rules ~dependencies () in let job_docgen = Documentation.job_docgen ~rules ~dependencies () in - let doc_build_dependencies = - Dependent - [Artifacts job_odoc; Artifacts job_manuals; Artifacts job_docgen] - in let job_build_all = - Documentation.job_build_all ~dependencies:doc_build_dependencies ~rules () + Documentation.job_build_all ~job_odoc ~job_manuals ~job_docgen ~rules () in let job_publish_documentation : tezos_job = - Documentation.job_publish_documentation - ~dependencies:(Dependent [Artifacts job_build_all]) - ~rules - () + Documentation.job_publish_documentation ~job_build_all ~rules () in [ job_odoc; job_manuals; job_docgen; job_build_all; job_publish_documentation; -- GitLab From 754c11090c0f7984ef5dd243600441bc61d1a048 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 13 Sep 2024 15:43:01 +0200 Subject: [PATCH 5/6] CI: document the documentation jobs --- ci/bin/common.ml | 47 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 5ce6fe83c084..0064acc3fc86 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -1372,6 +1372,13 @@ module Documentation = struct jobs dependencies + (** Create an odoc job. + + The job will build the target [odoc] (resp. [odoc-lite]) in the + directory [docs] if lite is [false] (resp. [true]), which is the + default. + + This job is one of the prerequisites to {!job_build_all}. *) let job_odoc ?(lite = false) ?rules ?dependencies () : tezos_job = let target = if lite then "odoc-lite" else "odoc" in job @@ -1391,6 +1398,12 @@ module Documentation = struct ["make -C docs " ^ target] |> enable_cargo_cache |> enable_sccache + (** Create the manuals job. + + This job builds the command-line interface manuals (i.e [man]) + of octez binaries, for inclusion in the documentation. + + This job is one of the prerequisites to {!job_build_all}. *) let job_manuals ?rules ?dependencies () : tezos_job = job ~__POS__ @@ -1412,6 +1425,13 @@ module Documentation = struct ["./scripts/ci/documentation:manuals.sh"] |> enable_cargo_cache |> enable_sccache + (** Create the docgen job. + + This job builds various generated reference material, for + inclusion in the documentation. This includes the RPC, P2p and + error reference. + + This job is one of the prerequisites to {!job_build_all}. *) let job_docgen ?rules ?dependencies () : tezos_job = job ~__POS__ @@ -1434,6 +1454,12 @@ module Documentation = struct ["make -C docs -j docexes-gen"] |> enable_cargo_cache |> enable_sccache + (** Create the [documentation:build_all] job. + + This jobs builds the RST sources in docs, which will include the + the material from the prerequisite jobs: {!job_odoc}, + {!job_manuals}, {!job_docgen}, on whose artifacts the + [documentation:build_all] job will depend. *) let job_build_all ~job_odoc ~job_manuals ~job_docgen ?dependencies ?rules () : tezos_job = let dependencies = @@ -1445,10 +1471,6 @@ module Documentation = struct ~image:Images.CI.test ~stage:Stages.doc ~dependencies - (* Warning: the [documentation:linkcheck] job must have at least the same - restrictions in the rules as [documentation:build_all], otherwise the CI - may complain that [documentation:linkcheck] depends on [documentation:build_all] - which does not exist. *) ?rules ~before_script:(before_script ~eval_opam:true ~init_python_venv:true []) ~artifacts: @@ -1459,6 +1481,7 @@ module Documentation = struct ["docs/_build/"]) ["make -C docs -j sphinx"] + (** Create a [documentation:linkcheck] job. *) let job_linkcheck ?dependencies ?rules () : tezos_job = job ~__POS__ @@ -1466,6 +1489,10 @@ module Documentation = struct ~image:Images.CI.test ~stage:Stages.doc ?dependencies + (* Warning: the [documentation:linkcheck] job must have at least the same + restrictions in the rules as [documentation:build_all], otherwise the CI + may complain that [documentation:linkcheck] depends on [documentation:build_all] + which does not exist. *) ?rules ~allow_failure:Yes ~before_script: @@ -1476,6 +1503,11 @@ module Documentation = struct []) ["make -C docs redirectcheck"; "make -C docs linkcheck"] + (** Create a [publish:documentation] job. + + This job, for inclusion on [master_branch] pipelines, publishes + built documentation received as artifact from a {!job_build_all} + to [tezos.gitlab.io]. *) let job_publish_documentation ~job_build_all ?dependencies ?rules () : tezos_job = let dependencies = mk_artifact_dependencies ?dependencies [job_build_all] in @@ -1491,6 +1523,13 @@ module Documentation = struct (* Load the environment poetry previously created in the docker image. Give access to the Python dependencies/executables. *) ~init_python_venv:true + (* [CI_PK_GITLAB_DOC] and [CI_KH] are set in the projects + GitLab setting and are only available on protected + refs. [CI_PK_GITLAB_DOC] contains the private key used + as deploy key for the Octez documentation on + [tezos.gitlab.io]. [CI_KH] contains the host key of + [gitlab.com], which is not really a secret (it's + public!). *) [ {|echo "${CI_PK_GITLAB_DOC}" > ~/.ssh/id_ed25519|}; {|echo "${CI_KH}" > ~/.ssh/known_hosts|}; -- GitLab From 4bb38ca824dfff6e338b0a9cdfa2762a896598ae Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 13 Sep 2024 15:44:18 +0200 Subject: [PATCH 6/6] CI: [linkcheck] depends on less stuff --- .gitlab/ci/pipelines/before_merging.yml | 4 ---- .gitlab/ci/pipelines/merge_train.yml | 4 ---- .gitlab/ci/pipelines/schedule_extended_test.yml | 4 ---- ci/bin/code_verification.ml | 8 +------- ci/bin/common.ml | 5 +++-- 5 files changed, 4 insertions(+), 21 deletions(-) diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index fbc10435693c..93b57e6619c5 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -4206,13 +4206,9 @@ documentation:linkcheck: allow_failure: true needs: - oc.docker:ci:amd64 - - documentation:manuals - - documentation:docgen - documentation:build_all dependencies: - oc.docker:ci:amd64 - - documentation:manuals - - documentation:docgen - documentation:build_all allow_failure: true timeout: 60 minutes diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index fbc10435693c..93b57e6619c5 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -4206,13 +4206,9 @@ documentation:linkcheck: allow_failure: true needs: - oc.docker:ci:amd64 - - documentation:manuals - - documentation:docgen - documentation:build_all dependencies: - oc.docker:ci:amd64 - - documentation:manuals - - documentation:docgen - documentation:build_all allow_failure: true timeout: 60 minutes diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 27dd5638faec..c791da0310ff 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -2770,13 +2770,9 @@ documentation:linkcheck: - when: on_success needs: - oc.docker:ci:amd64 - - documentation:manuals - - documentation:docgen - documentation:build_all dependencies: - oc.docker:ci:amd64 - - documentation:manuals - - documentation:docgen - documentation:build_all allow_failure: true timeout: 60 minutes diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index a393d336144e..14cf1d12cf51 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -1583,13 +1583,7 @@ let jobs pipeline_type = in let job_documentation_linkcheck : tezos_job = Documentation.job_linkcheck - ~dependencies: - (Dependent - [ - Artifacts job_manuals; - Artifacts job_docgen; - Artifacts job_build_all; - ]) + ~job_build_all ~rules: (make_rules ~dependent:true diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 0064acc3fc86..4591851b9ddd 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -1482,13 +1482,14 @@ module Documentation = struct ["make -C docs -j sphinx"] (** Create a [documentation:linkcheck] job. *) - let job_linkcheck ?dependencies ?rules () : tezos_job = + let job_linkcheck ~job_build_all ?dependencies ?rules () : tezos_job = + let dependencies = mk_artifact_dependencies ?dependencies [job_build_all] in job ~__POS__ ~name:"documentation:linkcheck" ~image:Images.CI.test ~stage:Stages.doc - ?dependencies + ~dependencies (* Warning: the [documentation:linkcheck] job must have at least the same restrictions in the rules as [documentation:build_all], otherwise the CI may complain that [documentation:linkcheck] depends on [documentation:build_all] -- GitLab