diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index fbc10435693ca25d077f8ba348d4a9d53c3773ed..93b57e6619c521ff70b3c3f4cc34d50221ca2ecd 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -4206,13 +4206,9 @@ documentation:linkcheck: allow_failure: true needs: - oc.docker:ci:amd64 - - documentation:manuals - - documentation:docgen - documentation:build_all dependencies: - oc.docker:ci:amd64 - - documentation:manuals - - documentation:docgen - documentation:build_all allow_failure: true timeout: 60 minutes diff --git a/.gitlab/ci/pipelines/master_branch.yml b/.gitlab/ci/pipelines/master_branch.yml index 110ac4fc399d9e901d55e753c1093624c6aa9008..c00bbaf563aaa47f66bf6e1fd01fcad4b89d603f 100644 --- a/.gitlab/ci/pipelines/master_branch.yml +++ b/.gitlab/ci/pipelines/master_branch.yml @@ -322,7 +322,7 @@ oc.unified_coverage: expose_as: Coverage report coverage: '/Coverage: ([^%]+%)/' -publish:documentation: +documentation:odoc: image: ${ci_image_name}/test:${ci_image_tag} stage: doc tags: @@ -345,11 +345,227 @@ publish:documentation: - src/**/* - tezt/**/* - vendors/**/* + when: on_success + needs: + - oc.docker:ci:amd64 + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + before_script: + - eval $(opam env) + - . ./scripts/ci/sccache-start.sh + script: + - make -C docs odoc + after_script: + - ./scripts/ci/sccache-stop.sh + variables: + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + artifacts: + expire_in: 1 hour + paths: + - docs/_build/api/odoc/ + - docs/odoc.log when: always + +documentation:manuals: + image: ${ci_image_name}/test:${ci_image_tag} + stage: doc + tags: + - gcp + rules: + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - brassaia/**/* + - client-libs/**/* + - data-encoding/**/* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/* + - irmin/**/* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/* + - tezt/**/* + - vendors/**/* + when: on_success + needs: + - oc.docker:ci:amd64 + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + before_script: + - eval $(opam env) + - . ./scripts/ci/sccache-start.sh + script: + - ./scripts/ci/documentation:manuals.sh + after_script: + - ./scripts/ci/sccache-stop.sh + variables: + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + artifacts: + expire_in: 1 week + paths: + - docs/*/octez-*.html + - docs/api/octez-*.txt + - docs/developer/metrics.csv + - docs/user/node-config.json + +documentation:docgen: + image: ${ci_image_name}/test:${ci_image_tag} + stage: doc + tags: + - gcp + rules: + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - brassaia/**/* + - client-libs/**/* + - data-encoding/**/* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/* + - irmin/**/* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/* + - tezt/**/* + - vendors/**/* + when: on_success + needs: + - oc.docker:ci:amd64 + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + cache: + - key: cargo-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/.cargo/registry/cache + policy: pull-push + - key: sccache-$CI_JOB_NAME_SLUG + paths: + - $CI_PROJECT_DIR/_sccache + policy: pull-push + before_script: + - eval $(opam env) + - . ./scripts/ci/sccache-start.sh + script: + - make -C docs -j docexes-gen + after_script: + - ./scripts/ci/sccache-stop.sh + variables: + CARGO_NET_OFFLINE: "false" + SCCACHE_DIR: $CI_PROJECT_DIR/_sccache + artifacts: + expire_in: 1 week + paths: + - docs/alpha/rpc.rst + - docs/shell/rpc.rst + - docs/user/default-acl.json + - docs/api/errors.rst + - docs/shell/p2p_api.rst + +documentation:build_all: + image: ${ci_image_name}/test:${ci_image_tag} + stage: doc + tags: + - gcp + rules: + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - brassaia/**/* + - client-libs/**/* + - data-encoding/**/* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/* + - irmin/**/* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/* + - tezt/**/* + - vendors/**/* + when: on_success + needs: + - oc.docker:ci:amd64 + - documentation:odoc + - documentation:manuals + - documentation:docgen + dependencies: + - oc.docker:ci:amd64 + - documentation:odoc + - documentation:manuals + - documentation:docgen + timeout: 60 minutes + before_script: + - eval $(opam env) + - . $HOME/.venv/bin/activate + script: + - make -C docs -j sphinx + artifacts: + expire_in: 1 week + paths: + - docs/_build/ + expose_as: Documentation - excluding old protocols + +publish:documentation: + image: ${ci_image_name}/test:${ci_image_tag} + stage: doc + tags: + - gcp + rules: + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - brassaia/**/* + - client-libs/**/* + - data-encoding/**/* + - docs/**/* + - dune + - dune-project + - dune-workspace + - etherlink/**/* + - irmin/**/* + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/* + - tezt/**/* + - vendors/**/* + when: on_success needs: - oc.docker:ci:amd64 + - documentation:build_all dependencies: - oc.docker:ci:amd64 + - documentation:build_all timeout: 60 minutes cache: - key: cargo-$CI_JOB_NAME_SLUG diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index fbc10435693ca25d077f8ba348d4a9d53c3773ed..93b57e6619c521ff70b3c3f4cc34d50221ca2ecd 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -4206,13 +4206,9 @@ documentation:linkcheck: allow_failure: true needs: - oc.docker:ci:amd64 - - documentation:manuals - - documentation:docgen - documentation:build_all dependencies: - oc.docker:ci:amd64 - - documentation:manuals - - documentation:docgen - documentation:build_all allow_failure: true timeout: 60 minutes diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 27dd5638faec26aa4f8c2599e6b6aff7f86cc5f9..c791da0310fff932491a9bfd7ec8c4d39b293f38 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -2770,13 +2770,9 @@ documentation:linkcheck: - when: on_success needs: - oc.docker:ci:amd64 - - documentation:manuals - - documentation:docgen - documentation:build_all dependencies: - oc.docker:ci:amd64 - - documentation:manuals - - documentation:docgen - documentation:build_all allow_failure: true timeout: 60 minutes diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index ae018a330407945339ed0cdc622375e5a9669cf0..14cf1d12cf511c95a010c16d47ff569b3758648b 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -1562,125 +1562,35 @@ let jobs pipeline_type = let rules = make_rules ~changes:changeset_octez_docs ~label:"ci--docs" () in + let dependencies = dependencies_needs_start in let job_odoc = - job - ~__POS__ - ~name:"documentation:odoc" - ~image:Images.CI.test - ~stage:Stages.doc - ~dependencies:dependencies_needs_start - ~rules - ~before_script:(before_script ~eval_opam:true []) - ~artifacts: - (artifacts - ~when_:Always - ~expire_in:(Duration (Hours 1)) - (* Path must be terminated with / to expose artifact (gitlab-org/gitlab#/36706) *) - ["docs/_build/api/odoc/"; "docs/odoc.log"]) - ["make -C docs odoc-lite"] - |> enable_cargo_cache |> enable_sccache - in - let job_manuals = - job - ~__POS__ - ~name:"documentation:manuals" - ~image:Images.CI.test - ~stage:Stages.doc - ~dependencies:dependencies_needs_start - ~rules - ~before_script:(before_script ~eval_opam:true []) - ~artifacts: - (artifacts - ~expire_in:(Duration (Weeks 1)) - [ - "docs/*/octez-*.html"; - "docs/api/octez-*.txt"; - "docs/developer/metrics.csv"; - "docs/user/node-config.json"; - ]) - ["./scripts/ci/documentation:manuals.sh"] - |> enable_cargo_cache |> enable_sccache - in - let job_docgen = - job - ~__POS__ - ~name:"documentation:docgen" - ~image:Images.CI.test - ~stage:Stages.doc - ~dependencies:dependencies_needs_start - ~rules - ~before_script:(before_script ~eval_opam:true []) - ~artifacts: - (artifacts - ~expire_in:(Duration (Weeks 1)) - [ - "docs/alpha/rpc.rst"; - "docs/shell/rpc.rst"; - "docs/user/default-acl.json"; - "docs/api/errors.rst"; - "docs/shell/p2p_api.rst"; - ]) - ["make -C docs -j docexes-gen"] - |> enable_cargo_cache |> enable_sccache - in - let doc_build_dependencies = - Dependent - [Artifacts job_odoc; Artifacts job_manuals; Artifacts job_docgen] + Documentation.job_odoc ~rules ~dependencies ~lite:true () in + let job_manuals = Documentation.job_manuals ~rules ~dependencies () in + let job_docgen = Documentation.job_docgen ~rules ~dependencies () in let job_build_all = - job - ~__POS__ - ~name:"documentation:build_all" - ~image:Images.CI.test - ~stage:Stages.doc - ~dependencies:doc_build_dependencies - (* Warning: the [documentation:linkcheck] job must have at least the same - restrictions in the rules as [documentation:build_all], otherwise the CI - may complain that [documentation:linkcheck] depends on [documentation:build_all] - which does not exist. *) + Documentation.job_build_all + ~job_odoc + ~job_manuals + ~job_docgen ~rules: (make_rules ~dependent:true ~changes:changeset_octez_docs ~label:"ci--docs" ()) - ~before_script: - (before_script ~eval_opam:true ~init_python_venv:true []) - ~artifacts: - (artifacts - ~expose_as:"Documentation - excluding old protocols" - ~expire_in:(Duration (Weeks 1)) - (* Path must be terminated with / to expose artifact (gitlab-org/gitlab#/36706) *) - ["docs/_build/"]) - ["make -C docs -j sphinx"] + () in let job_documentation_linkcheck : tezos_job = - job - ~__POS__ - ~name:"documentation:linkcheck" - ~image:Images.CI.test - ~stage:Stages.doc - ~dependencies: - (Dependent - [ - Artifacts job_manuals; - Artifacts job_docgen; - Artifacts job_build_all; - ]) + Documentation.job_linkcheck + ~job_build_all ~rules: (make_rules ~dependent:true ~label:"ci--docs" ~manual:(On_changes changeset_octez_docs) ()) - ~allow_failure:Yes - ~before_script: - (before_script - ~source_version:true - ~eval_opam:true - ~init_python_venv:true - []) - ["make -C docs redirectcheck"; "make -C docs linkcheck"] + () in [ job_odoc; diff --git a/ci/bin/common.ml b/ci/bin/common.ml index c92719214916ab23649d164ce8f45055d3e0f3bc..4591851b9ddda54d2cc28c629102b8c7759cd369 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -1363,3 +1363,181 @@ module Tezt = struct "tezt/records/*.json.broken"; ]) end + +module Documentation = struct + let mk_artifact_dependencies ?(dependencies = Dependent []) jobs = + List.fold_right + (fun job dependencies -> + dependencies_add_artifact_dependency dependencies job) + jobs + dependencies + + (** Create an odoc job. + + The job will build the target [odoc] (resp. [odoc-lite]) in the + directory [docs] if lite is [false] (resp. [true]), which is the + default. + + This job is one of the prerequisites to {!job_build_all}. *) + let job_odoc ?(lite = false) ?rules ?dependencies () : tezos_job = + let target = if lite then "odoc-lite" else "odoc" in + job + ~__POS__ + ~name:"documentation:odoc" + ~image:Images.CI.test + ~stage:Stages.doc + ?dependencies + ?rules + ~before_script:(before_script ~eval_opam:true []) + ~artifacts: + (artifacts + ~when_:Always + ~expire_in:(Duration (Hours 1)) + (* Path must be terminated with / to expose artifact (gitlab-org/gitlab#/36706) *) + ["docs/_build/api/odoc/"; "docs/odoc.log"]) + ["make -C docs " ^ target] + |> enable_cargo_cache |> enable_sccache + + (** Create the manuals job. + + This job builds the command-line interface manuals (i.e [man]) + of octez binaries, for inclusion in the documentation. + + This job is one of the prerequisites to {!job_build_all}. *) + let job_manuals ?rules ?dependencies () : tezos_job = + job + ~__POS__ + ~name:"documentation:manuals" + ~image:Images.CI.test + ~stage:Stages.doc + ?dependencies + ?rules + ~before_script:(before_script ~eval_opam:true []) + ~artifacts: + (artifacts + ~expire_in:(Duration (Weeks 1)) + [ + "docs/*/octez-*.html"; + "docs/api/octez-*.txt"; + "docs/developer/metrics.csv"; + "docs/user/node-config.json"; + ]) + ["./scripts/ci/documentation:manuals.sh"] + |> enable_cargo_cache |> enable_sccache + + (** Create the docgen job. + + This job builds various generated reference material, for + inclusion in the documentation. This includes the RPC, P2p and + error reference. + + This job is one of the prerequisites to {!job_build_all}. *) + let job_docgen ?rules ?dependencies () : tezos_job = + job + ~__POS__ + ~name:"documentation:docgen" + ~image:Images.CI.test + ~stage:Stages.doc + ?dependencies + ?rules + ~before_script:(before_script ~eval_opam:true []) + ~artifacts: + (artifacts + ~expire_in:(Duration (Weeks 1)) + [ + "docs/alpha/rpc.rst"; + "docs/shell/rpc.rst"; + "docs/user/default-acl.json"; + "docs/api/errors.rst"; + "docs/shell/p2p_api.rst"; + ]) + ["make -C docs -j docexes-gen"] + |> enable_cargo_cache |> enable_sccache + + (** Create the [documentation:build_all] job. + + This jobs builds the RST sources in docs, which will include the + the material from the prerequisite jobs: {!job_odoc}, + {!job_manuals}, {!job_docgen}, on whose artifacts the + [documentation:build_all] job will depend. *) + let job_build_all ~job_odoc ~job_manuals ~job_docgen ?dependencies ?rules () : + tezos_job = + let dependencies = + mk_artifact_dependencies ?dependencies [job_odoc; job_manuals; job_docgen] + in + job + ~__POS__ + ~name:"documentation:build_all" + ~image:Images.CI.test + ~stage:Stages.doc + ~dependencies + ?rules + ~before_script:(before_script ~eval_opam:true ~init_python_venv:true []) + ~artifacts: + (artifacts + ~expose_as:"Documentation - excluding old protocols" + ~expire_in:(Duration (Weeks 1)) + (* Path must be terminated with / to expose artifact (gitlab-org/gitlab#/36706) *) + ["docs/_build/"]) + ["make -C docs -j sphinx"] + + (** Create a [documentation:linkcheck] job. *) + let job_linkcheck ~job_build_all ?dependencies ?rules () : tezos_job = + let dependencies = mk_artifact_dependencies ?dependencies [job_build_all] in + job + ~__POS__ + ~name:"documentation:linkcheck" + ~image:Images.CI.test + ~stage:Stages.doc + ~dependencies + (* Warning: the [documentation:linkcheck] job must have at least the same + restrictions in the rules as [documentation:build_all], otherwise the CI + may complain that [documentation:linkcheck] depends on [documentation:build_all] + which does not exist. *) + ?rules + ~allow_failure:Yes + ~before_script: + (before_script + ~source_version:true + ~eval_opam:true + ~init_python_venv:true + []) + ["make -C docs redirectcheck"; "make -C docs linkcheck"] + + (** Create a [publish:documentation] job. + + This job, for inclusion on [master_branch] pipelines, publishes + built documentation received as artifact from a {!job_build_all} + to [tezos.gitlab.io]. *) + let job_publish_documentation ~job_build_all ?dependencies ?rules () : + tezos_job = + let dependencies = mk_artifact_dependencies ?dependencies [job_build_all] in + job + ~__POS__ + ~name:"publish:documentation" + ~image:Images.CI.test + ~stage:Stages.doc + ~dependencies + ~before_script: + (before_script + ~eval_opam:true + (* Load the environment poetry previously created in the docker image. + Give access to the Python dependencies/executables. *) + ~init_python_venv:true + (* [CI_PK_GITLAB_DOC] and [CI_KH] are set in the projects + GitLab setting and are only available on protected + refs. [CI_PK_GITLAB_DOC] contains the private key used + as deploy key for the Octez documentation on + [tezos.gitlab.io]. [CI_KH] contains the host key of + [gitlab.com], which is not really a secret (it's + public!). *) + [ + {|echo "${CI_PK_GITLAB_DOC}" > ~/.ssh/id_ed25519|}; + {|echo "${CI_KH}" > ~/.ssh/known_hosts|}; + {|chmod 400 ~/.ssh/id_ed25519|}; + ]) + ~interruptible:false + ?rules + ["./scripts/ci/doc_publish.sh"] + |> enable_cargo_cache |> enable_sccache +end diff --git a/ci/bin/master_branch.ml b/ci/bin/master_branch.ml index e6954b330a2b5c12631e065fa2f3e25a8dd3ac5c..4eed35425cd4eaa7623bfad59d670b8466f201c9 100644 --- a/ci/bin/master_branch.ml +++ b/ci/bin/master_branch.ml @@ -85,36 +85,23 @@ let jobs = ] |> enable_coverage_location |> enable_coverage_report in - let job_publish_documentation : tezos_job = - job - ~__POS__ - ~name:"publish:documentation" - ~image:Images.CI.test - ~stage:Stages.doc - ~dependencies:(Dependent []) - ~before_script: - (before_script - ~eval_opam:true - (* Load the environment poetry previously created in the docker image. - Give access to the Python dependencies/executables. *) - ~init_python_venv:true - [ - {|echo "${CI_PK_GITLAB_DOC}" > ~/.ssh/id_ed25519|}; - {|echo "${CI_KH}" > ~/.ssh/known_hosts|}; - {|chmod 400 ~/.ssh/id_ed25519|}; - ]) - ~interruptible:false - (* We publish documentation [Always] -- the job has no - dependencies and so [On_success] is actually equivalent to - [Always], but the latter is more explicit. *) - ~rules: - [ - job_rule - ~changes:(Changeset.encode changeset_octez_docs) - ~when_:Always - (); - ] - ["./scripts/ci/doc_publish.sh"] + let jobs_documentation : tezos_job list = + let rules = + [job_rule ~changes:(Changeset.encode changeset_octez_docs) ()] + in + let dependencies = Dependent [] in + let job_odoc = Documentation.job_odoc ~rules ~dependencies () in + let job_manuals = Documentation.job_manuals ~rules ~dependencies () in + let job_docgen = Documentation.job_docgen ~rules ~dependencies () in + let job_build_all = + Documentation.job_build_all ~job_odoc ~job_manuals ~job_docgen ~rules () + in + let job_publish_documentation : tezos_job = + Documentation.job_publish_documentation ~job_build_all ~rules () + in + [ + job_odoc; job_manuals; job_docgen; job_build_all; job_publish_documentation; + ] in (* Smart Rollup: Kernel SDK @@ -152,10 +139,12 @@ let jobs = job_docker_arm64_experimental; (* Stage: test_coverage *) job_unified_coverage_default; - (* Stage: doc *) - job_publish_documentation |> enable_cargo_cache |> enable_sccache; - (* Stage: prepare_release *) - job_docker_merge_manifests; - (* Stage: manual *) - job_publish_kernel_sdk; ] + (* Stage: doc *) + @ jobs_documentation + @ [ + (* Stage: prepare_release *) + job_docker_merge_manifests; + (* Stage: manual *) + job_publish_kernel_sdk; + ] diff --git a/scripts/ci/doc_publish.sh b/scripts/ci/doc_publish.sh index 19093526b07b98b565c30bf456de3f914a8afa0c..d5e56f19ad518729d689cc7e4f1df61586471a24 100755 --- a/scripts/ci/doc_publish.sh +++ b/scripts/ci/doc_publish.sh @@ -4,9 +4,6 @@ set -e cd "${CI_PROJECT_DIR}" || exit 1 -make all -make -C docs -j full - if [ "${CI_COMMIT_REF_NAME}" == "master" ]; then git clone --depth 5 git@gitlab.com:"${CI_PROJECT_NAMESPACE}"/"${CI_PROJECT_NAMESPACE}".gitlab.io gitlab.io rsync --recursive --links --perms --delete --exclude=.doctrees --exclude={{main,alpha,zero}net,master}/index.html docs/_build/ gitlab.io/public/