diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index bb3f0d3326a8c8fcbfa8118f4d5e9ee3d670e8c6..887e1dd5076992c4bb038db0e09ea6bd8700bec3 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -264,6 +264,25 @@ check_rust_fmt: script: - scripts/check-format-rust.sh +documentation:rst-check: + image: ${ci_image_name}/test:${ci_image_tag} + stage: sanity + tags: + - gcp + rules: + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - docs/**/*.rst + when: on_success + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + before_script: + - . $HOME/.venv/bin/activate + script: + - make --silent -C docs sphinx-check + nix: image: nixos/nix:2.22.1 stage: sanity @@ -438,6 +457,8 @@ oc.build:static-x86_64-linux-binaries: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -515,6 +536,8 @@ oc.build_x86_64-released: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -602,6 +625,8 @@ oc.build_x86_64-exp-dev-extra: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -682,6 +707,8 @@ ocaml-check: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -1001,6 +1028,8 @@ opam:prepare: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -1833,6 +1862,8 @@ kaitai_checks: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -1921,6 +1952,8 @@ oc.check_lift_limits_patch: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -1978,6 +2011,8 @@ oc.python_check: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -2095,6 +2130,8 @@ oc.script:test-gen-genesis: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -2231,6 +2268,8 @@ oc.script:b58_prefix: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -3049,6 +3088,8 @@ oc.install_opam_jammy: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: [] @@ -3084,6 +3125,8 @@ oc.compile_sources_doc_bookworm: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: [] @@ -3702,6 +3745,8 @@ commit_titles: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -3806,6 +3851,8 @@ oc.install_python_bookworm: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: [] @@ -3855,6 +3902,8 @@ documentation:odoc: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -3927,6 +3976,8 @@ documentation:manuals: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -4000,6 +4051,8 @@ documentation:docgen: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index bb3f0d3326a8c8fcbfa8118f4d5e9ee3d670e8c6..887e1dd5076992c4bb038db0e09ea6bd8700bec3 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -264,6 +264,25 @@ check_rust_fmt: script: - scripts/check-format-rust.sh +documentation:rst-check: + image: ${ci_image_name}/test:${ci_image_tag} + stage: sanity + tags: + - gcp + rules: + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - docs/**/*.rst + when: on_success + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + before_script: + - . $HOME/.venv/bin/activate + script: + - make --silent -C docs sphinx-check + nix: image: nixos/nix:2.22.1 stage: sanity @@ -438,6 +457,8 @@ oc.build:static-x86_64-linux-binaries: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -515,6 +536,8 @@ oc.build_x86_64-released: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -602,6 +625,8 @@ oc.build_x86_64-exp-dev-extra: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -682,6 +707,8 @@ ocaml-check: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -1001,6 +1028,8 @@ opam:prepare: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -1833,6 +1862,8 @@ kaitai_checks: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -1921,6 +1952,8 @@ oc.check_lift_limits_patch: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -1978,6 +2011,8 @@ oc.python_check: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -2095,6 +2130,8 @@ oc.script:test-gen-genesis: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -2231,6 +2268,8 @@ oc.script:b58_prefix: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -3049,6 +3088,8 @@ oc.install_opam_jammy: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: [] @@ -3084,6 +3125,8 @@ oc.compile_sources_doc_bookworm: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: [] @@ -3702,6 +3745,8 @@ commit_titles: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -3806,6 +3851,8 @@ oc.install_python_bookworm: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: [] @@ -3855,6 +3902,8 @@ documentation:odoc: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -3927,6 +3976,8 @@ documentation:manuals: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: @@ -4000,6 +4051,8 @@ documentation:docgen: optional: true - job: check_rust_fmt optional: true + - job: documentation:rst-check + optional: true - job: nix optional: true dependencies: diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index d95062db218fd51747499b3e54c2ff3bfca99521..dff2c81868862ca7f57a4543f85023c7cc1c673c 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -196,6 +196,21 @@ check_rust_fmt: script: - scripts/check-format-rust.sh +documentation:rst-check: + image: ${ci_image_name}/test:${ci_image_tag} + stage: sanity + tags: + - gcp + rules: + - when: always + dependencies: + - oc.docker:ci:amd64 + timeout: 60 minutes + before_script: + - . $HOME/.venv/bin/activate + script: + - make --silent -C docs sphinx-check + oc.build_arm64-released: image: ${ci_image_name}/build:${ci_image_tag} stage: build diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 646d978818a0db53188443d979da7352bb95577e..a30a8bf1ba8b5fa024e40485e1878844a7f32736 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -428,6 +428,16 @@ let jobs pipeline_type = ~rules:(make_rules ~dependent:true ~changes:changeset_rust_fmt_files ()) ["scripts/check-format-rust.sh"] in + let job_check_rst = + job + ~__POS__ + ~name:"documentation:rst-check" + ~image:Images.CI.test + ~stage + ~rules:(make_rules ~changes:changeset_octez_docs_rst ()) + ~before_script:(before_script ~init_python_venv:true []) + ["make --silent -C docs sphinx-check"] + in let mr_only_jobs = match pipeline_type with | Before_merging -> @@ -445,6 +455,7 @@ let jobs pipeline_type = job_semgrep; job_oc_misc_checks; job_check_rust_fmt; + job_check_rst; ] @ mr_only_jobs in diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 951628b5fe6d8c2e8efc569f7ba8c16e8a8f2243..82d5b602653dfa648ddbb643ccfa5515d4ff165d 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -402,6 +402,10 @@ let changeset_octez_docs = "docs/**/*"; ]) +(** Only if reStructured Text files have changed *) +let changeset_octez_docs_rst = + Changeset.(changeset_base @ make ["docs/**/*.rst"]) + let changeset_octez_docker_changes_or_master = Changeset.( changeset_base diff --git a/docs/Makefile b/docs/Makefile index 3ede0e192f6d1584d398e78e594993cd04cacb36..ac94c4cc1df99c0cab30b8018ce8accbbc1834c6 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,7 +1,7 @@ # TODO #2170: search for old/preceding protocol name AND number, and adapt # You can set these variables from the command line. -SPHINXOPTS = -aE -n -W --keep-going +SPHINXOPTS = --write-all --fresh-env --nitpicky --fail-on-warning --keep-going --quiet SPHINXBUILD = poetry run sphinx-build SOURCEDIR = . TMPDOCDIR = /tmp/octezdoc @@ -32,17 +32,20 @@ SCRIPTSDIR = scripts CHECKXREFS = $(SCRIPTSDIR)/check_proto_xrefs.py # Build all documentation (with CLI manuals and odoc) but without old PROTOCOLS +.PHONY: odoc-lite all: odoc-lite # Html needs the API pages for OCaml modules, generated by odoc, because of # the :package: custom roles in RST files; so run html *after* odoc ${MAKE} html # Build the full documentation including archived protocols; for publishing only +.PHONY: odoc full: odoc # Same ordering constraint: html *after* odoc (see target "all") ${MAKE} html # Build html without odoc (if odoc already ran or if not checking custom roles) +.PHONY: html html: octez-gen docexes-gen ${MAKE} sphinx @@ -54,26 +57,37 @@ html: octez-gen docexes-gen %/octez-accuser.html: @../octez-accuser-$($(@D)_short) man -verbosity 3 -format html | sed "s#${HOME}#\$$HOME#g" > $@ -manuals: \ - $(PROTOCOLS:%=%/octez-client.html) \ - $(PROTOCOLS:%=%/octez-baker.html) \ - $(PROTOCOLS:%=%/octez-accuser.html) - # generic - @../octez-admin-client man -verbosity 3 -format html | sed "s#${HOME}#\$$HOME#g" > api/octez-admin-client.html - @../octez-signer man -verbosity 3 -format html | sed "s#${HOME}#\$$HOME#g" > api/octez-signer.html - @../octez-codec man -verbosity 3 -format html | sed "s#${HOME}#\$$HOME#g" > api/octez-codec.html - @../octez-snoop man -verbosity 3 -format html | sed "s#${HOME}#\$$HOME#g" > api/octez-snoop.html - @ ../octez-dac-node man -verbosity 3 -format html | sed "s#${HOME}#\$$HOME#g" > api/octez-dac-node.html - @ ../octez-dac-client man -verbosity 3 -format html | sed "s#${HOME}#\$$HOME#g" > api/octez-dac-client.html +# Binary non-protocol specific manuals for clic binaries. +api/%.html: + @../$* man -verbosity 3 -format html | sed "s#${HOME}#\$$HOME#g" > api/$*.html + +# Binary non-protocol specific manuals for cmdliner binaries. +api/octez-node.txt: @../octez-node --help | grep -v '^Octez-node' | grep -v 'OCTEZ-NODE(1)' | \ ./scripts/man2txt.py -r > api/octez-node.txt - @../octez-smart-rollup-node man -verbosity 3 -format html | \ - sed "s#${HOME}#\$$HOME#g" > api/octez-smart-rollup-node.html +api/octez-dal-node.txt: @../octez-dal-node --help | grep -v '^Octez-dal-node' | grep -v 'OCTEZ-DAL-NODE(1)' | \ ./scripts/man2txt.py -r > api/octez-dal-node.txt +api/octez-dal-node-config-init.txt: @../octez-dal-node config init --help | grep -v '^Octez-dal-node' | grep -v 'OCTEZ-DAL-NODE(1)' | \ ./scripts/man2txt.py > api/octez-dal-node-config-init.txt +.PHONY: manuals +manuals: \ + $(PROTOCOLS:%=%/octez-client.html) \ + $(PROTOCOLS:%=%/octez-baker.html) \ + $(PROTOCOLS:%=%/octez-accuser.html) \ + api/octez-admin-client.html \ + api/octez-signer.html \ + api/octez-codec.html \ + api/octez-snoop.html \ + api/octez-dac-node.html \ + api/octez-dac-client.html \ + api/octez-smart-rollup-node.html \ + api/octez-node.txt \ + api/octez-dal-node.txt \ + api/octez-dal-node-config-init.txt + .PHONY: odoc odoc: $(MAKE) odoc-raw slim_mode=off odoc_errors=false @@ -103,7 +117,7 @@ odoc-raw: # Run this after make html linkcheck: @echo "Running linkcheck, outputs messages to $(BUILDDIR)/output.txt" - @$(SPHINXBUILD) -b linkcheck "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) >$@;\ + @$(SPHINXBUILD) --builder linkcheck "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) >$@;\ RESULT=$$?; cat $(BUILDDIR)/output.txt; exit $$RESULT .PHONY: fixlinkcheck @@ -118,11 +132,13 @@ redirectcheck: # xrefs should be checked: # - on the active protocol using its numeric name (not the "active" symlink) # - on each other protocol including alpha, also checking label defs (option -l) +.PHONY: xrefscheck xrefscheck: $(CHECKXREFS) quebeca $(CHECKXREFS) paris $(CHECKXREFS) -l alpha +.PHONY: installcheck installcheck: scripts/test_install_scripts.sh install-bin-noble scripts/test_install_scripts.sh install-bin-jammy @@ -137,6 +153,7 @@ installcheck: scripts/test_install_scripts.sh compile-sources-mantic # build dune targets together to avoid concurrent dunes +.PHONY: docexes docexes: cd .. && dune build docs/$(DOCERRORDIR)/error_doc.exe docs/$(DOCGENDIR)/rpc_doc.exe docs/$(DOCGENDIR)/p2p_doc.exe @@ -159,10 +176,12 @@ shell/rpc.rst: $(RPCDOCEXE) user/default-acl.json: $(RPCDOCEXE) $(RPCDOCEXE) acl > $@ +.PHONY: rpc rpc: user/default-acl.json shell/rpc.rst \ alpha/rpc.rst # To be triggered manually when needed +.PHONY: rpc-protocols rpc-protocols: $(NAMED_PROTOS:%=%/rpc.rst) .PHONY: openapi @@ -173,7 +192,7 @@ openapi: else echo "OK: Your docs/api/*.json are up to date";\ fi -p2p: $(P2PDOCEXE) shell/p2p_usage.rst.inc +shell/p2p_api.rst: $(P2PDOCEXE) shell/p2p_usage.rst.inc @$(P2PDOCEXE) < shell/p2p_usage.rst.inc > shell/p2p_api.rst .PHONY: install-dependencies @@ -184,17 +203,39 @@ user/node-config.json: user/node-config.sh cd ..; ./docs/user/node-config.sh >docs/$@ # Pages generated by running Octez binaries +.PHONY: octez-gen octez-gen: developer/metrics.csv user/node-config.json manuals # Pages generated by the generators under docs/doc_gen/ -docexes-gen: api/errors.rst p2p rpc +.PHONY: docexes-gen +docexes-gen: api/errors.rst shell/p2p_api.rst rpc # Run Sphinx, assuming all generated pages have been built previously +.PHONY: sphinx sphinx: install-dependencies (echo ':orphan:'; echo ''; cat ../CHANGES.rst) > CHANGES-dev.rst - @$(SPHINXBUILD) -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) + @$(SPHINXBUILD) --builder html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) @ln -s active "$(BUILDDIR)/paris" || true +# Syntax check RST files. +# Run Sphinx, but first create dummy version of all generated files, +# and write no output. Can be used to syntax check RST files. +sphinx-check: install-dependencies + @(echo ':orphan:'; echo ''; cat ../CHANGES.rst) > CHANGES-dev.rst +# Build a dummy version of all prerequisites + @${MAKE} --touch html >/dev/null +# Unless this 'alpha/rpc.rst' contains a title, a non-suppressable warning is raised: + @echo "DUMMY TITLE" > alpha/rpc.rst + @echo "===========" >> alpha/rpc.rst +# Unless this file contain the following lines, a non-suppressable warning is raised: + @echo "OPTIONS" > api/octez-dal-node-config-init.txt + @echo "COMMON OPTIONS" >> api/octez-dal-node-config-init.txt +# metrics.csv must actually contain some CSV rows with 4 fields each. + @echo "a,b,c,d" > developer/metrics.csv + @echo "1,2,3,4" >> developer/metrics.csv +# Build with dummy output + @$(SPHINXBUILD) --builder dummy $(SPHINXOPTS) "$(SOURCEDIR)" "$(BUILDDIR)" + # pylint does not support pyproject.toml configuration for now # It is configured by ./.pylint.rc # In pylint, we ignore the following additional errors: @@ -223,32 +264,40 @@ BLACK=poetry run black PYTHON_SRCS?=conf.py _extensions/*.py scripts/*.py +.PHONY: pylint pylint: @echo "Linting with pylint, version:" @poetry run pylint --version | sed 's/^/ /' $(PYLINT) $(PYTHON_SRCS) +.PHONY: lint_black lint_black: @echo "Running black formatter as style checker" $(BLACK) --check $(PYTHON_SRCS) +.PHONY: pycodestyle pycodestyle: @echo "Linting with pycodestyle version `poetry run pycodestyle --version` (`poetry run which pycodestyle`)" $(PYCODESTYLE) $(PYCODESTYLE_OPTIONS) $(PYTHON_SRCS) +.PHONY: typecheck typecheck: @echo "Typechecking with mypy version `poetry run mypy --version`" $(TYPECHECK) $(PYTHON_SRCS) +.PHONY: lint lint: pylint pycodestyle lint_black typecheck # black options are set in pyproject.toml +.PHONY: black black: @echo "Running black formatter" $(BLACK) $(PYTHON_SRCS) +.PHONY: fmt fmt: black +.PHONY: clean clean: @-rm -Rf "$(BUILDDIR)" linkcheck odoc.log @-rm -f $(ERRDOCEXE) $(RPCDOCEXE) $(P2PDOCEXE) diff --git a/docs/conf.py b/docs/conf.py index f934183a9a171f9bc0530c79aa75ead41ae686c7..6f8d99339a9e51c74afb408d0727e9f7f0ed37f6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -280,3 +280,9 @@ html_domain_indices = False default_role = 'default' html_favicon = 'favicon.ico' + +# Suppress some specific warnings +# suppress_warnings = [ +# 'toc.not_readable', +# 'ref.doc', +# ]