From 4d33d52568c4ef2ff5db6ee73815118df8d140a2 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 10 Apr 2024 15:19:54 +0200 Subject: [PATCH 1/5] CI: inline [jobs_unit_tests] This is more consistent with other jobs. --- ci/bin/code_verification.ml | 355 ++++++++++++++++++------------------ 1 file changed, 175 insertions(+), 180 deletions(-) diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 2a423a843a76..5b8d023dea90 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -194,180 +194,6 @@ let read_opam_packages = Some {name; group; batch_index} | _ -> fail () -let jobs_unit_tests ~job_build_x86_64_release ~job_build_x86_64_exp_dev_extra - ~job_build_arm64_release ~job_build_arm64_exp_dev_extra pipeline_type : - tezos_job list = - let build_dependencies = function - | Amd64 -> - Dependent - [Job job_build_x86_64_release; Job job_build_x86_64_exp_dev_extra] - | Arm64 -> - Dependent - [Job job_build_arm64_release; Job job_build_arm64_exp_dev_extra] - in - let rules = - (* TODO: Note that all jobs defined in this function are - [dependent] in the sense that they all have - [dependencies:]. However, AFAICT, these dependencies are all - dummy dependencies for ordering -- not for artifacts. This - means that these unit tests can very well run even if their - dependencies failed. Moreover, the ordering serves no purpose - on scheduled pipelines, so we might even remove them for that - [pipeline_type]. *) - make_rules ~changes:changeset_octez ~dependent:true pipeline_type - in - let job_unit_test ~__POS__ ?(image = Images.runtime_build_dependencies) - ?timeout ?parallel_vector ~arch ~name ?(enable_coverage = true) - ~make_targets () : tezos_job = - let arch_string = arch_to_string arch in - let script = - ["make $MAKE_TARGETS"] - @ if enable_coverage then ["./scripts/ci/merge_coverage.sh"] else [] - in - let dependencies = build_dependencies arch in - let variables = - [("ARCH", arch_string); ("MAKE_TARGETS", String.concat " " make_targets)] - in - - let variables, parallel = - (* When parallel_vector is set to non-zero (translating to the - [parallel_vector:] clause), set the variable - [DISTRIBUTE_TESTS_TO_PARALLELS] to [true], so that - [scripts/test_wrapper.sh] partitions the set of @runtest - targets to build. *) - match parallel_vector with - | Some n -> - ( variables @ [("DISTRIBUTE_TESTS_TO_PARALLELS", "true")], - Some (Vector n) ) - | None -> (variables, None) - in - let job = - job - ?timeout - ?parallel - ~__POS__ - ~retry:2 - ~name - ~stage:Stages.test - ~image - ~arch - ~dependencies - ~rules - ~variables - ~artifacts: - (artifacts - ~name:"$CI_JOB_NAME-$CI_COMMIT_SHA-${ARCH}" - ["test_results"] - ~reports:(reports ~junit:"test_results/*.xml" ()) - ~expire_in:(Duration (Days 1)) - ~when_:Always) - ~before_script:(before_script ~source_version:true ~eval_opam:true []) - script - in - if enable_coverage then - job |> enable_coverage_instrumentation |> enable_coverage_output_artifact - else job - in - let oc_unit_non_proto_x86_64 = - job_unit_test - ~__POS__ - ~name:"oc.unit:non-proto-x86_64" - ~arch:Amd64 (* The [lib_benchmark] unit tests require Python *) - ~image:Images.runtime_build_test_dependencies - ~make_targets:["test-nonproto-unit"] - () - in - let oc_unit_other_x86_64 = - (* Runs unit tests for contrib. *) - job_unit_test - ~__POS__ - ~name:"oc.unit:other-x86_64" - ~arch:Amd64 - ~make_targets:["test-other-unit"] - () - in - let oc_unit_proto_x86_64 = - (* Runs unit tests for protocol. *) - job_unit_test - ~__POS__ - ~name:"oc.unit:proto-x86_64" - ~arch:Amd64 - ~make_targets:["test-proto-unit"] - () - in - let oc_unit_non_proto_arm64 = - job_unit_test - ~__POS__ - ~name:"oc.unit:non-proto-arm64" - ~parallel_vector:2 - ~arch:Arm64 (* The [lib_benchmark] unit tests require Python *) - ~image:Images.runtime_build_test_dependencies - ~make_targets:["test-nonproto-unit"; "test-webassembly"] - (* No coverage for arm64 jobs -- the code they test is a - subset of that tested by x86_64 unit tests. *) - ~enable_coverage:false - () - in - let oc_unit_webassembly_x86_64 = - job - ~__POS__ - ~name:"oc.unit:webassembly-x86_64" - ~arch:Amd64 (* The wasm tests are written in Python *) - ~image:Images.runtime_build_test_dependencies - ~stage:Stages.test - ~dependencies:(build_dependencies Amd64) - ~rules - ~before_script:(before_script ~source_version:true ~eval_opam:true []) - (* TODO: https://gitlab.com/tezos/tezos/-/issues/4663 - This test takes around 2 to 4min to complete, but it sometimes - hangs. We use a timeout to retry the test in this case. The - underlying issue should be fixed eventually, turning this timeout - unnecessary. *) - ~timeout:(Minutes 20) - ["make test-webassembly"] - in - let oc_unit_js_components = - job - ~__POS__ - ~name:"oc.unit:js_components" - ~arch:Amd64 - ~image:Images.runtime_build_test_dependencies - ~stage:Stages.test - ~dependencies:(build_dependencies Amd64) - ~rules - ~retry:2 - ~variables:[("RUNTEZTALIAS", "true")] - ~before_script: - (before_script - ~take_ownership:true - ~source_version:true - ~eval_opam:true - ~install_js_deps:true - []) - ["make test-js"] - in - let oc_unit_protocol_compiles = - job - ~__POS__ - ~name:"oc.unit:protocol_compiles" - ~arch:Amd64 - ~image:Images.runtime_build_dependencies - ~stage:Stages.test - ~dependencies:(build_dependencies Amd64) - ~rules - ~before_script:(before_script ~source_version:true ~eval_opam:true []) - ["dune build @runtest_compile_protocol"] - in - [ - oc_unit_non_proto_x86_64; - oc_unit_other_x86_64; - oc_unit_proto_x86_64; - oc_unit_non_proto_arm64; - oc_unit_webassembly_x86_64; - oc_unit_js_components; - oc_unit_protocol_compiles; - ] - (* These are the set of Linux distributions and their release for which we test installation of the deprecated Serokell PPA binary packages. *) @@ -1158,12 +984,181 @@ let jobs pipeline_type = |> job_external_split in let jobs_unit : tezos_job list = - jobs_unit_tests - ~job_build_x86_64_release - ~job_build_x86_64_exp_dev_extra - ~job_build_arm64_release - ~job_build_arm64_exp_dev_extra - pipeline_type + let build_dependencies = function + | Amd64 -> + Dependent + [Job job_build_x86_64_release; Job job_build_x86_64_exp_dev_extra] + | Arm64 -> + Dependent + [Job job_build_arm64_release; Job job_build_arm64_exp_dev_extra] + in + let rules = + (* TODO: Note that all jobs defined here are + [dependent] in the sense that they all have + [dependencies:]. However, AFAICT, these dependencies are all + dummy dependencies for ordering -- not for artifacts. This + means that these unit tests can very well run even if their + dependencies failed. Moreover, the ordering serves no purpose + on scheduled pipelines, so we might even remove them for that + [pipeline_type]. *) + make_rules ~changes:changeset_octez ~dependent:true () + in + let job_unit_test ~__POS__ ?(image = Images.runtime_build_dependencies) + ?timeout ?parallel_vector ~arch ~name ?(enable_coverage = true) + ~make_targets () : tezos_job = + let arch_string = arch_to_string arch in + let script = + ["make $MAKE_TARGETS"] + @ if enable_coverage then ["./scripts/ci/merge_coverage.sh"] else [] + in + let dependencies = build_dependencies arch in + let variables = + [ + ("ARCH", arch_string); + ("MAKE_TARGETS", String.concat " " make_targets); + ] + in + + let variables, parallel = + (* When parallel_vector is set to non-zero (translating to the + [parallel_vector:] clause), set the variable + [DISTRIBUTE_TESTS_TO_PARALLELS] to [true], so that + [scripts/test_wrapper.sh] partitions the set of @runtest + targets to build. *) + match parallel_vector with + | Some n -> + ( variables @ [("DISTRIBUTE_TESTS_TO_PARALLELS", "true")], + Some (Vector n) ) + | None -> (variables, None) + in + let job = + job + ?timeout + ?parallel + ~__POS__ + ~retry:2 + ~name + ~stage:Stages.test + ~image + ~arch + ~dependencies + ~rules + ~variables + ~artifacts: + (artifacts + ~name:"$CI_JOB_NAME-$CI_COMMIT_SHA-${ARCH}" + ["test_results"] + ~reports:(reports ~junit:"test_results/*.xml" ()) + ~expire_in:(Duration (Days 1)) + ~when_:Always) + ~before_script: + (before_script ~source_version:true ~eval_opam:true []) + script + in + if enable_coverage then + job |> enable_coverage_instrumentation + |> enable_coverage_output_artifact + else job + in + let oc_unit_non_proto_x86_64 = + job_unit_test + ~__POS__ + ~name:"oc.unit:non-proto-x86_64" + ~arch:Amd64 (* The [lib_benchmark] unit tests require Python *) + ~image:Images.runtime_build_test_dependencies + ~make_targets:["test-nonproto-unit"] + () + in + let oc_unit_other_x86_64 = + (* Runs unit tests for contrib. *) + job_unit_test + ~__POS__ + ~name:"oc.unit:other-x86_64" + ~arch:Amd64 + ~make_targets:["test-other-unit"] + () + in + let oc_unit_proto_x86_64 = + (* Runs unit tests for protocol. *) + job_unit_test + ~__POS__ + ~name:"oc.unit:proto-x86_64" + ~arch:Amd64 + ~make_targets:["test-proto-unit"] + () + in + let oc_unit_non_proto_arm64 = + job_unit_test + ~__POS__ + ~name:"oc.unit:non-proto-arm64" + ~parallel_vector:2 + ~arch:Arm64 (* The [lib_benchmark] unit tests require Python *) + ~image:Images.runtime_build_test_dependencies + ~make_targets:["test-nonproto-unit"; "test-webassembly"] + (* No coverage for arm64 jobs -- the code they test is a + subset of that tested by x86_64 unit tests. *) + ~enable_coverage:false + () + in + let oc_unit_webassembly_x86_64 = + job + ~__POS__ + ~name:"oc.unit:webassembly-x86_64" + ~arch:Amd64 (* The wasm tests are written in Python *) + ~image:Images.runtime_build_test_dependencies + ~stage:Stages.test + ~dependencies:(build_dependencies Amd64) + ~rules + ~before_script:(before_script ~source_version:true ~eval_opam:true []) + (* TODO: https://gitlab.com/tezos/tezos/-/issues/4663 + This test takes around 2 to 4min to complete, but it sometimes + hangs. We use a timeout to retry the test in this case. The + underlying issue should be fixed eventually, turning this timeout + unnecessary. *) + ~timeout:(Minutes 20) + ["make test-webassembly"] + in + let oc_unit_js_components = + job + ~__POS__ + ~name:"oc.unit:js_components" + ~arch:Amd64 + ~image:Images.runtime_build_test_dependencies + ~stage:Stages.test + ~dependencies:(build_dependencies Amd64) + ~rules + ~retry:2 + ~variables:[("RUNTEZTALIAS", "true")] + ~before_script: + (before_script + ~take_ownership:true + ~source_version:true + ~eval_opam:true + ~install_js_deps:true + []) + ["make test-js"] + in + let oc_unit_protocol_compiles = + job + ~__POS__ + ~name:"oc.unit:protocol_compiles" + ~arch:Amd64 + ~image:Images.runtime_build_dependencies + ~stage:Stages.test + ~dependencies:(build_dependencies Amd64) + ~rules + ~before_script:(before_script ~source_version:true ~eval_opam:true []) + ["dune build @runtest_compile_protocol"] + in + [ + oc_unit_non_proto_x86_64; + oc_unit_other_x86_64; + oc_unit_proto_x86_64; + oc_unit_non_proto_arm64; + oc_unit_webassembly_x86_64; + oc_unit_js_components; + oc_unit_protocol_compiles; + ] |> jobs_external_split ~path:"test/oc.unit" in let job_oc_integration_compiler_rejections : tezos_job = -- GitLab From 185328547be08c2eee903dd1f79edbbf7c0093c3 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 27 Mar 2024 16:58:54 +0100 Subject: [PATCH 2/5] CI: generate [oc.install_python] --- .../doc/oc.install_python-before_merging.yml | 62 +++++++++++++++++++ ...install_python-scheduled_extended_test.yml | 38 ++++++++++++ .gitlab/ci/jobs/doc/oc.install_python.yml | 31 ---------- .gitlab/ci/jobs/shared/images.yml | 2 + .gitlab/ci/pipelines/before_merging.yml | 2 +- .../ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/code_verification.ml | 44 ++++++++++++- ci/bin/common.ml | 3 + ci/bin/main.ml | 1 - 9 files changed, 150 insertions(+), 35 deletions(-) create mode 100644 .gitlab/ci/jobs/doc/oc.install_python-before_merging.yml create mode 100644 .gitlab/ci/jobs/doc/oc.install_python-scheduled_extended_test.yml delete mode 100644 .gitlab/ci/jobs/doc/oc.install_python.yml diff --git a/.gitlab/ci/jobs/doc/oc.install_python-before_merging.yml b/.gitlab/ci/jobs/doc/oc.install_python-before_merging.yml new file mode 100644 index 000000000000..ec95d0606a57 --- /dev/null +++ b/.gitlab/ci/jobs/doc/oc.install_python-before_merging.yml @@ -0,0 +1,62 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.install_python_focal: + image: public.ecr.aws/lts/ubuntu:20.04_stable + stage: doc + tags: + - gcp + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success + - changes: + - docs/developer/install-python-debian-ubuntu.sh + when: on_success + - when: manual + allow_failure: true + needs: + - trigger + dependencies: [] + script: + - ./docs/developer/install-python-debian-ubuntu.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} + ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} + +oc.install_python_jammy: + image: public.ecr.aws/lts/ubuntu:22.04_stable + stage: doc + tags: + - gcp + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success + - changes: + - docs/developer/install-python-debian-ubuntu.sh + when: on_success + - when: manual + allow_failure: true + needs: + - trigger + dependencies: [] + script: + - ./docs/developer/install-python-debian-ubuntu.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} + ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} + +oc.install_python_bullseye: + image: debian:bullseye + stage: doc + tags: + - gcp + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success + - changes: + - docs/developer/install-python-debian-ubuntu.sh + when: on_success + - when: manual + allow_failure: true + needs: + - trigger + dependencies: [] + script: + - ./docs/developer/install-python-debian-ubuntu.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} + ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} diff --git a/.gitlab/ci/jobs/doc/oc.install_python-scheduled_extended_test.yml b/.gitlab/ci/jobs/doc/oc.install_python-scheduled_extended_test.yml new file mode 100644 index 000000000000..caa00bb3a172 --- /dev/null +++ b/.gitlab/ci/jobs/doc/oc.install_python-scheduled_extended_test.yml @@ -0,0 +1,38 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +oc.install_python_focal: + image: public.ecr.aws/lts/ubuntu:20.04_stable + stage: doc + tags: + - gcp + rules: + - when: always + dependencies: [] + script: + - ./docs/developer/install-python-debian-ubuntu.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} + ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} + +oc.install_python_jammy: + image: public.ecr.aws/lts/ubuntu:22.04_stable + stage: doc + tags: + - gcp + rules: + - when: always + dependencies: [] + script: + - ./docs/developer/install-python-debian-ubuntu.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} + ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} + +oc.install_python_bullseye: + image: debian:bullseye + stage: doc + tags: + - gcp + rules: + - when: always + dependencies: [] + script: + - ./docs/developer/install-python-debian-ubuntu.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} + ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} diff --git a/.gitlab/ci/jobs/doc/oc.install_python.yml b/.gitlab/ci/jobs/doc/oc.install_python.yml deleted file mode 100644 index 5dca6ce53451..000000000000 --- a/.gitlab/ci/jobs/doc/oc.install_python.yml +++ /dev/null @@ -1,31 +0,0 @@ -.oc.install_python_debian_ubuntu_template: - extends: - - .default_settings_template - - .needs__trigger - stage: doc - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule" && $TZ_SCHEDULE_KIND == "EXTENDED_TESTS"' - when: always - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - changes: - - docs/developer/install-python-debian-ubuntu.sh - when: on_success - # Run when there is label on the merge request - - if: '$CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/' - when: on_success - - when: manual - allow_failure: true - script: - - ./docs/developer/install-python-debian-ubuntu.sh ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master} - -oc.install_python_focal: - image: public.ecr.aws/lts/ubuntu:20.04_stable - extends: .oc.install_python_debian_ubuntu_template - -oc.install_python_jammy: - image: public.ecr.aws/lts/ubuntu:22.04_stable - extends: .oc.install_python_debian_ubuntu_template - -oc.install_python_bullseye: - image: debian:bullseye - extends: .oc.install_python_debian_ubuntu_template diff --git a/.gitlab/ci/jobs/shared/images.yml b/.gitlab/ci/jobs/shared/images.yml index 6affd59cdf19..c25b7f247263 100644 --- a/.gitlab/ci/jobs/shared/images.yml +++ b/.gitlab/ci/jobs/shared/images.yml @@ -9,6 +9,8 @@ image: ${client_libs_dependencies_image_name}:${client_libs_dependencies_image_tag} .image_template__debian_bookworm: image: debian:bookworm +.image_template__debian_bullseye: + image: debian:bullseye .image_template__debian_dependencies_image: image: $DEP_IMAGE:${CI_COMMIT_REF_SLUG} .image_template__docker: diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 18645fe1c027..f86e4c25aafd 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -61,4 +61,4 @@ include: # Stage: doc - .gitlab/ci/jobs/doc/documentation:linkcheck.yml - - .gitlab/ci/jobs/doc/oc.install_python.yml + - .gitlab/ci/jobs/doc/oc.install_python-before_merging.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index e9ee444e3fbe..944d49890b51 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -58,4 +58,4 @@ include: # Stage: doc - .gitlab/ci/jobs/doc/documentation:linkcheck.yml - - .gitlab/ci/jobs/doc/oc.install_python.yml + - .gitlab/ci/jobs/doc/oc.install_python-scheduled_extended_test.yml diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 5b8d023dea90..6a76b87875da 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -1651,7 +1651,49 @@ let jobs pipeline_type = [job_commit_titles] | Schedule_extended_test -> [] in - let doc = [] in + let doc = + let jobs_install_python = + (* Creates a job that tests installation of the python environment in [image] *) + let job_install_python ~__POS__ ~name ~image = + job + ~__POS__ + ~name + ~image + ~stage:Stages.doc + ~dependencies:dependencies_needs_trigger + ~rules: + (make_rules + ~changes: + (Changeset.make + ["docs/developer/install-python-debian-ubuntu.sh"]) + ~manual:Yes + ~label:"ci--docs" + ()) + [ + "./docs/developer/install-python-debian-ubuntu.sh \ + ${CI_MERGE_REQUEST_SOURCE_PROJECT_PATH:-tezos/tezos} \ + ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-master}"; + ] + in + (* The set of python installation test jobs. *) + [ + job_install_python + ~__POS__ + ~name:"oc.install_python_focal" + ~image:Images.ubuntu_focal; + job_install_python + ~__POS__ + ~name:"oc.install_python_jammy" + ~image:Images.ubuntu_jammy; + job_install_python + ~__POS__ + ~name:"oc.install_python_bullseye" + ~image:Images.debian_bullseye; + ] + |> jobs_external_split ~path:"doc/oc.install_python" + in + jobs_install_python + in let manual = match pipeline_type with | Before_merging -> diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 0f06a9864b50..09c7fd728472 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -145,6 +145,9 @@ module Images = struct let debian_bookworm = Image.register ~name:"debian_bookworm" ~image_path:"debian:bookworm" + let debian_bullseye = + Image.register ~name:"debian_bullseye" ~image_path:"debian:bullseye" + let ubuntu_focal = Image.register ~name:"ubuntu_focal" diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 6e56a8864fe1..0360f39915ac 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -189,7 +189,6 @@ let () = | ".gitlab/ci/jobs/coverage/oc.unified_coverage-before_merging.yml" | ".gitlab/ci/jobs/doc/documentation.yml" | ".gitlab/ci/jobs/doc/documentation:linkcheck.yml" - | ".gitlab/ci/jobs/doc/oc.install_python.yml" | ".gitlab/ci/jobs/shared/images.yml" | ".gitlab/ci/jobs/shared/templates.yml" | ".gitlab/ci/jobs/test/common.yml" | ".gitlab/ci/pipelines/before_merging.yml" -- GitLab From 7ca8caa1410234c4f65f0d64bb3d7b156098c00e Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 6 Mar 2024 14:02:36 +0100 Subject: [PATCH 3/5] CI: refactor, more precise [before_script:] in [documentation.yml] --- .gitlab/ci/jobs/doc/documentation.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.gitlab/ci/jobs/doc/documentation.yml b/.gitlab/ci/jobs/doc/documentation.yml index d8e72b890661..7a6ae563e9c5 100644 --- a/.gitlab/ci/jobs/doc/documentation.yml +++ b/.gitlab/ci/jobs/doc/documentation.yml @@ -10,13 +10,12 @@ - .needs__trigger - .rules__octez_docs_changes stage: doc - before_script: - - eval $(opam env) - - . $HOME/.venv/bin/activate documentation:odoc: extends: - .doc_job_template + before_script: + - eval $(opam env) script: - make -C docs odoc-lite artifacts: @@ -27,6 +26,8 @@ documentation:odoc: documentation:manuals: extends: - .doc_job_template + before_script: + - eval $(opam env) script: - ./.gitlab/ci/jobs/doc/documentation:manuals.sh artifacts: @@ -40,6 +41,8 @@ documentation:manuals: documentation:docgen: extends: - .doc_job_template + before_script: + - eval $(opam env) script: - make -C docs -j docexes-gen artifacts: @@ -62,6 +65,9 @@ documentation:build_all: - documentation:odoc - documentation:manuals - documentation:docgen + before_script: + - eval $(opam env) + - . $HOME/.venv/bin/activate script: - make -C docs -j sphinx artifacts: -- GitLab From ad273aded08003d18b30ce6b83710ac3d2b72dde Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 27 Mar 2024 17:07:26 +0100 Subject: [PATCH 4/5] CI: generate [documentation.yml] --- .../jobs/doc/documentation-before_merging.yml | 148 ++++++++++++++++++ .../documentation-scheduled_extended_test.yml | 86 ++++++++++ .gitlab/ci/jobs/doc/documentation.yml | 78 --------- .gitlab/ci/pipelines/before_merging.yml | 2 +- .../ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/code_verification.ml | 97 +++++++++++- ci/bin/main.ml | 1 - 7 files changed, 332 insertions(+), 82 deletions(-) create mode 100644 .gitlab/ci/jobs/doc/documentation-before_merging.yml create mode 100644 .gitlab/ci/jobs/doc/documentation-scheduled_extended_test.yml delete mode 100644 .gitlab/ci/jobs/doc/documentation.yml diff --git a/.gitlab/ci/jobs/doc/documentation-before_merging.yml b/.gitlab/ci/jobs/doc/documentation-before_merging.yml new file mode 100644 index 000000000000..65cbaedf6ee8 --- /dev/null +++ b/.gitlab/ci/jobs/doc/documentation-before_merging.yml @@ -0,0 +1,148 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +documentation:odoc: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: doc + tags: + - gcp + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - docs/**/* + - dune + - dune-project + - dune-workspace + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/* + - tezt/**/* + - vendors/**/* + when: on_success + needs: + - trigger + dependencies: [] + before_script: + - eval $(opam env) + script: + - make -C docs odoc-lite + artifacts: + expire_in: 1 hour + paths: + - docs/_build/api/odoc/ + +documentation:manuals: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: doc + tags: + - gcp + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - docs/**/* + - dune + - dune-project + - dune-workspace + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/* + - tezt/**/* + - vendors/**/* + when: on_success + needs: + - trigger + dependencies: [] + before_script: + - eval $(opam env) + script: + - ./.gitlab/ci/jobs/doc/documentation:manuals.sh + artifacts: + expire_in: 1 week + paths: + - docs/*/octez-*.html + - docs/api/octez-*.txt + - docs/developer/metrics.csv + - docs/user/node-config.json + +documentation:docgen: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: doc + tags: + - gcp + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - docs/**/* + - dune + - dune-project + - dune-workspace + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/* + - tezt/**/* + - vendors/**/* + when: on_success + needs: + - trigger + dependencies: [] + before_script: + - eval $(opam env) + script: + - make -C docs -j docexes-gen + artifacts: + expire_in: 1 week + paths: + - docs/alpha/rpc.rst + - docs/shell/rpc.rst + - docs/user/default-acl.json + - docs/api/errors.rst + - docs/shell/p2p_api.rst + +documentation:build_all: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: doc + tags: + - gcp + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - docs/**/* + - dune + - dune-project + - dune-workspace + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/* + - tezt/**/* + - vendors/**/* + when: on_success + needs: + - documentation:odoc + - documentation:manuals + - documentation:docgen + dependencies: + - documentation:odoc + - documentation:manuals + - documentation:docgen + before_script: + - eval $(opam env) + - . $HOME/.venv/bin/activate + script: + - make -C docs -j sphinx + artifacts: + expire_in: 1 week + paths: + - docs/_build/ + expose_as: Documentation - excluding old protocols diff --git a/.gitlab/ci/jobs/doc/documentation-scheduled_extended_test.yml b/.gitlab/ci/jobs/doc/documentation-scheduled_extended_test.yml new file mode 100644 index 000000000000..82910e52a8e2 --- /dev/null +++ b/.gitlab/ci/jobs/doc/documentation-scheduled_extended_test.yml @@ -0,0 +1,86 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +documentation:odoc: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: doc + tags: + - gcp + rules: + - when: always + dependencies: [] + before_script: + - eval $(opam env) + script: + - make -C docs odoc-lite + artifacts: + expire_in: 1 hour + paths: + - docs/_build/api/odoc/ + +documentation:manuals: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: doc + tags: + - gcp + rules: + - when: always + dependencies: [] + before_script: + - eval $(opam env) + script: + - ./.gitlab/ci/jobs/doc/documentation:manuals.sh + artifacts: + expire_in: 1 week + paths: + - docs/*/octez-*.html + - docs/api/octez-*.txt + - docs/developer/metrics.csv + - docs/user/node-config.json + +documentation:docgen: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: doc + tags: + - gcp + rules: + - when: always + dependencies: [] + before_script: + - eval $(opam env) + script: + - make -C docs -j docexes-gen + artifacts: + expire_in: 1 week + paths: + - docs/alpha/rpc.rst + - docs/shell/rpc.rst + - docs/user/default-acl.json + - docs/api/errors.rst + - docs/shell/p2p_api.rst + +documentation:build_all: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: doc + tags: + - gcp + rules: + - when: on_success + needs: + - documentation:odoc + - documentation:manuals + - documentation:docgen + dependencies: + - documentation:odoc + - documentation:manuals + - documentation:docgen + before_script: + - eval $(opam env) + - . $HOME/.venv/bin/activate + script: + - make -C docs -j sphinx + artifacts: + expire_in: 1 week + paths: + - docs/_build/ + expose_as: Documentation - excluding old protocols diff --git a/.gitlab/ci/jobs/doc/documentation.yml b/.gitlab/ci/jobs/doc/documentation.yml deleted file mode 100644 index 7a6ae563e9c5..000000000000 --- a/.gitlab/ci/jobs/doc/documentation.yml +++ /dev/null @@ -1,78 +0,0 @@ -# Warning: the documentation:linkcheck job must have at least the same -# restrictions in the rules as documentation:build_all, otherwise the CI -# may complain that documentation:linkcheck depends on documentation:build_all -# which does not exist. -.doc_job_template: - extends: - - .default_settings_template - - .image_template__runtime_build_test_dependencies - - .tags_template__build - - .needs__trigger - - .rules__octez_docs_changes - stage: doc - -documentation:odoc: - extends: - - .doc_job_template - before_script: - - eval $(opam env) - script: - - make -C docs odoc-lite - artifacts: - paths: - - docs/_build/api/odoc/ - expire_in: 1 hour - -documentation:manuals: - extends: - - .doc_job_template - before_script: - - eval $(opam env) - script: - - ./.gitlab/ci/jobs/doc/documentation:manuals.sh - artifacts: - paths: - - docs/*/octez-*.html - - docs/api/octez-*.txt - - docs/developer/metrics.csv - - docs/user/node-config.json - expire_in: 1 week - -documentation:docgen: - extends: - - .doc_job_template - before_script: - - eval $(opam env) - script: - - make -C docs -j docexes-gen - artifacts: - paths: - - docs/alpha/rpc.rst - - docs/shell/rpc.rst - - docs/user/default-acl.json - - docs/api/errors.rst - - docs/shell/p2p_api.rst - expire_in: 1 week - -documentation:build_all: - extends: - - .doc_job_template - needs: - - documentation:odoc - - documentation:manuals - - documentation:docgen - dependencies: - - documentation:odoc - - documentation:manuals - - documentation:docgen - before_script: - - eval $(opam env) - - . $HOME/.venv/bin/activate - script: - - make -C docs -j sphinx - artifacts: - expose_as: 'Documentation - excluding old protocols' - paths: - # Path must be terminated with / to expose artifact (gitlab-org/gitlab#/36706) - - docs/_build/ - expire_in: 1 week diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index f86e4c25aafd..7c679a95fdea 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -21,7 +21,7 @@ include: - .gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra-before_merging.yml - .gitlab/ci/jobs/build/oc.build_x86_64-released-before_merging.yml - .gitlab/ci/jobs/build/oc.build_kernels-before_merging.yml - - .gitlab/ci/jobs/doc/documentation.yml + - .gitlab/ci/jobs/doc/documentation-before_merging.yml - .gitlab/ci/jobs/build/oc.docker:amd64-test_manual.yml - .gitlab/ci/jobs/build/oc.docker:arm64-test_manual.yml - .gitlab/ci/jobs/build/oc.build:rpm:amd64-manual.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 944d49890b51..f8c62f40464a 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -23,7 +23,7 @@ include: # because of absence of certain changes - .gitlab/ci/jobs/build/ocaml-check-scheduled_extended_test.yml - .gitlab/ci/jobs/build/oc.build:static-x86_64-linux-binaries-scheduled_extended_test.yml - - .gitlab/ci/jobs/doc/documentation.yml + - .gitlab/ci/jobs/doc/documentation-scheduled_extended_test.yml # Stage: packaging - .gitlab/ci/jobs/packaging/opam:prepare-scheduled_extended_test.yml diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 6a76b87875da..f6b3b111f6e9 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -1692,7 +1692,102 @@ let jobs pipeline_type = ] |> jobs_external_split ~path:"doc/oc.install_python" in - jobs_install_python + let jobs_documentation : tezos_job list = + let rules = + make_rules ~changes:changeset_octez_docs ~label:"ci--docs" () + in + let job_odoc = + job + ~__POS__ + ~name:"documentation:odoc" + ~image:Images.runtime_build_test_dependencies + ~stage:Stages.doc + ~dependencies:dependencies_needs_trigger + ~rules + ~before_script:(before_script ~eval_opam:true []) + ~artifacts: + (artifacts + ~expire_in:(Duration (Hours 1)) + (* Path must be terminated with / to expose artifact (gitlab-org/gitlab#/36706) *) + ["docs/_build/api/odoc/"]) + ["make -C docs odoc-lite"] + in + let job_manuals = + job + ~__POS__ + ~name:"documentation:manuals" + ~image:Images.runtime_build_test_dependencies + ~stage:Stages.doc + ~dependencies:dependencies_needs_trigger + ~rules + ~before_script:(before_script ~eval_opam:true []) + ~artifacts: + (artifacts + ~expire_in:(Duration (Weeks 1)) + [ + "docs/*/octez-*.html"; + "docs/api/octez-*.txt"; + "docs/developer/metrics.csv"; + "docs/user/node-config.json"; + ]) + ["./.gitlab/ci/jobs/doc/documentation:manuals.sh"] + in + let job_docgen = + job + ~__POS__ + ~name:"documentation:docgen" + ~image:Images.runtime_build_test_dependencies + ~stage:Stages.doc + ~dependencies:dependencies_needs_trigger + ~rules + ~before_script:(before_script ~eval_opam:true []) + ~artifacts: + (artifacts + ~expire_in:(Duration (Weeks 1)) + [ + "docs/alpha/rpc.rst"; + "docs/shell/rpc.rst"; + "docs/user/default-acl.json"; + "docs/api/errors.rst"; + "docs/shell/p2p_api.rst"; + ]) + ["make -C docs -j docexes-gen"] + in + let doc_build_dependencies = + Dependent + [Artifacts job_odoc; Artifacts job_manuals; Artifacts job_docgen] + in + let job_build_all = + job + ~__POS__ + ~name:"documentation:build_all" + ~image:Images.runtime_build_test_dependencies + ~stage:Stages.doc + ~dependencies:doc_build_dependencies + (* Warning: the [documentation:linkcheck] job must have at least the same + restrictions in the rules as [documentation:build_all], otherwise the CI + may complain that [documentation:linkcheck] depends on [documentation:build_all] + which does not exist. *) + ~rules: + (make_rules + ~dependent:true + ~changes:changeset_octez_docs + ~label:"ci--docs" + ()) + ~before_script: + (before_script ~eval_opam:true ~init_python_venv:true []) + ~artifacts: + (artifacts + ~expose_as:"Documentation - excluding old protocols" + ~expire_in:(Duration (Weeks 1)) + (* Path must be terminated with / to expose artifact (gitlab-org/gitlab#/36706) *) + ["docs/_build/"]) + ["make -C docs -j sphinx"] + in + [job_odoc; job_manuals; job_docgen; job_build_all] + |> jobs_external_split ~path:"doc/documentation" + in + jobs_install_python @ jobs_documentation in let manual = match pipeline_type with diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 0360f39915ac..229869d0a93a 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -187,7 +187,6 @@ let () = let exclude = function | ".gitlab/ci/jobs/coverage/common.yml" | ".gitlab/ci/jobs/coverage/oc.unified_coverage-before_merging.yml" - | ".gitlab/ci/jobs/doc/documentation.yml" | ".gitlab/ci/jobs/doc/documentation:linkcheck.yml" | ".gitlab/ci/jobs/shared/images.yml" | ".gitlab/ci/jobs/shared/templates.yml" | ".gitlab/ci/jobs/test/common.yml" -- GitLab From 274642e6545a2b770f7224af6ba3238c221eba0d Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 27 Mar 2024 17:14:19 +0100 Subject: [PATCH 5/5] CI: generate [documentation:linkcheck] into [documentation.yml] For convenience, I generated it into the file [documentation.yml]. When all jobs are eventually inlined, this will make no difference. --- .../jobs/doc/documentation-before_merging.yml | 39 +++++++++++++++++++ .../documentation-scheduled_extended_test.yml | 24 ++++++++++++ .../ci/jobs/doc/documentation:linkcheck.yml | 22 ----------- .gitlab/ci/pipelines/before_merging.yml | 1 - .../ci/pipelines/schedule_extended_test.yml | 1 - ci/bin/code_verification.ml | 36 ++++++++++++++++- ci/bin/main.ml | 1 - 7 files changed, 98 insertions(+), 26 deletions(-) delete mode 100644 .gitlab/ci/jobs/doc/documentation:linkcheck.yml diff --git a/.gitlab/ci/jobs/doc/documentation-before_merging.yml b/.gitlab/ci/jobs/doc/documentation-before_merging.yml index 65cbaedf6ee8..f454d459b74c 100644 --- a/.gitlab/ci/jobs/doc/documentation-before_merging.yml +++ b/.gitlab/ci/jobs/doc/documentation-before_merging.yml @@ -146,3 +146,42 @@ documentation:build_all: paths: - docs/_build/ expose_as: Documentation - excluding old protocols + +documentation:linkcheck: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: doc + tags: + - gcp + rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /(?:^|,)ci--docs(?:$|,)/ + when: on_success + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - docs/**/* + - dune + - dune-project + - dune-workspace + - script-inputs/**/*/ + - scripts/**/*/ + - src/**/* + - tezt/**/* + - vendors/**/* + when: manual + allow_failure: true + needs: + - documentation:manuals + - documentation:docgen + - documentation:build_all + dependencies: + - documentation:manuals + - documentation:docgen + - documentation:build_all + allow_failure: true + before_script: + - . ./scripts/version.sh + - eval $(opam env) + - . $HOME/.venv/bin/activate + script: + - make -C docs redirectcheck + - make -C docs linkcheck diff --git a/.gitlab/ci/jobs/doc/documentation-scheduled_extended_test.yml b/.gitlab/ci/jobs/doc/documentation-scheduled_extended_test.yml index 82910e52a8e2..d40162fd2f95 100644 --- a/.gitlab/ci/jobs/doc/documentation-scheduled_extended_test.yml +++ b/.gitlab/ci/jobs/doc/documentation-scheduled_extended_test.yml @@ -84,3 +84,27 @@ documentation:build_all: paths: - docs/_build/ expose_as: Documentation - excluding old protocols + +documentation:linkcheck: + image: ${build_deps_image_name}:runtime-build-test-dependencies--${build_deps_image_version} + stage: doc + tags: + - gcp + rules: + - when: on_success + needs: + - documentation:manuals + - documentation:docgen + - documentation:build_all + dependencies: + - documentation:manuals + - documentation:docgen + - documentation:build_all + allow_failure: true + before_script: + - . ./scripts/version.sh + - eval $(opam env) + - . $HOME/.venv/bin/activate + script: + - make -C docs redirectcheck + - make -C docs linkcheck diff --git a/.gitlab/ci/jobs/doc/documentation:linkcheck.yml b/.gitlab/ci/jobs/doc/documentation:linkcheck.yml deleted file mode 100644 index e81df765c479..000000000000 --- a/.gitlab/ci/jobs/doc/documentation:linkcheck.yml +++ /dev/null @@ -1,22 +0,0 @@ -documentation:linkcheck: - extends: - - .default_settings_template - - .image_template__runtime_build_test_dependencies - - .rules__octez_docs_changes_manual - stage: doc - needs: - - documentation:manuals - - documentation:docgen - - documentation:build_all - dependencies: - - documentation:manuals - - documentation:docgen - - documentation:build_all - before_script: - - . ./scripts/version.sh - - eval $(opam env) - - . $HOME/.venv/bin/activate - script: - - make -C docs redirectcheck - - make -C docs linkcheck - allow_failure: true diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 7c679a95fdea..ec7c63a7abae 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -60,5 +60,4 @@ include: - if: '$CI_MERGE_REQUEST_LABELS !~ /(?:^|,)ci--no-coverage(?:$|,)/' # Stage: doc - - .gitlab/ci/jobs/doc/documentation:linkcheck.yml - .gitlab/ci/jobs/doc/oc.install_python-before_merging.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index f8c62f40464a..9cf7cdc80e4f 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -57,5 +57,4 @@ include: - .gitlab/ci/jobs/test/test_evm_compatibility-scheduled_extended_test.yml # Stage: doc - - .gitlab/ci/jobs/doc/documentation:linkcheck.yml - .gitlab/ci/jobs/doc/oc.install_python-scheduled_extended_test.yml diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index f6b3b111f6e9..2b54bed7991b 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -1784,7 +1784,41 @@ let jobs pipeline_type = ["docs/_build/"]) ["make -C docs -j sphinx"] in - [job_odoc; job_manuals; job_docgen; job_build_all] + let job_documentation_linkcheck : tezos_job = + job + ~__POS__ + ~name:"documentation:linkcheck" + ~image:Images.runtime_build_test_dependencies + ~stage:Stages.doc + ~dependencies: + (Dependent + [ + Artifacts job_manuals; + Artifacts job_docgen; + Artifacts job_build_all; + ]) + ~rules: + (make_rules + ~dependent:true + ~label:"ci--docs" + ~manual:(On_changes changeset_octez_docs) + ()) + ~allow_failure:Yes + ~before_script: + (before_script + ~source_version:true + ~eval_opam:true + ~init_python_venv:true + []) + ["make -C docs redirectcheck"; "make -C docs linkcheck"] + in + [ + job_odoc; + job_manuals; + job_docgen; + job_build_all; + job_documentation_linkcheck; + ] |> jobs_external_split ~path:"doc/documentation" in jobs_install_python @ jobs_documentation diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 229869d0a93a..495414f3b95c 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -187,7 +187,6 @@ let () = let exclude = function | ".gitlab/ci/jobs/coverage/common.yml" | ".gitlab/ci/jobs/coverage/oc.unified_coverage-before_merging.yml" - | ".gitlab/ci/jobs/doc/documentation:linkcheck.yml" | ".gitlab/ci/jobs/shared/images.yml" | ".gitlab/ci/jobs/shared/templates.yml" | ".gitlab/ci/jobs/test/common.yml" | ".gitlab/ci/pipelines/before_merging.yml" -- GitLab