From 4df342bcecf67215ef2b3dac1de3528279c2add8 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Mon, 8 Apr 2024 11:30:48 +0200 Subject: [PATCH 1/9] CIAO: add helper [script_propagate_exit_code] --- ci/bin/code_verification.ml | 4 ++-- ci/bin/common.ml | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 6463d2aad3e6..a19d2575df39 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -687,7 +687,7 @@ let jobs pipeline_type = ~image:Images.runtime_prebuild_dependencies ~stage:Stages.build ~before_script:(before_script ~take_ownership:true ~eval_opam:true []) - ["scripts/ci/select_tezts.sh || exit $?"] + (script_propagate_exit_code "scripts/ci/select_tezts.sh") ~allow_failure:(With_exit_codes [17]) ~artifacts: (artifacts @@ -1146,7 +1146,7 @@ let jobs pipeline_type = ~dependencies:dependencies_needs_trigger (* ./scripts/ci/check_commit_messages.sh exits with code 65 when a git history contains invalid commits titles in situations where that is allowed. *) - ["./scripts/ci/check_commit_messages.sh || exit $?"] + (script_propagate_exit_code "./scripts/ci/check_commit_messages.sh") ~allow_failure:(With_exit_codes [65]) |> job_external in diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 5abfd2be1f7d..db162cbbb86b 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -214,6 +214,21 @@ let before_script ?(take_ownership = false) ?(source_version = false) @ toggle install_js_deps ". ./scripts/install_build_deps.js.sh" @ before_script +(** A [script:] that executes [script] and propagates its exit code. + + This might seem like a noop but is in fact necessary to please his + majesty GitLab. + + For more info, see: + - https://gitlab.com/tezos/tezos/-/merge_requests/9923#note_1538894754; + - https://gitlab.com/tezos/tezos/-/merge_requests/12141; and + - https://gitlab.com/groups/gitlab-org/-/epics/6074 + + TODO: replace this with [FF_USE_NEW_BASH_EVAL_STRATEGY=true], see + {{:https://docs.gitlab.com/runner/configuration/feature-flags.html}GitLab + Runner feature flags}. *) +let script_propagate_exit_code script = [script ^ " || exit $?"] + let opt_var name f = function Some value -> [(name, f value)] | None -> [] (** Add variable for bisect_ppx instrumentation. -- GitLab From b8d7fadb8ec685ed7d838b668249c77b63479976 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Thu, 28 Mar 2024 16:12:38 +0100 Subject: [PATCH 2/9] CIAO: add on [changes:] manual rules in [code_verification] --- ci/bin/code_verification.ml | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index a19d2575df39..cc85092930e7 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -39,6 +39,12 @@ open Common unconditional variant [schedule_extended_test]. *) type code_verification_pipeline = Before_merging | Schedule_extended_test +(** Manual trigger configuration for [make_rules] *) +type manual = + | No (** Do not add rule for manual trigger. *) + | Yes (** Add rule for manual trigger. *) + | On_changes of string list (** Add manual trigger on certain [changes:] *) + (* [make_rules] makes rules for jobs that are: - automatic in scheduled pipelines; - conditional in [before_merging] pipelines. @@ -47,19 +53,24 @@ type code_verification_pipeline = Before_merging | Schedule_extended_test set to [true] to ensure that we only run the job in case previous jobs succeeded (setting [when: on_success]). + If [label] is set, add rule that selects the job in + [Before_merging] pipelines for merge requests with the given + label. Rules for manual triggers can be configured using + [manual]. + If [label], [changes] and [manual] are omitted, then rules will enable the job [On_success] in the [before_merging] pipeline. This is safe, but prefer specifying a [changes] clause if possible. *) -let make_rules ?label ?changes ?(manual = false) ?(dependent = false) - pipeline_type = +let make_rules ?label ?changes ?(manual = No) ?(dependent = false) pipeline_type + = match pipeline_type with | Schedule_extended_test -> (* The scheduled pipeline always runs all jobs unconditionally -- unless they are dependent on a previous, non-trigger job, in the pipeline. *) [job_rule ~when_:(if dependent then On_success else Always) ()] - | Before_merging -> + | Before_merging -> ( (* MR labels can be used to force tests to run. *) (match label with | Some label -> @@ -70,7 +81,11 @@ let make_rules ?label ?changes ?(manual = false) ?(dependent = false) | None -> [] | Some changes -> [job_rule ~changes ~when_:On_success ()]) (* For some tests, it can be relevant to have a manual trigger. *) - @ if manual then [job_rule ~when_:Manual ()] else [] + @ + match manual with + | No -> [] + | Yes -> [job_rule ~when_:Manual ()] + | On_changes changes -> [job_rule ~when_:Manual ~changes ()]) type opam_package_group = Executable | All @@ -496,7 +511,7 @@ let jobs pipeline_type = let job_docker_rust_toolchain = job_docker_rust_toolchain ~__POS__ - ~rules:(make_rules ~changes:changeset_octez_or_kernels ~manual:true ()) + ~rules:(make_rules ~changes:changeset_octez_or_kernels ~manual:Yes ()) ~dependencies:dependencies_needs_trigger () |> job_external_split @@ -545,7 +560,7 @@ let jobs pipeline_type = () |> job_external_split in - let build_arm_rules = make_rules ~label:"ci--arm64" ~manual:true () in + let build_arm_rules = make_rules ~label:"ci--arm64" ~manual:Yes () in let job_build_arm64_release : Tezos_ci.tezos_job = job_build_arm64_release ~rules:build_arm_rules () |> job_external_split in @@ -1012,7 +1027,7 @@ let jobs pipeline_type = ["docs/introduction/install*.sh"; "docs/introduction/compile*.sh"] in let install_octez_rules = - make_rules ~changes:changeset_install_jobs ~manual:true () + make_rules ~changes:changeset_install_jobs ~manual:Yes () in let job_install_bin ~__POS__ ~name ?allow_failure ?(rc = false) distribution = @@ -1041,7 +1056,7 @@ let jobs pipeline_type = ~name:"oc.install_opam_focal" ~image:Images.opam_ubuntu_focal ~dependencies:dependencies_needs_trigger - ~rules:(make_rules ~manual:true ()) + ~rules:(make_rules ~manual:Yes ()) ~allow_failure:Yes ~stage:Stages.test (* The default behavior of opam is to use `nproc` to determine its level of -- GitLab From d2860e3fb8185d77ff1e26398c760c0fcfefb3ab Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Thu, 28 Mar 2024 16:23:06 +0100 Subject: [PATCH 3/9] CIAO: extract tezt job dependencies from [build] stage We have lift these jobs from the definition of the [build] stage so that jobs in the [test] stage can depend on them. --- ci/bin/code_verification.ml | 223 ++++++++++++++++++------------------ 1 file changed, 111 insertions(+), 112 deletions(-) diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index cc85092930e7..8301cdb6d5a4 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -568,20 +568,118 @@ let jobs pipeline_type = job_build_arm64_exp_dev_extra ~rules:build_arm_rules () |> job_external_split in + (* Used in [before_merging] and [schedule_extended_tests]. + + Fetch records for Tezt generated on the last merge request pipeline + on the most recently merged MR and makes them available in artifacts + for future merge request pipelines. *) + let job_select_tezts : tezos_job = + job + ~__POS__ + ~name:"select_tezts" + (* We need: + - Git (to run git diff) + - ocamlyacc, ocamllex and ocamlc (to build manifest/manifest) *) + ~image:Images.runtime_prebuild_dependencies + ~stage:Stages.build + ~before_script:(before_script ~take_ownership:true ~eval_opam:true []) + (script_propagate_exit_code "scripts/ci/select_tezts.sh") + ~allow_failure:(With_exit_codes [17]) + ~artifacts: + (artifacts + ~expire_in:(Duration (Days 3)) + ~when_:Always + ["selected_tezts.tsl"]) + |> job_external_once + in + let job_build_kernels : tezos_job = + job + ~__POS__ + ~name:"oc.build_kernels" + ~image:Images.rust_toolchain + ~stage:Stages.build + ~dependencies:(Dependent [Artifacts job_docker_rust_toolchain]) + ~rules:(make_rules ~changes:changeset_octez_or_kernels ~dependent:true ()) + [ + "make -f kernels.mk build"; + "make -f etherlink.mk evm_kernel.wasm"; + "make -C src/risc_v risc-v-sandbox risc-v-dummy.elf"; + "make -C src/risc_v/tests/ build"; + ] + ~artifacts: + (artifacts + ~name:"build-kernels-$CI_COMMIT_REF_SLUG" + ~expire_in:(Duration (Days 1)) + ~when_:On_success + [ + "evm_kernel.wasm"; + "smart-rollup-installer"; + "sequenced_kernel.wasm"; + "tx_kernel.wasm"; + "tx_kernel_dal.wasm"; + "dal_echo_kernel.wasm"; + "src/risc_v/risc-v-sandbox"; + "src/risc_v/risc-v-dummy.elf"; + "src/risc_v/tests/inline_asm/rv64-inline-asm-tests"; + ]) + ~cache: + [ + {key = "kernels"; paths = ["cargo/"]}; + {key = "kernels-sccache"; paths = ["_sccache"]}; + ] + |> enable_kernels |> enable_sccache |> job_external_split + in + (* Fetch records for Tezt generated on the last merge request pipeline + on the most recently merged MR and makes them available in artifacts + for future merge request pipelines. *) + let job_tezt_fetch_records : tezos_job = + job + ~__POS__ + ~name:"oc.tezt:fetch-records" + ~image:Images.runtime_build_dependencies + ~stage:Stages.build + ~before_script: + (before_script + ~take_ownership:true + ~source_version:true + ~eval_opam:true + []) + ~rules:(make_rules ~changes:changeset_octez ()) + [ + "dune exec scripts/ci/update_records/update.exe -- --log-file \ + tezt-fetch-records.log --from last-successful-schedule-extended-test \ + --info"; + ] + ~after_script:["./scripts/ci/filter_corrupted_records.sh"] + (* Allow failure of this job, since Tezt can use the records + stored in the repo as backup for balancing. *) + ~allow_failure:Yes + ~artifacts: + (artifacts + ~expire_in:(Duration (Hours 4)) + ~when_:Always + [ + "tezt-fetch-records.log"; + "tezt/records/*.json"; + (* Keep broken records for debugging *) + "tezt/records/*.json.broken"; + ]) + |> job_external_split + in + let job_static_x86_64_experimental = + job_build_static_binaries + ~__POS__ + ~arch:Amd64 + (* Even though not many tests depend on static executables, some + of those that do are limiting factors in the total duration + of pipelines. So we start this job as early as possible, + without waiting for sanity_ci. *) + ~dependencies:dependencies_needs_trigger + ~rules:(make_rules ~changes:changeset_octez ()) + () + |> job_external_split + in let build = - let job_static_x86_64_experimental = - job_build_static_binaries - ~__POS__ - ~arch:Amd64 - (* Even though not many tests depend on static executables, some - of those that do are limiting factors in the total duration - of pipelines. So we start this job as early as possible, - without waiting for sanity_ci. *) - ~dependencies:dependencies_needs_trigger - ~rules:(make_rules ~changes:changeset_octez ()) - () - |> job_external_split - in (* TODO: The code is a bit convulted here because these jobs are either in the build or in the manual stage depending on the pipeline type. However, we can put them in the build stage on @@ -612,105 +710,6 @@ let jobs pipeline_type = ["dune build @check"] |> job_external_split in - let job_build_kernels : tezos_job = - job - ~__POS__ - ~name:"oc.build_kernels" - ~image:Images.rust_toolchain - ~stage:Stages.build - ~dependencies:(Dependent [Artifacts job_docker_rust_toolchain]) - ~rules: - (make_rules ~changes:changeset_octez_or_kernels ~dependent:true ()) - [ - "make -f kernels.mk build"; - "make -f etherlink.mk evm_kernel.wasm"; - "make -C src/risc_v risc-v-sandbox risc-v-dummy.elf"; - "make -C src/risc_v/tests/ build"; - ] - ~artifacts: - (artifacts - ~name:"build-kernels-$CI_COMMIT_REF_SLUG" - ~expire_in:(Duration (Days 1)) - ~when_:On_success - [ - "evm_kernel.wasm"; - "smart-rollup-installer"; - "sequenced_kernel.wasm"; - "tx_kernel.wasm"; - "tx_kernel_dal.wasm"; - "dal_echo_kernel.wasm"; - "src/risc_v/risc-v-sandbox"; - "src/risc_v/risc-v-dummy.elf"; - "src/risc_v/tests/inline_asm/rv64-inline-asm-tests"; - ]) - ~cache: - [ - {key = "kernels"; paths = ["cargo/"]}; - {key = "kernels-sccache"; paths = ["_sccache"]}; - ] - |> enable_kernels |> enable_sccache |> job_external_split - in - (* Fetch records for Tezt generated on the last merge request pipeline - on the most recently merged MR and makes them available in artifacts - for future merge request pipelines. *) - let job_tezt_fetch_records : tezos_job = - job - ~__POS__ - ~name:"oc.tezt:fetch-records" - ~image:Images.runtime_build_dependencies - ~stage:Stages.build - ~before_script: - (before_script - ~take_ownership:true - ~source_version:true - ~eval_opam:true - []) - ~rules:(make_rules ~changes:changeset_octez ()) - [ - "dune exec scripts/ci/update_records/update.exe -- --log-file \ - tezt-fetch-records.log --from \ - last-successful-schedule-extended-test --info"; - ] - ~after_script:["./scripts/ci/filter_corrupted_records.sh"] - (* Allow failure of this job, since Tezt can use the records - stored in the repo as backup for balancing. *) - ~allow_failure:Yes - ~artifacts: - (artifacts - ~expire_in:(Duration (Hours 4)) - ~when_:Always - [ - "tezt-fetch-records.log"; - "tezt/records/*.json"; - (* Keep broken records for debugging *) - "tezt/records/*.json.broken"; - ]) - |> job_external_split - in - (* Used in [before_merging] and [schedule_extended_tests]. - - Fetch records for Tezt generated on the last merge request pipeline - on the most recently merged MR and makes them available in artifacts - for future merge request pipelines. *) - let job_select_tezts : tezos_job = - job - ~__POS__ - ~name:"select_tezts" - (* We need: - - Git (to run git diff) - - ocamlyacc, ocamllex and ocamlc (to build manifest/manifest) *) - ~image:Images.runtime_prebuild_dependencies - ~stage:Stages.build - ~before_script:(before_script ~take_ownership:true ~eval_opam:true []) - (script_propagate_exit_code "scripts/ci/select_tezts.sh") - ~allow_failure:(With_exit_codes [17]) - ~artifacts: - (artifacts - ~expire_in:(Duration (Days 3)) - ~when_:Always - ["selected_tezts.tsl"]) - |> job_external_once - in [ job_docker_rust_toolchain; job_docker_client_libs_dependencies; -- GitLab From 53e3677db03d7dc6a9d3abb03c34917370a2b447 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Thu, 28 Mar 2024 16:24:51 +0100 Subject: [PATCH 4/9] CI: generate [tezt-flaky] --- .../jobs/test/tezt-flaky-before_merging.yml | 70 ++++++- .../tezt-flaky-schedule_extended_test.yml | 6 - .../tezt-flaky-scheduled_extended_test.yml | 58 ++++++ .gitlab/ci/jobs/test/tezt-flaky.yml | 15 -- .../ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/code_verification.ml | 172 ++++++++++++++++++ ci/bin/dune | 3 +- ci/bin/main.ml | 3 - manifest/product_ciao.ml | 2 +- 9 files changed, 299 insertions(+), 32 deletions(-) delete mode 100644 .gitlab/ci/jobs/test/tezt-flaky-schedule_extended_test.yml create mode 100644 .gitlab/ci/jobs/test/tezt-flaky-scheduled_extended_test.yml delete mode 100644 .gitlab/ci/jobs/test/tezt-flaky.yml diff --git a/.gitlab/ci/jobs/test/tezt-flaky-before_merging.yml b/.gitlab/ci/jobs/test/tezt-flaky-before_merging.yml index 9a48f9e67090..f37489ea2192 100644 --- a/.gitlab/ci/jobs/test/tezt-flaky-before_merging.yml +++ b/.gitlab/ci/jobs/test/tezt-flaky-before_merging.yml @@ -1,7 +1,67 @@ -include: .gitlab/ci/jobs/test/tezt-flaky.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. -# Runs flaky Tezts in the 'before_merging' pipeline. tezt-flaky: - extends: - - .tezt-flaky - - .rules__octez_changes_manual + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: manual + allow_failure: true + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} + --retry ${TEZT_RETRY:-1} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: -flaky + TESTS: /ci_disabled && /memory_3k && /memory_4k && /time_sensitive && /slow && + /cloud && flaky + TEZT_RETRY: "3" + TEZT_PARALLEL: "1" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE + reports: + junit: $JUNIT + when: always + retry: 2 diff --git a/.gitlab/ci/jobs/test/tezt-flaky-schedule_extended_test.yml b/.gitlab/ci/jobs/test/tezt-flaky-schedule_extended_test.yml deleted file mode 100644 index 59bbab4a4674..000000000000 --- a/.gitlab/ci/jobs/test/tezt-flaky-schedule_extended_test.yml +++ /dev/null @@ -1,6 +0,0 @@ -include: .gitlab/ci/jobs/test/tezt-flaky.yml - -# Runs flaky Tezts in the 'schedule_extended_test' pipeline. -tezt-flaky: - extends: - - .tezt-flaky diff --git a/.gitlab/ci/jobs/test/tezt-flaky-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/tezt-flaky-scheduled_extended_test.yml new file mode 100644 index 000000000000..f9841c0f17c3 --- /dev/null +++ b/.gitlab/ci/jobs/test/tezt-flaky-scheduled_extended_test.yml @@ -0,0 +1,58 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +tezt-flaky: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - when: on_success + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} + --retry ${TEZT_RETRY:-1} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: -flaky + TESTS: /ci_disabled && /memory_3k && /memory_4k && /time_sensitive && /slow && + /cloud && flaky + TEZT_RETRY: "3" + TEZT_PARALLEL: "1" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE + reports: + junit: $JUNIT + when: always + retry: 2 diff --git a/.gitlab/ci/jobs/test/tezt-flaky.yml b/.gitlab/ci/jobs/test/tezt-flaky.yml deleted file mode 100644 index cbf15b6fa748..000000000000 --- a/.gitlab/ci/jobs/test/tezt-flaky.yml +++ /dev/null @@ -1,15 +0,0 @@ -include: .gitlab/ci/jobs/test/common.yml - -# Runs flaky Tezts in the 'schedule_extended_test' pipeline. -.tezt-flaky: - extends: - - .tezt_tests - variables: - # Run only flaky tests - TESTS: "/ci_disabled /memory_3k /memory_4k /time_sensitive /slow /cloud flaky" - # To handle flakiness, consider tweaking TEZT_PARALLEL (passed to - # Tezt's '--job-count'), and TEZT_RETRY (passed to Tezt's - # '--retry') - TEZT_RETRY: 3 - TEZT_PARALLEL: 1 - TEZT_VARIANT: "-flaky" diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index d646d102053c..190121ef820d 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -32,7 +32,7 @@ include: # Stage: test - .gitlab/ci/jobs/test/install_octez-scheduled_extended_test.yml # Flaky tezts - - .gitlab/ci/jobs/test/tezt-flaky-schedule_extended_test.yml + - .gitlab/ci/jobs/test/tezt-flaky-scheduled_extended_test.yml # Slow tezts - .gitlab/ci/jobs/test/tezt-slow-schedule_extended_test.yml # Tests that may not have been run in before_merging pipeline diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 8301cdb6d5a4..96a7b2ea5c93 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -373,6 +373,146 @@ let image_of_distribution = function | Ubuntu_jammy -> Images.ubuntu_jammy | Fedora_37 -> Images.fedora_37 +let job_tezt ~__POS__ ?rules ?parallel ?(tags = ["gcp_tezt"]) ~name + ~(tezt_tests : Tezt_core.TSL_AST.t) ?(retry = 2) ?(tezt_retry = 1) + ?(tezt_parallel = 1) ?(tezt_variant = "") + ?(before_script = before_script ~source_version:true ~eval_opam:true []) + ~dependencies () : tezos_job = + let variables = + [ + ("JUNIT", "tezt-junit.xml"); + ("TEZT_VARIANT", tezt_variant); + ("TESTS", Tezt_core.TSL.show tezt_tests); + ("TEZT_RETRY", string_of_int tezt_retry); + ("TEZT_PARALLEL", string_of_int tezt_parallel); + ] + in + let artifacts = + artifacts + ~reports:(reports ~junit:"$JUNIT" ()) + [ + "selected_tezts.tsv"; + "tezt.log"; + "tezt-*.log"; + "tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json"; + "$JUNIT"; + ] + (* The record artifacts [tezt-results-$CI_NODE_INDEX.json] + should be stored for as long as a given commit on master is + expected to be HEAD in order to support auto-balancing. At + the time of writing, we have approximately 6 merges per day, + so 1 day should more than enough. However, we set it to 3 + days to keep records over the weekend. The tezt artifacts + (including records and coverage) take up roughly 2MB / + job. Total artifact storage becomes [N*P*T*W] where [N] is + the days of retention (3 atm), [P] the number of pipelines + per day (~200 atm), [T] the number of Tezt jobs per pipeline + (60) and [W] the artifact size per tezt job (2MB). This makes + 35GB which is less than 0.5% than our + {{:https://gitlab.com/tezos/tezos/-/artifacts}total artifact + usage}. *) + ~expire_in:(Duration (Days 7)) + ~when_:Always + in + let print_variables = + [ + "TESTS"; + "JUNIT"; + "CI_NODE_INDEX"; + "CI_NODE_TOTAL"; + "TEZT_PARALLEL"; + "TEZT_VARIANT"; + ] + in + let retry = if retry = 0 then None else Some retry in + job + ~__POS__ + ~image:Images.runtime_e2etest_dependencies + ~name + ?parallel + ~tags + ~stage:Stages.test + ?rules + ~artifacts + ~variables + ~dependencies + ?retry + ~before_script + [ + (* Print [print_variables] in a shell-friendly manner for easier debugging *) + "echo \"" + ^ String.concat + " " + (List.map (fun var -> sf {|%s=\"${%s}\"|} var var) print_variables) + ^ "\""; + (* Store the list of tests that have been scheduled for execution for later debugging. + It is imperative this this first call to tezt receives any flags passed to the + second call that affect test selection.Note that TESTS must be quoted (here and below) + since it will contain e.g. '&&' which we want to interpreted as TSL and not shell + syntax. *) + "./scripts/ci/tezt.sh \"${TESTS}\" --from-record tezt/records --job \ + ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --list-tsv > selected_tezts.tsv"; + (* For Tezt tests, there are multiple timeouts: + - --global-timeout is the internal timeout of Tezt, which only works if tests + are cooperative; + - the "timeout" command, which we set to send SIGTERM to Tezt 60s after --global-timeout + in case tests are not cooperative; + - the "timeout" command also sends SIGKILL 60s after having sent SIGTERM in case + Tezt is still stuck; + - the CI timeout. + The use of the "timeout" command is to make sure that Tezt eventually exits, + because if the CI timeout is reached, there are no artefacts, + and thus no logs to investigate. + See also: https://gitlab.com/gitlab-org/gitlab/-/issues/19818 *) + (* TODO: there is no point in providing default values for the + expansion of [TEZT_PARALLEL] and [TEZT_RETRY] as they are + always set in this job (see the definition of variables + above). Remove this once all tezt jobs are translated. *) + "./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh \ + \"${TESTS}\" --color --log-buffer-size 5000 --log-file tezt.log \ + --global-timeout 1800 --on-unknown-regression-files fail --junit \ + ${JUNIT} --from-record tezt/records --job \ + ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record \ + tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count \ + ${TEZT_PARALLEL:-3} --retry ${TEZT_RETRY:-1}"; + "if [ -n \"${BISECT_FILE:-}\" ]; then ./scripts/ci/merge_coverage.sh; fi"; + ] + +(** Tezt tag selector string. + + It returns a TSL expression that: + - always deselects tags with [ci_disabled]; + - selects, respectively deselects, the tests with the tags + [memory_3k], [memory_4k], [time_sensitive], [slow] or [cloud], + depending on the value of the corresponding function + argument. These arguments all default to false. + + See [src/lib_test/tag.mli] for a description of the above tags. + + The list of TSL expressions [and_] are appended to the final + selector, allowing to modify the selection further. *) +let tezt_tests ?(memory_3k = false) ?(memory_4k = false) + ?(time_sensitive = false) ?(slow = false) ?(cloud = false) + (and_ : Tezt_core.TSL_AST.t list) : Tezt_core.TSL_AST.t = + let tags = + [ + (false, "ci_disabled"); + (memory_3k, "memory_3k"); + (memory_4k, "memory_4k"); + (time_sensitive, "time_sensitive"); + (slow, "slow"); + (cloud, "cloud"); + ] + in + let positive, negative = List.partition fst tags in + let positive = List.map snd positive in + let negative = List.map snd negative in + Tezt_core.( + TSL.conjunction + @@ List.map (fun tag -> TSL_AST.Has_tag tag) positive + @ List.map (fun tag -> TSL_AST.Not (Has_tag tag)) negative + @ and_) + (* Encodes the conditional [before_merging] pipeline and its unconditional variant [schedule_extended_test]. *) let jobs pipeline_type = @@ -1133,6 +1273,37 @@ let jobs pipeline_type = ] |> jobs_external_split ~path:"test/install_octez" in + let tezt_dependencies = + Dependent + [ + Artifacts job_select_tezts; + Artifacts job_build_x86_64_release; + Artifacts job_build_x86_64_exp_dev_extra; + Artifacts job_build_kernels; + Artifacts job_tezt_fetch_records; + ] + in + let job_tezt_flaky : tezos_job = + job_tezt + ~__POS__ + ~name:"tezt-flaky" + ~tezt_tests:(tezt_tests [Has_tag "flaky"]) + ~tezt_variant:"-flaky" + (* To handle flakiness, consider tweaking [~tezt_parallel] (passed to + Tezt's '--job-count'), and [~tezt_retry] (passed to Tezt's + '--retry') *) + ~retry:2 + ~tezt_retry:3 + ~tezt_parallel:1 + ~dependencies:tezt_dependencies + ~rules: + (* This job can only be manually triggered when it's + artifact dependencies exists, which they do when + [changeset_octez] is changed. *) + (make_rules ~dependent:true ~manual:(On_changes changeset_octez) ()) + () + |> enable_coverage_output_artifact |> job_external_split + in [ job_kaitai_checks; job_kaitai_e2e_checks; @@ -1146,6 +1317,7 @@ let jobs pipeline_type = job_oc_script_test_release_versions; job_oc_script_b58_prefix; job_oc_test_liquidity_baking_scripts; + job_tezt_flaky; ] @ jobs_unit @ jobs_install_octez @ diff --git a/ci/bin/dune b/ci/bin/dune index 7438f8014a36..5063369192db 100644 --- a/ci/bin/dune +++ b/ci/bin/dune @@ -6,7 +6,8 @@ (libraries gitlab_ci yaml - unix) + unix + tezt.core) (flags (:standard) -open Gitlab_ci.Base)) diff --git a/ci/bin/main.ml b/ci/bin/main.ml index c1d817a1505e..46bf2c872e4b 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -199,9 +199,6 @@ let () = | ".gitlab/ci/jobs/test/test_kernels.yml" | ".gitlab/ci/jobs/test/test_risc_v_kernels-before_merging.yml" | ".gitlab/ci/jobs/test/test_risc_v_kernels-schedule_extended_test.yml" - | ".gitlab/ci/jobs/test/tezt-flaky-before_merging.yml" - | ".gitlab/ci/jobs/test/tezt-flaky-schedule_extended_test.yml" - | ".gitlab/ci/jobs/test/tezt-flaky.yml" | ".gitlab/ci/jobs/test/tezt-slow-before_merging.yml" | ".gitlab/ci/jobs/test/tezt-slow-schedule_extended_test.yml" | ".gitlab/ci/jobs/test/tezt-slow.yml" | ".gitlab/ci/jobs/test/tezt.yml" diff --git a/manifest/product_ciao.ml b/manifest/product_ciao.ml index 17099f0d09be..86c35968e963 100644 --- a/manifest/product_ciao.ml +++ b/manifest/product_ciao.ml @@ -30,5 +30,5 @@ let _ci_bin_main = ~opam:"" ~path:"ci/bin" ~bisect_ppx:No - ~deps:[ci_lib_gitlab_ci_main |> open_ ~m:"Base"; yaml; unix] + ~deps:[ci_lib_gitlab_ci_main |> open_ ~m:"Base"; yaml; unix; tezt_core_lib] ~release_status:Unreleased -- GitLab From 1887ecea4f955e50bdf91ed6fd63322afecbcffd Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 3 Apr 2024 15:26:50 +0200 Subject: [PATCH 5/9] CI: generate [tezt-slow] --- .../ci/jobs/test/tezt-slow-before_merging.yml | 70 +++++++++++++++++-- .../test/tezt-slow-schedule_extended_test.yml | 5 -- .../tezt-slow-scheduled_extended_test.yml | 59 ++++++++++++++++ .gitlab/ci/jobs/test/tezt-slow.yml | 17 ----- .../ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/code_verification.ml | 31 ++++++++ ci/bin/main.ml | 4 +- 7 files changed, 158 insertions(+), 30 deletions(-) delete mode 100644 .gitlab/ci/jobs/test/tezt-slow-schedule_extended_test.yml create mode 100644 .gitlab/ci/jobs/test/tezt-slow-scheduled_extended_test.yml delete mode 100644 .gitlab/ci/jobs/test/tezt-slow.yml diff --git a/.gitlab/ci/jobs/test/tezt-slow-before_merging.yml b/.gitlab/ci/jobs/test/tezt-slow-before_merging.yml index 4e9695004a1c..fbece5516a77 100644 --- a/.gitlab/ci/jobs/test/tezt-slow-before_merging.yml +++ b/.gitlab/ci/jobs/test/tezt-slow-before_merging.yml @@ -1,6 +1,68 @@ -include: .gitlab/ci/jobs/test/tezt-slow.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. tezt-slow: - extends: - - .tezt-slow - - .rules__octez_changes_manual + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: manual + allow_failure: true + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} + --retry ${TEZT_RETRY:-1} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: -slow + TESTS: slow && /ci_disabled && /memory_3k && /memory_4k && /time_sensitive && + /cloud && not (file = src/proto_019_PtParisA/lib_protocol/test/integration/test_adaptive_issuance_launch.ml) + TEZT_RETRY: "1" + TEZT_PARALLEL: "3" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE + reports: + junit: $JUNIT + when: always + retry: 2 + parallel: 10 diff --git a/.gitlab/ci/jobs/test/tezt-slow-schedule_extended_test.yml b/.gitlab/ci/jobs/test/tezt-slow-schedule_extended_test.yml deleted file mode 100644 index 26b845b05596..000000000000 --- a/.gitlab/ci/jobs/test/tezt-slow-schedule_extended_test.yml +++ /dev/null @@ -1,5 +0,0 @@ -include: .gitlab/ci/jobs/test/tezt-slow.yml - -tezt-slow: - extends: - - .tezt-slow diff --git a/.gitlab/ci/jobs/test/tezt-slow-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/tezt-slow-scheduled_extended_test.yml new file mode 100644 index 000000000000..705367aac22d --- /dev/null +++ b/.gitlab/ci/jobs/test/tezt-slow-scheduled_extended_test.yml @@ -0,0 +1,59 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +tezt-slow: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - when: on_success + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} + --retry ${TEZT_RETRY:-1} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: -slow + TESTS: slow && /ci_disabled && /memory_3k && /memory_4k && /time_sensitive && + /cloud && not (file = src/proto_019_PtParisA/lib_protocol/test/integration/test_adaptive_issuance_launch.ml) + TEZT_RETRY: "1" + TEZT_PARALLEL: "3" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE + reports: + junit: $JUNIT + when: always + retry: 2 + parallel: 10 diff --git a/.gitlab/ci/jobs/test/tezt-slow.yml b/.gitlab/ci/jobs/test/tezt-slow.yml deleted file mode 100644 index fc19a9656f1d..000000000000 --- a/.gitlab/ci/jobs/test/tezt-slow.yml +++ /dev/null @@ -1,17 +0,0 @@ -include: .gitlab/ci/jobs/test/common.yml - -# Runs Tezts tagged as 'slow' in the 'schedule_extended_test' -# pipeline. -.tezt-slow: - extends: - - .tezt_tests - variables: - # Run only slow tests - # TODO: https://gitlab.com/tezos/tezos/-/issues/7063 - # The deselection of Paris [test_adaptive_issuance_launch.ml] - # should be removed once the fixes to its slowness has been - # snapshotted from Alpha. - TESTS: "/ci_disabled /cloud slow --not-file src/proto_019_PtParisA/lib_protocol/test/integration/test_adaptive_issuance_launch.ml" - TEZT_PARALLEL: 3 - TEZT_VARIANT: "-slow" - parallel: 10 diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 190121ef820d..5821f0c94286 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -34,7 +34,7 @@ include: # Flaky tezts - .gitlab/ci/jobs/test/tezt-flaky-scheduled_extended_test.yml # Slow tezts - - .gitlab/ci/jobs/test/tezt-slow-schedule_extended_test.yml + - .gitlab/ci/jobs/test/tezt-slow-scheduled_extended_test.yml # Tests that may not have been run in before_merging pipeline # because of absence of certain changes - .gitlab/ci/jobs/test/kaitai_checks-scheduled_extended_test.yml diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 96a7b2ea5c93..02fc860754ce 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -1304,6 +1304,36 @@ let jobs pipeline_type = () |> enable_coverage_output_artifact |> job_external_split in + let job_tezt_slow : tezos_job = + job_tezt + ~__POS__ + ~name:"tezt-slow" + ~rules: + (* See comment for [job_tezt_flaky] *) + (make_rules ~dependent:true ~manual:(On_changes changeset_octez) ()) + ~tezt_tests: + (tezt_tests + ~slow:true + (* TODO: https://gitlab.com/tezos/tezos/-/issues/7063 + The deselection of Paris [test_adaptive_issuance_launch.ml] + should be removed once the fixes to its slowness has been + snapshotted from Alpha. *) + [ + Not + (String_predicate + ( File, + Is + "src/proto_019_PtParisA/lib_protocol/test/integration/test_adaptive_issuance_launch.ml" + )); + ]) + ~tezt_variant:"-slow" + ~retry:2 + ~tezt_parallel:3 + ~parallel:(Vector 10) + ~dependencies:tezt_dependencies + () + |> enable_coverage_output_artifact |> job_external_split + in [ job_kaitai_checks; job_kaitai_e2e_checks; @@ -1318,6 +1348,7 @@ let jobs pipeline_type = job_oc_script_b58_prefix; job_oc_test_liquidity_baking_scripts; job_tezt_flaky; + job_tezt_slow; ] @ jobs_unit @ jobs_install_octez @ diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 46bf2c872e4b..646dbd16f553 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -199,9 +199,7 @@ let () = | ".gitlab/ci/jobs/test/test_kernels.yml" | ".gitlab/ci/jobs/test/test_risc_v_kernels-before_merging.yml" | ".gitlab/ci/jobs/test/test_risc_v_kernels-schedule_extended_test.yml" - | ".gitlab/ci/jobs/test/tezt-slow-before_merging.yml" - | ".gitlab/ci/jobs/test/tezt-slow-schedule_extended_test.yml" - | ".gitlab/ci/jobs/test/tezt-slow.yml" | ".gitlab/ci/jobs/test/tezt.yml" + | ".gitlab/ci/jobs/test/tezt.yml" | ".gitlab/ci/pipelines/before_merging.yml" | ".gitlab/ci/pipelines/schedule_extended_test.yml" -> true -- GitLab From f83e0e429782851a1e18624413dea57359308140 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 27 Mar 2024 16:14:00 +0100 Subject: [PATCH 6/9] CI: remove coverage output artifacts from [tezt-slow] We do not run this job by default on [before_merging] pipelines, and we do not collect produce coverage reports on [scheduled_extended_test] pipelines. Therefore, there is no point in collecting coverage artifacts from this job. --- .gitlab/ci/jobs/test/tezt-slow-before_merging.yml | 3 --- .gitlab/ci/jobs/test/tezt-slow-scheduled_extended_test.yml | 3 --- ci/bin/code_verification.ml | 2 +- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.gitlab/ci/jobs/test/tezt-slow-before_merging.yml b/.gitlab/ci/jobs/test/tezt-slow-before_merging.yml index fbece5516a77..627add63250f 100644 --- a/.gitlab/ci/jobs/test/tezt-slow-before_merging.yml +++ b/.gitlab/ci/jobs/test/tezt-slow-before_merging.yml @@ -50,9 +50,7 @@ tezt-slow: /cloud && not (file = src/proto_019_PtParisA/lib_protocol/test/integration/test_adaptive_issuance_launch.ml) TEZT_RETRY: "1" TEZT_PARALLEL: "3" - BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ artifacts: - name: coverage-files-$CI_JOB_ID expire_in: 7 days paths: - selected_tezts.tsv @@ -60,7 +58,6 @@ tezt-slow: - tezt-*.log - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json - $JUNIT - - $BISECT_FILE reports: junit: $JUNIT when: always diff --git a/.gitlab/ci/jobs/test/tezt-slow-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/tezt-slow-scheduled_extended_test.yml index 705367aac22d..e9292f9998d6 100644 --- a/.gitlab/ci/jobs/test/tezt-slow-scheduled_extended_test.yml +++ b/.gitlab/ci/jobs/test/tezt-slow-scheduled_extended_test.yml @@ -41,9 +41,7 @@ tezt-slow: /cloud && not (file = src/proto_019_PtParisA/lib_protocol/test/integration/test_adaptive_issuance_launch.ml) TEZT_RETRY: "1" TEZT_PARALLEL: "3" - BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ artifacts: - name: coverage-files-$CI_JOB_ID expire_in: 7 days paths: - selected_tezts.tsv @@ -51,7 +49,6 @@ tezt-slow: - tezt-*.log - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json - $JUNIT - - $BISECT_FILE reports: junit: $JUNIT when: always diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 02fc860754ce..f614925cd2e1 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -1332,7 +1332,7 @@ let jobs pipeline_type = ~parallel:(Vector 10) ~dependencies:tezt_dependencies () - |> enable_coverage_output_artifact |> job_external_split + |> job_external_split in [ job_kaitai_checks; -- GitLab From 1f98de454879b31eeab30803e8fec73f7c127518 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Thu, 4 Apr 2024 12:20:40 +0200 Subject: [PATCH 7/9] CI: generate [tezt.yml] and remove [test/common.yml] The latter file is no longer needed by any job. --- .gitlab/ci/jobs/test/common.yml | 121 ------- .gitlab/ci/jobs/test/tezt-before_merging.yml | 317 ++++++++++++++++++ .../test/tezt-scheduled_extended_test.yml | 277 +++++++++++++++ .gitlab/ci/jobs/test/tezt.yml | 89 ----- .gitlab/ci/pipelines/before_merging.yml | 2 +- .../ci/pipelines/schedule_extended_test.yml | 2 +- ci/bin/code_verification.ml | 87 ++++- ci/bin/main.ml | 1 - 8 files changed, 682 insertions(+), 214 deletions(-) delete mode 100644 .gitlab/ci/jobs/test/common.yml create mode 100644 .gitlab/ci/jobs/test/tezt-before_merging.yml create mode 100644 .gitlab/ci/jobs/test/tezt-scheduled_extended_test.yml delete mode 100644 .gitlab/ci/jobs/test/tezt.yml diff --git a/.gitlab/ci/jobs/test/common.yml b/.gitlab/ci/jobs/test/common.yml deleted file mode 100644 index b66ce24785ee..000000000000 --- a/.gitlab/ci/jobs/test/common.yml +++ /dev/null @@ -1,121 +0,0 @@ -.test_template: - extends: - - .default_settings_template - - .image_template__runtime_build_dependencies - - .rules__octez_changes - needs: - - "oc.build_x86_64-released" - - "oc.build_x86_64-exp-dev-extra" - stage: test - before_script: - - . ./scripts/version.sh - - eval $(opam env) - -# We use the --job option to split tests into jobs of roughly the same -# duration. This is based on a file that contains timings of test results, -# generated with --record. To rebalance jobs, update this record with: -# -# make && dune exec tezt/tests/main.exe -- --record tezt/test-results.json - -.tezt_template: - variables: - JUNIT: "tezt-junit.xml" - TEZT_VARIANT: "" - # The variable TESTS controls test selection. It passed directly - # to tezt so can contain tags, their negation, or an arbitrary - # TSL expression. - TESTS: "" - artifacts: - reports: - junit: $JUNIT - paths: - - selected_tezts.tsv - - tezt.log - - tezt-*.log - - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json - - $JUNIT - # Record artifacts are fetched from the scheduled pipeline. - # If this pipeline is broken on a Friday and only gets fixed on Monday, - # the next pipeline runs on Tuesday, so artifacts are unavailable - # for 4 days. We thus set the expiration to 7 days for safety. - # It also allows to make statistics on MRs that are less than 1 week old. - # The tezt artifacts (including records and coverage) - # take up roughly 2MB / job. Total artifact storage - # becomes N*P*T*W where N is the days of retention (7 atm), P the - # number of pipelines per day (~200 atm), T the number of Tezt - # jobs per pipeline (60) and W the artifact size per tezt job - # (2MB). This makes 82GB which is less than 1.5% of our total - # artifact usage (https://gitlab.com/tezos/tezos/-/artifacts). - expire_in: 7 days - when: always - script: - # Print variables in a shell-friendly format. - - 'echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\""' - # Store the list of tests that have been scheduled for execution for later debugging. - # It is imperative this this first call to tezt receives any flags passed to the - # second call that affect test selection. - - ./scripts/ci/tezt.sh - ${TESTS} - --from-record tezt/records - --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --list-tsv > selected_tezts.tsv - # For Tezt tests, there are multiple timeouts: - # - --global-timeout is the internal timeout of Tezt, which only works if tests - # are cooperative; - # - the "timeout" command, which we set to send SIGTERM to Tezt 60s after --global-timeout - # in case tests are not cooperative; - # - the "timeout" command also sends SIGKILL 60s after having sent SIGTERM in case - # Tezt is still stuck; - # - the CI timeout. - # The use of the "timeout" command is to make sure that Tezt eventually exits, - # because if the CI timeout is reached, there are no artefacts, - # and thus no logs to investigate. - # See also: https://gitlab.com/gitlab-org/gitlab/-/issues/19818 - - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh - ${TESTS} - --color - --log-buffer-size 5000 - --log-file tezt.log - --global-timeout 1800 - --on-unknown-regression-files fail - --junit ${JUNIT} - --from-record tezt/records - --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json - --job-count ${TEZT_PARALLEL:-3} - --retry ${TEZT_RETRY:-1} - # Only merge coverage traces if coverage output is enabled. - - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi - -# Common template for tezt jobs that use x86_64 binaries and produce -# coverage. -.tezt_tests: - extends: - - .test_template - - .image_template__runtime_e2etest_dependencies - - .oc.template__coverage_output_artifact - - .tezt_template - - .oc.template__retry_flaky - dependencies: - - "select_tezts" - - "oc.build_x86_64-released" - - "oc.build_x86_64-exp-dev-extra" - - "oc.build_kernels" - - "oc.tezt:fetch-records" - needs: - - "select_tezts" - - "oc.build_x86_64-released" - - "oc.build_x86_64-exp-dev-extra" - - "oc.build_kernels" - - "oc.tezt:fetch-records" - tags: - - gcp_tezt - # Add [BISECT_FILE] to the set of artifacts defined in [.tezt_template] - artifacts: - paths: - - selected_tezts.tsv - - tezt.log - - tezt-*.log - - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json - - $JUNIT - - $BISECT_FILE diff --git a/.gitlab/ci/jobs/test/tezt-before_merging.yml b/.gitlab/ci/jobs/test/tezt-before_merging.yml new file mode 100644 index 000000000000..8ae10aaef30b --- /dev/null +++ b/.gitlab/ci/jobs/test/tezt-before_merging.yml @@ -0,0 +1,317 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +tezt: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} + --retry ${TEZT_RETRY:-1} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: "" + TESTS: /ci_disabled && /memory_3k && /memory_4k && /time_sensitive && /slow && + /cloud && /flaky + TEZT_RETRY: "1" + TEZT_PARALLEL: "3" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE + reports: + junit: $JUNIT + when: always + retry: 2 + parallel: 60 + +tezt-memory-4k: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} + --retry ${TEZT_RETRY:-1} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: -memory_4k + TESTS: memory_4k && /ci_disabled && /memory_3k && /time_sensitive && /slow && + /cloud + TEZT_RETRY: "1" + TEZT_PARALLEL: "1" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE + reports: + junit: $JUNIT + when: always + retry: 2 + parallel: 4 + +tezt-memory-3k: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} + --retry ${TEZT_RETRY:-1} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: -memory_3k + TESTS: memory_3k && /ci_disabled && /memory_4k && /time_sensitive && /slow && + /cloud + TEZT_RETRY: "1" + TEZT_PARALLEL: "1" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE + reports: + junit: $JUNIT + when: always + retry: 2 + +tezt-time-sensitive: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} + --retry ${TEZT_RETRY:-1} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: -time_sensitive + TESTS: time_sensitive && /ci_disabled && /memory_3k && /memory_4k && /slow && + /cloud + TEZT_RETRY: "1" + TEZT_PARALLEL: "1" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE + reports: + junit: $JUNIT + when: always + retry: 2 + +tezt:static-binaries: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - changes: + - src/**/* + - etherlink/**/* + - tezt/**/* + - .gitlab/**/* + - .gitlab-ci.yml + - michelson_test_scripts/**/* + - tzt_reference_test_suite/**/* + when: on_success + needs: + - select_tezts + - oc.build_x86_64-exp-dev-extra + - oc.build:static-x86_64-linux-binaries + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-exp-dev-extra + - oc.build:static-x86_64-linux-binaries + - oc.tezt:fetch-records + before_script: + - mv octez-binaries/x86_64/octez-* . + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} + --retry ${TEZT_RETRY:-1} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: "" + TESTS: /ci_disabled && /memory_3k && /memory_4k && /time_sensitive && /slow && + /cloud && cli && /flaky + TEZT_RETRY: "1" + TEZT_PARALLEL: "3" + artifacts: + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + reports: + junit: $JUNIT + when: always diff --git a/.gitlab/ci/jobs/test/tezt-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/tezt-scheduled_extended_test.yml new file mode 100644 index 000000000000..48b95fb9c369 --- /dev/null +++ b/.gitlab/ci/jobs/test/tezt-scheduled_extended_test.yml @@ -0,0 +1,277 @@ +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +tezt: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - when: on_success + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} + --retry ${TEZT_RETRY:-1} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: "" + TESTS: /ci_disabled && /memory_3k && /memory_4k && /time_sensitive && /slow && + /cloud && /flaky + TEZT_RETRY: "1" + TEZT_PARALLEL: "3" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE + reports: + junit: $JUNIT + when: always + retry: 2 + parallel: 60 + +tezt-memory-4k: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - when: on_success + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} + --retry ${TEZT_RETRY:-1} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: -memory_4k + TESTS: memory_4k && /ci_disabled && /memory_3k && /time_sensitive && /slow && + /cloud + TEZT_RETRY: "1" + TEZT_PARALLEL: "1" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE + reports: + junit: $JUNIT + when: always + retry: 2 + parallel: 4 + +tezt-memory-3k: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - when: on_success + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} + --retry ${TEZT_RETRY:-1} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: -memory_3k + TESTS: memory_3k && /ci_disabled && /memory_4k && /time_sensitive && /slow && + /cloud + TEZT_RETRY: "1" + TEZT_PARALLEL: "1" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE + reports: + junit: $JUNIT + when: always + retry: 2 + +tezt-time-sensitive: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp_tezt + rules: + - when: on_success + needs: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-released + - oc.build_x86_64-exp-dev-extra + - oc.build_kernels + - oc.tezt:fetch-records + before_script: + - . ./scripts/version.sh + - eval $(opam env) + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} + --retry ${TEZT_RETRY:-1} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: -time_sensitive + TESTS: time_sensitive && /ci_disabled && /memory_3k && /memory_4k && /slow && + /cloud + TEZT_RETRY: "1" + TEZT_PARALLEL: "1" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + artifacts: + name: coverage-files-$CI_JOB_ID + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + - $BISECT_FILE + reports: + junit: $JUNIT + when: always + retry: 2 + +tezt:static-binaries: + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test + tags: + - gcp + rules: + - when: on_success + needs: + - select_tezts + - oc.build_x86_64-exp-dev-extra + - oc.build:static-x86_64-linux-binaries + - oc.tezt:fetch-records + dependencies: + - select_tezts + - oc.build_x86_64-exp-dev-extra + - oc.build:static-x86_64-linux-binaries + - oc.tezt:fetch-records + before_script: + - mv octez-binaries/x86_64/octez-* . + script: + - echo "TESTS=\"${TESTS}\" JUNIT=\"${JUNIT}\" CI_NODE_INDEX=\"${CI_NODE_INDEX}\" + CI_NODE_TOTAL=\"${CI_NODE_TOTAL}\" TEZT_PARALLEL=\"${TEZT_PARALLEL}\" TEZT_VARIANT=\"${TEZT_VARIANT}\"" + - ./scripts/ci/tezt.sh "${TESTS}" --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --list-tsv > selected_tezts.tsv + - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color + --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files + fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} + --retry ${TEZT_RETRY:-1} + - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi + variables: + JUNIT: tezt-junit.xml + TEZT_VARIANT: "" + TESTS: /ci_disabled && /memory_3k && /memory_4k && /time_sensitive && /slow && + /cloud && cli && /flaky + TEZT_RETRY: "1" + TEZT_PARALLEL: "3" + artifacts: + expire_in: 7 days + paths: + - selected_tezts.tsv + - tezt.log + - tezt-*.log + - tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json + - $JUNIT + reports: + junit: $JUNIT + when: always diff --git a/.gitlab/ci/jobs/test/tezt.yml b/.gitlab/ci/jobs/test/tezt.yml deleted file mode 100644 index 155b83c70e84..000000000000 --- a/.gitlab/ci/jobs/test/tezt.yml +++ /dev/null @@ -1,89 +0,0 @@ -include: .gitlab/ci/jobs/test/common.yml - -tezt: - extends: - - .tezt_tests - variables: - # Exclude tests with tags 'ci_disabled', 'flaky', 'memory_{3,4}k', - # 'time_sensitive' and 'slow'. - # See tezt/lib_tezos/tag.mli for more information. - TESTS: "/ci_disabled /flaky /memory_3k /memory_4k /time_sensitive /slow /cloud" - # the -j option of tezt - TEZT_PARALLEL: 3 - # WARNING: if you increase the number of parallel jobs, you need to - # update .gitlab/ci/jobs/coverage/oc.unified_coverage-before_merging.yml with the - # new list of jobs. - parallel: 60 - -# the following memory hungry tests are executed with -j 1 -tezt-memory-4k: - extends: [.tezt_tests] - variables: - TESTS: "/ci_disabled /cloud memory_4k" - TEZT_PARALLEL: 1 - TEZT_VARIANT: "-memory_4k" - parallel: 4 - -tezt-memory-3k: - extends: [.tezt_tests] - variables: - TESTS: "/ci_disabled /cloud memory_3k" - TEZT_PARALLEL: 1 - TEZT_VARIANT: "-memory_3k" - -# the following tests are executed with -j 1 to ensure that other -# tests do not affect their executions. However, these tests are not -# particularly cpu/memory-intensive hence they do not need to run on a -# particular machine contrary to performance regression tests. -tezt-time-sensitive: - extends: [.tezt_tests] - variables: - TESTS: "/ci_disabled /cloud time_sensitive" - TEZT_PARALLEL: 1 - TEZT_VARIANT: "-time_sensitive" - -# these are tezt tests as above, but run using the static binaries -tezt:static-binaries: - stage: test - extends: - # Expansion of .integration_template but without coverage. - - .default_settings_template - - .image_template__runtime_e2etest_dependencies - - .tezt_template - - .rules__octez_changes - dependencies: - - select_tezts - # Fetch src/proto_*/parameters/*.json and tezt/tests/main.exe from - # oc.build_x86_64-exp-dev-extra - - oc.build_x86_64-exp-dev-extra - # And fetch the static executables from build:static - - oc.build:static-x86_64-linux-binaries - - oc.tezt:fetch-records - needs: - - select_tezts - - oc.build_x86_64-exp-dev-extra - - oc.build:static-x86_64-linux-binaries - - oc.tezt:fetch-records - variables: - TESTS: "/ci_disabled /cloud cli" - before_script: - - mv octez-binaries/x86_64/octez-* . - -# Note: if you reactivate this test and if you keep it manual, put it in the "manual" stage. -# -#tezt:manual:migration: -# extends: -# - .test_template -# before_script: -# - export TEZOS_CLIENT_UNSAFE_DISABLE_DISCLAIMER=Y -# - curl -s https://api.github.com/repos/Phlogi/tezos-snapshots/releases/latest | jq -r ".assets[] | select(.name) | .browser_download_url" | grep roll | xargs wget -q -# - block_hash=$(echo mainnet.roll.* | sed -r 's/mainnet\.roll\.[0-9_-]+\.(.*)\.[0-9]+\.chain\.xz/\1/g') -# - cat mainnet.roll.* | xz -d -v -T0 > mainnet.rolling -# - scripts/prepare_migration_test.sh auto mainnet.rolling "$block_hash" -# script: -# - dune exec ./tezt/manual_tests/main.exe -- migration --color --log-buffer-size 5000 --log-file tezt-migration.log -# artifacts: -# when: always -# paths: -# - tezt-migration.log -# expire_in: 30 days diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index 651c5a601548..17bbdb1ba841 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -45,7 +45,7 @@ include: - .gitlab/ci/jobs/test/oc.script:b58_prefix-before_merging.yml - .gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts-before_merging.yml - .gitlab/ci/jobs/test/install_octez-before_merging.yml - - .gitlab/ci/jobs/test/tezt.yml + - .gitlab/ci/jobs/test/tezt-before_merging.yml - .gitlab/ci/jobs/test/tezt-slow-before_merging.yml - .gitlab/ci/jobs/test/tezt-flaky-before_merging.yml - .gitlab/ci/jobs/test/test_kernels.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 5821f0c94286..88b0011f3a1e 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -50,7 +50,7 @@ include: - .gitlab/ci/jobs/test/oc.script:test_octez_release_versions-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.script:b58_prefix-scheduled_extended_test.yml - .gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts-scheduled_extended_test.yml - - .gitlab/ci/jobs/test/tezt.yml + - .gitlab/ci/jobs/test/tezt-scheduled_extended_test.yml - .gitlab/ci/jobs/test/test_kernels.yml - .gitlab/ci/jobs/test/test_etherlink_kernel-schedule_extended_test.yml - .gitlab/ci/jobs/test/test_risc_v_kernels-schedule_extended_test.yml diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index f614925cd2e1..bf8b24c18ae2 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -1334,6 +1334,91 @@ let jobs pipeline_type = () |> job_external_split in + let jobs_tezt = + let rules = make_rules ~dependent:true ~changes:changeset_octez () in + let coverage_expiry = Duration (Days 3) in + let tezt : tezos_job = + job_tezt + ~__POS__ + ~name:"tezt" + (* Exclude all tests with tags in [tezt_tags_always_disable] or + [tezt_tags_exclusive_tags]. *) + ~tezt_tests:(tezt_tests [Not (Has_tag "flaky")]) + ~tezt_parallel:3 + ~parallel:(Vector 60) + ~rules + ~dependencies:tezt_dependencies + () + |> enable_coverage_output_artifact ~expire_in:coverage_expiry + in + let tezt_memory_4k : tezos_job = + job_tezt + ~__POS__ + ~name:"tezt-memory-4k" + ~tezt_tests:(tezt_tests ~memory_4k:true []) + ~tezt_variant:"-memory_4k" + ~parallel:(Vector 4) + ~dependencies:tezt_dependencies + ~rules + () + |> enable_coverage_output_artifact ~expire_in:coverage_expiry + in + let tezt_memory_3k : tezos_job = + job_tezt + ~__POS__ + ~name:"tezt-memory-3k" + ~tezt_tests:(tezt_tests ~memory_3k:true []) + ~tezt_variant:"-memory_3k" + ~dependencies:tezt_dependencies + ~rules + () + |> enable_coverage_output_artifact ~expire_in:coverage_expiry + in + let tezt_time_sensitive : tezos_job = + (* the following tests are executed with [~tezt_parallel:1] to ensure + that other tests do not affect their executions. However, these + tests are not particularly cpu/memory-intensive hence they do not + need to run on a particular machine contrary to performance + regression tests. *) + job_tezt + ~__POS__ + ~name:"tezt-time-sensitive" + ~tezt_tests:(tezt_tests ~time_sensitive:true []) + ~tezt_variant:"-time_sensitive" + ~dependencies:tezt_dependencies + ~rules + () + |> enable_coverage_output_artifact ~expire_in:coverage_expiry + in + let tezt_static_binaries : tezos_job = + job_tezt + ~__POS__ + ~tags:["gcp"] + ~name:"tezt:static-binaries" + ~tezt_tests:(tezt_tests [Has_tag "cli"; Not (Has_tag "flaky")]) + ~tezt_parallel:3 + ~retry:0 + ~dependencies: + (Dependent + [ + Artifacts job_select_tezts; + Artifacts job_build_x86_64_exp_dev_extra; + Artifacts job_static_x86_64_experimental; + Artifacts job_tezt_fetch_records; + ]) + ~rules + ~before_script:(before_script ["mv octez-binaries/x86_64/octez-* ."]) + () + in + [ + tezt; + tezt_memory_4k; + tezt_memory_3k; + tezt_time_sensitive; + tezt_static_binaries; + ] + |> jobs_external_split ~path:"test/tezt" + in [ job_kaitai_checks; job_kaitai_e2e_checks; @@ -1350,7 +1435,7 @@ let jobs pipeline_type = job_tezt_flaky; job_tezt_slow; ] - @ jobs_unit @ jobs_install_octez + @ jobs_unit @ jobs_install_octez @ jobs_tezt @ match pipeline_type with | Before_merging -> diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 646dbd16f553..e1c852a15cbe 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -199,7 +199,6 @@ let () = | ".gitlab/ci/jobs/test/test_kernels.yml" | ".gitlab/ci/jobs/test/test_risc_v_kernels-before_merging.yml" | ".gitlab/ci/jobs/test/test_risc_v_kernels-schedule_extended_test.yml" - | ".gitlab/ci/jobs/test/tezt.yml" | ".gitlab/ci/pipelines/before_merging.yml" | ".gitlab/ci/pipelines/schedule_extended_test.yml" -> true -- GitLab From b4a9ae55615113cfc24077fb7791a819128ec03d Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Wed, 3 Apr 2024 15:29:14 +0200 Subject: [PATCH 8/9] CI: remove redundant default values for [TEZT_{RETRY,PARALLEL}] --- .gitlab/ci/jobs/test/tezt-before_merging.yml | 20 +++++++++---------- .../jobs/test/tezt-flaky-before_merging.yml | 4 ++-- .../tezt-flaky-scheduled_extended_test.yml | 4 ++-- .../test/tezt-scheduled_extended_test.yml | 20 +++++++++---------- .../ci/jobs/test/tezt-slow-before_merging.yml | 4 ++-- .../tezt-slow-scheduled_extended_test.yml | 4 ++-- ci/bin/code_verification.ml | 6 +----- 7 files changed, 29 insertions(+), 33 deletions(-) diff --git a/.gitlab/ci/jobs/test/tezt-before_merging.yml b/.gitlab/ci/jobs/test/tezt-before_merging.yml index 8ae10aaef30b..76d56055de5f 100644 --- a/.gitlab/ci/jobs/test/tezt-before_merging.yml +++ b/.gitlab/ci/jobs/test/tezt-before_merging.yml @@ -39,8 +39,8 @@ tezt: - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} - --retry ${TEZT_RETRY:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi variables: JUNIT: tezt-junit.xml @@ -104,8 +104,8 @@ tezt-memory-4k: - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} - --retry ${TEZT_RETRY:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi variables: JUNIT: tezt-junit.xml @@ -169,8 +169,8 @@ tezt-memory-3k: - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} - --retry ${TEZT_RETRY:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi variables: JUNIT: tezt-junit.xml @@ -233,8 +233,8 @@ tezt-time-sensitive: - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} - --retry ${TEZT_RETRY:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi variables: JUNIT: tezt-junit.xml @@ -294,8 +294,8 @@ tezt:static-binaries: - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} - --retry ${TEZT_RETRY:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi variables: JUNIT: tezt-junit.xml diff --git a/.gitlab/ci/jobs/test/tezt-flaky-before_merging.yml b/.gitlab/ci/jobs/test/tezt-flaky-before_merging.yml index f37489ea2192..7c3635f545cf 100644 --- a/.gitlab/ci/jobs/test/tezt-flaky-before_merging.yml +++ b/.gitlab/ci/jobs/test/tezt-flaky-before_merging.yml @@ -40,8 +40,8 @@ tezt-flaky: - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} - --retry ${TEZT_RETRY:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi variables: JUNIT: tezt-junit.xml diff --git a/.gitlab/ci/jobs/test/tezt-flaky-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/tezt-flaky-scheduled_extended_test.yml index f9841c0f17c3..4420604e5664 100644 --- a/.gitlab/ci/jobs/test/tezt-flaky-scheduled_extended_test.yml +++ b/.gitlab/ci/jobs/test/tezt-flaky-scheduled_extended_test.yml @@ -31,8 +31,8 @@ tezt-flaky: - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} - --retry ${TEZT_RETRY:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi variables: JUNIT: tezt-junit.xml diff --git a/.gitlab/ci/jobs/test/tezt-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/tezt-scheduled_extended_test.yml index 48b95fb9c369..cc93511388f2 100644 --- a/.gitlab/ci/jobs/test/tezt-scheduled_extended_test.yml +++ b/.gitlab/ci/jobs/test/tezt-scheduled_extended_test.yml @@ -31,8 +31,8 @@ tezt: - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} - --retry ${TEZT_RETRY:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi variables: JUNIT: tezt-junit.xml @@ -88,8 +88,8 @@ tezt-memory-4k: - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} - --retry ${TEZT_RETRY:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi variables: JUNIT: tezt-junit.xml @@ -145,8 +145,8 @@ tezt-memory-3k: - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} - --retry ${TEZT_RETRY:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi variables: JUNIT: tezt-junit.xml @@ -201,8 +201,8 @@ tezt-time-sensitive: - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} - --retry ${TEZT_RETRY:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi variables: JUNIT: tezt-junit.xml @@ -254,8 +254,8 @@ tezt:static-binaries: - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} - --retry ${TEZT_RETRY:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi variables: JUNIT: tezt-junit.xml diff --git a/.gitlab/ci/jobs/test/tezt-slow-before_merging.yml b/.gitlab/ci/jobs/test/tezt-slow-before_merging.yml index 627add63250f..89c66d86fb3f 100644 --- a/.gitlab/ci/jobs/test/tezt-slow-before_merging.yml +++ b/.gitlab/ci/jobs/test/tezt-slow-before_merging.yml @@ -40,8 +40,8 @@ tezt-slow: - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} - --retry ${TEZT_RETRY:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi variables: JUNIT: tezt-junit.xml diff --git a/.gitlab/ci/jobs/test/tezt-slow-scheduled_extended_test.yml b/.gitlab/ci/jobs/test/tezt-slow-scheduled_extended_test.yml index e9292f9998d6..a0cf7167b073 100644 --- a/.gitlab/ci/jobs/test/tezt-slow-scheduled_extended_test.yml +++ b/.gitlab/ci/jobs/test/tezt-slow-scheduled_extended_test.yml @@ -31,8 +31,8 @@ tezt-slow: - ./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh "${TESTS}" --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 1800 --on-unknown-regression-files fail --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL:-3} - --retry ${TEZT_RETRY:-1} + --record tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count ${TEZT_PARALLEL} + --retry ${TEZT_RETRY} - if [ -n "${BISECT_FILE:-}" ]; then ./scripts/ci/merge_coverage.sh; fi variables: JUNIT: tezt-junit.xml diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index bf8b24c18ae2..b3a1a7c407cb 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -464,17 +464,13 @@ let job_tezt ~__POS__ ?rules ?parallel ?(tags = ["gcp_tezt"]) ~name because if the CI timeout is reached, there are no artefacts, and thus no logs to investigate. See also: https://gitlab.com/gitlab-org/gitlab/-/issues/19818 *) - (* TODO: there is no point in providing default values for the - expansion of [TEZT_PARALLEL] and [TEZT_RETRY] as they are - always set in this job (see the definition of variables - above). Remove this once all tezt jobs are translated. *) "./scripts/ci/exit_code.sh timeout -k 60 1860 ./scripts/ci/tezt.sh \ \"${TESTS}\" --color --log-buffer-size 5000 --log-file tezt.log \ --global-timeout 1800 --on-unknown-regression-files fail --junit \ ${JUNIT} --from-record tezt/records --job \ ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} --record \ tezt-results-${CI_NODE_INDEX:-1}${TEZT_VARIANT}.json --job-count \ - ${TEZT_PARALLEL:-3} --retry ${TEZT_RETRY:-1}"; + ${TEZT_PARALLEL} --retry ${TEZT_RETRY}"; "if [ -n \"${BISECT_FILE:-}\" ]; then ./scripts/ci/merge_coverage.sh; fi"; ] -- GitLab From e4e36f14febd0f3c4ba685564f8f7b707673a9c9 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Thu, 4 Apr 2024 18:26:58 +0200 Subject: [PATCH 9/9] CIAO: document Tezt test selection --- ci/bin/code_verification.ml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index b3a1a7c407cb..caa217a4f79d 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -1269,6 +1269,30 @@ let jobs pipeline_type = ] |> jobs_external_split ~path:"test/install_octez" in + (* Tezt jobs. + + The tezt jobs are split into a set of special-purpose jobs running the + tests of the corresponding tag: + - [tezt-memory-3k]: runs the jobs with tag [memory_3k], + - [tezt-memory-4k]: runs the jobs with tag [memory_4k], + - [tezt-time_sensitive]: runs the jobs with tag [time-sensitive], + - [tezt-slow]: runs the jobs with tag [slow]. + - [tezt-flaky]: runs the jobs with tag [flaky] and + none of the tags above. + + and a job [tezt] that runs all remaining tests (excepting those + that are tagged [ci_disabled], that are disabled in the CI.) + + There is an implicit rule that the Tezt tags [memory_3k], + [memory_4k], [time_sensitive], [slow] and [cloud] are mutually + exclusive. The [flaky] tag is not exclusive to these tags. If + e.g. a test has both tags [slow] and [flaky], it will run in + [tezt-slow], to prevent flaky tests to run in the [tezt-flaky] + job if they also have another special tag. Tests tagged [cloud] are + meant to be used with Tezt cloud (see [tezt/lib_cloud/README.md]) and + do not run in the CI. + + For more information on tags, see [src/lib_test/tag.mli]. *) let tezt_dependencies = Dependent [ -- GitLab