From 3f303a0215d92e6126416ee0cd6a384cb92528ff Mon Sep 17 00:00:00 2001 From: Pietro Abate Date: Thu, 27 Jul 2023 15:46:43 +0200 Subject: [PATCH 1/2] tezt: tags greedy tests --- src/lib_test/tag.ml | 4 ++++ src/lib_test/tag.mli | 8 ++++++++ tezt/tests/nonce_seed_revelation.ml | 2 +- tezt/tests/p2p.ml | 6 +++--- tezt/tests/precheck.ml | 4 ++-- tezt/tests/storage_snapshots.ml | 2 +- tezt/tests/synchronisation_heuristic.ml | 15 +++++++++++++-- 7 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/lib_test/tag.ml b/src/lib_test/tag.ml index 9a2c94815bc5..9cdffbf8ad88 100644 --- a/src/lib_test/tag.ml +++ b/src/lib_test/tag.ml @@ -26,3 +26,7 @@ let ci_disabled = "ci_disabled" let flaky = "flaky" + +let memory_3k = "memory_3k" + +let memory_4k = "memory_4k" diff --git a/src/lib_test/tag.mli b/src/lib_test/tag.mli index 2c9f45fe4f8e..c0f71719bb3d 100644 --- a/src/lib_test/tag.mli +++ b/src/lib_test/tag.mli @@ -52,3 +52,11 @@ val flaky : string You must provide a comment to explain why the test is disabled. For flaky tests, {!flaky} should be preferred to [ci_disabled]. *) val ci_disabled : string + +(** ["memory_3k"]: tag memory hungry tests ( >3 GB of memory ). *) + +val memory_3k : string + +(** ["memory_4k"]: tag memory hungry tests ( >4 GB of memory ). *) + +val memory_4k : string diff --git a/tezt/tests/nonce_seed_revelation.ml b/tezt/tests/nonce_seed_revelation.ml index 5097eb19064c..884af874cee4 100644 --- a/tezt/tests/nonce_seed_revelation.ml +++ b/tezt/tests/nonce_seed_revelation.ml @@ -54,7 +54,7 @@ let test_nonce_seed_revelation = Protocol.register_test ~__FILE__ ~title:"Nonce seed revelation" - ~tags:["nonce"; "seed"; "revelation"] + ~tags:["nonce"; "seed"; "revelation"; Tag.memory_3k] @@ fun protocol -> (* Run a node and a baker. The node runs in archive mode to obtain metadata with [RPC.get_chain_block]. *) diff --git a/tezt/tests/p2p.ml b/tezt/tests/p2p.ml index d84c72bcb582..e9623ce368a8 100644 --- a/tezt/tests/p2p.ml +++ b/tezt/tests/p2p.ml @@ -321,7 +321,7 @@ module Maintenance = struct Test.register ~__FILE__ ~title:"p2p-maintenance-init-expected_connections" - ~tags:["p2p"; "node"; "maintenance"] + ~tags:["p2p"; "node"; "maintenance"; Tag.memory_4k] @@ fun () -> (* Connections values evaluated from --connections option. *) let min_connections = expected_connections / 2 in @@ -568,7 +568,7 @@ module Swap = struct Test.register ~__FILE__ ~title:"p2p-swap-disable" - ~tags:["p2p"; "node"; "swap"] + ~tags:["p2p"; "node"; "swap"; Tag.memory_4k] @@ fun () -> (* Since we try to verify that something does not happen, we need to find when we consider having waited enough time to consider @@ -1219,7 +1219,7 @@ module P2p_stat = struct Test.register ~__FILE__ ~title:"Test [octez-admin-client p2p stat]" - ~tags:["p2p"; "connections"; "p2p_stat"] + ~tags:["p2p"; "connections"; "p2p_stat"; Tag.memory_3k] @@ fun () -> let num_nodes = 5 in Log.info "Start a clique of %d nodes" num_nodes ; diff --git a/tezt/tests/precheck.ml b/tezt/tests/precheck.ml index fec8c151ac4f..113667366234 100644 --- a/tezt/tests/precheck.ml +++ b/tezt/tests/precheck.ml @@ -163,7 +163,7 @@ let propagate_precheckable_bad_block = Protocol.register_test ~__FILE__ ~title:"forge fake block" - ~tags:["precheck"; "fake_block"; "propagation"] + ~tags:["precheck"; "fake_block"; "propagation"; Tag.memory_3k] @@ fun protocol -> (* Expected topology is : N3 @@ -276,7 +276,7 @@ let propagate_precheckable_bad_block_payload = Protocol.register_test ~__FILE__ ~title:"forge block with wrong payload" - ~tags:["precheck"; "fake_block"; "propagation"; "payload"] + ~tags:["precheck"; "fake_block"; "propagation"; "payload"; Tag.memory_3k] @@ fun protocol -> (* Expected topology is : N3 diff --git a/tezt/tests/storage_snapshots.ml b/tezt/tests/storage_snapshots.ml index eab7796fd2ec..3345fe733217 100644 --- a/tezt/tests/storage_snapshots.ml +++ b/tezt/tests/storage_snapshots.ml @@ -255,7 +255,7 @@ let test_export_import_snapshots = Protocol.register_test ~__FILE__ ~title:"storage snapshot export and import" - ~tags:["storage"; "snapshot"; "export"; "import"] + ~tags:["storage"; "snapshot"; "export"; "import"; Tag.memory_4k] @@ fun protocol -> let archive_node = Node.create diff --git a/tezt/tests/synchronisation_heuristic.ml b/tezt/tests/synchronisation_heuristic.ml index 64815b18f719..86c680076aed 100644 --- a/tezt/tests/synchronisation_heuristic.ml +++ b/tezt/tests/synchronisation_heuristic.ml @@ -77,7 +77,12 @@ let check_node_synchronization_state = ~title:"check synchronization state" ~tags: [ - Tag.ci_disabled; "synchronisation_threshold"; "bootstrap"; "node"; "sync"; + Tag.ci_disabled; + "synchronisation_threshold"; + "bootstrap"; + "node"; + "sync"; + Tag.memory_3k; ] @@ fun protocol -> let* main_node = Node.init ~name:"main_node" [] in @@ -501,7 +506,13 @@ let test_many_nodes_bootstrap = ~__FILE__ ~title:"bootstrap: many nodes bootstrap" ~tags: - [Tag.ci_disabled; "synchronisation_threshold"; "bootstrap"; "threshold"] + [ + Tag.ci_disabled; + "synchronisation_threshold"; + "bootstrap"; + "threshold"; + Tag.memory_4k; + ] @@ fun protocol -> let num_nodes = 8 in let running_time = 10.0 in -- GitLab From 4ec37ab112631759f310467aeb58042b5188f7c9 Mon Sep 17 00:00:00 2001 From: Pietro Abate Date: Thu, 27 Jul 2023 15:47:15 +0200 Subject: [PATCH 2/2] tezt: add greedy tests to ci --- .gitlab/ci/jobs/coverage/coverage.yml | 5 + .gitlab/ci/jobs/test/tezt.yml | 39 ++++- scripts/ci/update_records/update.ml | 35 +++- src/lib_test/tag.mli | 2 - tezt/records/memory_3k/1.json | 205 ++++++++++++++++++++++++ tezt/records/memory_4k/1.json | 18 +++ tezt/records/memory_4k/2.json | 18 +++ tezt/records/memory_4k/3.json | 18 +++ tezt/records/memory_4k/4.json | 75 +++++++++ tezt/tests/synchronisation_heuristic.ml | 9 +- 10 files changed, 404 insertions(+), 20 deletions(-) create mode 100644 tezt/records/memory_3k/1.json create mode 100644 tezt/records/memory_4k/1.json create mode 100644 tezt/records/memory_4k/2.json create mode 100644 tezt/records/memory_4k/3.json create mode 100644 tezt/records/memory_4k/4.json diff --git a/.gitlab/ci/jobs/coverage/coverage.yml b/.gitlab/ci/jobs/coverage/coverage.yml index 77d284e25a73..a4c049ac3259 100644 --- a/.gitlab/ci/jobs/coverage/coverage.yml +++ b/.gitlab/ci/jobs/coverage/coverage.yml @@ -87,6 +87,11 @@ unified_coverage: - "tezt 58/60" - "tezt 59/60" - "tezt 60/60" + - "tezt-greedy-4k 1/4" + - "tezt-greedy-4k 2/4" + - "tezt-greedy-4k 3/4" + - "tezt-greedy-4k 4/4" + - "tezt-greedy-3k 1/1" - "unit:non-proto-x86_64" - "unit:proto-x86_64" - "unit:other-x86_64" diff --git a/.gitlab/ci/jobs/test/tezt.yml b/.gitlab/ci/jobs/test/tezt.yml index b1af9793dd0d..ac10d3845e26 100644 --- a/.gitlab/ci/jobs/test/tezt.yml +++ b/.gitlab/ci/jobs/test/tezt.yml @@ -9,16 +9,17 @@ include: .gitlab/ci/jobs/test/common.yml .tezt_template: variables: JUNIT: "tezt-junit.xml" - # if we do not set BISECT_FILE here, it will be empty, and include the + # if we do not set BISECT_FILE here, it will be empty, and include the # full CI_PROJECT_DIR in artifacts as per the template .tezt_template. BISECT_FILE: "$CI_PROJECT_DIR/_coverage_output/" + TEZT_VARIANT: "" artifacts: reports: junit: $JUNIT paths: - tezt.log - tezt-*.log - - tezt-results-$CI_NODE_INDEX.json + - tezt-results-${CI_NODE_INDEX}${TEZT_VARIANT}.json - $BISECT_FILE - $JUNIT # The record artifacts (tezt-results-$CI_NODE_INDEX.json) should @@ -36,7 +37,7 @@ include: .gitlab/ci/jobs/test/common.yml expire_in: 3 day when: always script: - - 'echo "TESTS: ${TESTS}, JUNIT: ${JUNIT}, CI_NODE_INDEX: ${CI_NODE_INDEX}, CI_NODE_TOTAL: ${CI_NODE_TOTAL}"' + - 'echo "TESTS: ${TESTS}, JUNIT: ${JUNIT}, CI_NODE_INDEX: ${CI_NODE_INDEX}, CI_NODE_TOTAL: ${CI_NODE_TOTAL}" TEZT_PARALLEL: ${TEZT_PARALLEL} TEZT_VARIANT: ${TEZT_VARIANT}' # For Tezt tests, there are multiple timeouts: # - --global-timeout is the internal timeout of Tezt, which only works if tests # are cooperative; @@ -59,12 +60,12 @@ include: .gitlab/ci/jobs/test/common.yml --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --record tezt-results-${CI_NODE_INDEX}.json - --job-count 3 + --record tezt-results-${CI_NODE_INDEX}${TEZT_VARIANT}.json + --job-count ${TEZT_PARALLEL:-3} --retry 1 - ./scripts/ci/merge_coverage.sh -tezt: +.tezt_tests: extends: - .test_template - .image_template__runtime_e2etest_dependencies @@ -89,6 +90,32 @@ tezt: # update test_coverage.yml with the new list of jobs. parallel: 60 +tezt: + extends: [.tezt_tests] + variables: + # Exclude tests with tags 'ci_disabled', 'flaky' and `memory_{3,4}k`. + # See tezt/lib_tezos/tag.mli for more information. + TESTS: "/ci_disabled /flaky /memory_3k /memory_4k" + # the -j option of tezt + TEZT_PARALLEL: 3 + +# the following memory hungry tests are executed with -j 1 +tezt-greedy-4k: + extends: [.tezt_tests] + variables: + TESTS: "memory_4k" + TEZT_PARALLEL: 1 + TEZT_VARIANT: "-memory_4k" + parallel: 4 + +tezt-greedy-3k: + extends: [.tezt_tests] + variables: + TESTS: "memory_3k" + TEZT_PARALLEL: 1 + TEZT_VARIANT: "-memory_3k" + parallel: 1 + # these are tezt tests as above, but run using the static binaries tezt:static-binaries: extends: diff --git a/scripts/ci/update_records/update.ml b/scripts/ci/update_records/update.ml index 4176e3c0702f..9855b96e7a95 100644 --- a/scripts/ci/update_records/update.ml +++ b/scripts/ci/update_records/update.ml @@ -31,9 +31,11 @@ let default_branch = let records_directory = "tezt/records" -let fetch_record (uri, index) = +let fetch_record (uri, index, kind) = let local_filename = index ^ ".json" in - let local = records_directory // local_filename in + let local_dir = records_directory // kind in + let local = local_dir // local_filename in + if not @@ Sys.file_exists local_dir then Sys.mkdir local_dir 0o755 ; let* () = Gitlab.get_output uri ~output_path:local in Log.info "Downloaded: %s" local ; match JSON.parse_file local with @@ -60,7 +62,31 @@ let fetch_pipeline_records_from_jobs pipeline = let job_id = JSON.(job |-> "id" |> as_int) in let name = JSON.(job |-> "name" |> as_string) in match name =~* rex "^tezt (\\d+)/\\d+$" with - | None -> None + | None -> ( + match name =~* rex "^tezt-greedy-4k (\\d+)/\\d+$" with + | None -> ( + match name =~* rex "^tezt-greedy-3k (\\d+)/\\d+$" with + | None -> None + | Some index -> + Some + ( Gitlab.project_job_artifact + ~project + ~job_id + ~artifact_path: + ("tezt-results-" ^ index ^ "-memory_3k" ^ ".json") + (), + index, + "memory_3k" )) + | Some index -> + Some + ( Gitlab.project_job_artifact + ~project + ~job_id + ~artifact_path: + ("tezt-results-" ^ index ^ "-memory_4k" ^ ".json") + (), + index, + "memory_4k" )) | Some index -> Some ( Gitlab.project_job_artifact @@ -68,7 +94,8 @@ let fetch_pipeline_records_from_jobs pipeline = ~job_id ~artifact_path:("tezt-results-" ^ index ^ ".json") (), - index ) + index, + "" ) in let records = List.filter_map get_record jobs in Log.info "Found %d Tezt jobs." (List.length records) ; diff --git a/src/lib_test/tag.mli b/src/lib_test/tag.mli index c0f71719bb3d..9e50eaca9603 100644 --- a/src/lib_test/tag.mli +++ b/src/lib_test/tag.mli @@ -54,9 +54,7 @@ val flaky : string val ci_disabled : string (** ["memory_3k"]: tag memory hungry tests ( >3 GB of memory ). *) - val memory_3k : string (** ["memory_4k"]: tag memory hungry tests ( >4 GB of memory ). *) - val memory_4k : string diff --git a/tezt/records/memory_3k/1.json b/tezt/records/memory_3k/1.json new file mode 100644 index 000000000000..a84d212ae2ed --- /dev/null +++ b/tezt/records/memory_3k/1.json @@ -0,0 +1,205 @@ +[ + { + "file": "tezt/tests/p2p.ml", + "title": "Test [octez-admin-client p2p stat]", + "tags": [ + "p2p", + "connections", + "p2p_stat", + "memory_3k" + ], + "successful_runs": { + "total_time": "10136370", + "count": "1" + } + }, + { + "file": "tezt/tests/nonce_seed_revelation.ml", + "title": "Nairobi: Nonce seed revelation", + "tags": [ + "nairobi", + "nonce", + "seed", + "revelation", + "memory_3k" + ], + "successful_runs": { + "total_time": "20584517", + "count": "1" + } + }, + { + "file": "tezt/tests/nonce_seed_revelation.ml", + "title": "Oxford: Nonce seed revelation", + "tags": [ + "oxford", + "nonce", + "seed", + "revelation", + "memory_3k" + ], + "successful_runs": { + "total_time": "20031038", + "count": "1" + } + }, + { + "file": "tezt/tests/nonce_seed_revelation.ml", + "title": "Alpha: Nonce seed revelation", + "tags": [ + "alpha", + "nonce", + "seed", + "revelation", + "memory_3k" + ], + "successful_runs": { + "total_time": "19991964", + "count": "1" + } + }, + { + "file": "tezt/tests/precheck.ml", + "title": "Nairobi: forge fake block", + "tags": [ + "nairobi", + "precheck", + "fake_block", + "propagation", + "memory_3k" + ], + "successful_runs": { + "total_time": "14106458", + "count": "1" + } + }, + { + "file": "tezt/tests/precheck.ml", + "title": "Oxford: forge fake block", + "tags": [ + "oxford", + "precheck", + "fake_block", + "propagation", + "memory_3k" + ], + "successful_runs": { + "total_time": "13950037", + "count": "1" + } + }, + { + "file": "tezt/tests/precheck.ml", + "title": "Alpha: forge fake block", + "tags": [ + "alpha", + "precheck", + "fake_block", + "propagation", + "memory_3k" + ], + "successful_runs": { + "total_time": "14071660", + "count": "1" + } + }, + { + "file": "tezt/tests/precheck.ml", + "title": "Nairobi: forge block with wrong payload", + "tags": [ + "nairobi", + "precheck", + "fake_block", + "propagation", + "payload", + "memory_3k" + ], + "successful_runs": { + "total_time": "21414777", + "count": "1" + } + }, + { + "file": "tezt/tests/precheck.ml", + "title": "Oxford: forge block with wrong payload", + "tags": [ + "oxford", + "precheck", + "fake_block", + "propagation", + "payload", + "memory_3k" + ], + "successful_runs": { + "total_time": "21365196", + "count": "1" + } + }, + { + "file": "tezt/tests/precheck.ml", + "title": "Alpha: forge block with wrong payload", + "tags": [ + "alpha", + "precheck", + "fake_block", + "propagation", + "payload", + "memory_3k" + ], + "successful_runs": { + "total_time": "26430145", + "count": "1" + } + }, + { + "file": "tezt/tests/synchronisation_heuristic.ml", + "title": "Nairobi: check synchronization state", + "tags": [ + "nairobi", + "ci_disabled", + "synchronisation_threshold", + "bootstrap", + "node", + "sync", + "memory_3k" + ], + "successful_runs": { + "total_time": "16502947", + "count": "1" + } + }, + { + "file": "tezt/tests/synchronisation_heuristic.ml", + "title": "Oxford: check synchronization state", + "tags": [ + "oxford", + "ci_disabled", + "synchronisation_threshold", + "bootstrap", + "node", + "sync", + "memory_3k" + ], + "successful_runs": { + "total_time": "33719892", + "count": "1" + } + }, + { + "file": "tezt/tests/synchronisation_heuristic.ml", + "title": "Alpha: check synchronization state", + "tags": [ + "alpha", + "ci_disabled", + "synchronisation_threshold", + "bootstrap", + "node", + "sync", + "memory_3k" + ], + "successful_runs": { + "total_time": "18405060", + "count": "1" + } + } +] \ No newline at end of file diff --git a/tezt/records/memory_4k/1.json b/tezt/records/memory_4k/1.json new file mode 100644 index 000000000000..1632aa285060 --- /dev/null +++ b/tezt/records/memory_4k/1.json @@ -0,0 +1,18 @@ +[ + { + "file": "tezt/tests/storage_snapshots.ml", + "title": "Nairobi: storage snapshot export and import", + "tags": [ + "nairobi", + "storage", + "snapshot", + "export", + "import", + "memory_4k" + ], + "successful_runs": { + "total_time": "81737108", + "count": "1" + } + } +] \ No newline at end of file diff --git a/tezt/records/memory_4k/2.json b/tezt/records/memory_4k/2.json new file mode 100644 index 000000000000..1ba23c58f992 --- /dev/null +++ b/tezt/records/memory_4k/2.json @@ -0,0 +1,18 @@ +[ + { + "file": "tezt/tests/storage_snapshots.ml", + "title": "Alpha: storage snapshot export and import", + "tags": [ + "alpha", + "storage", + "snapshot", + "export", + "import", + "memory_4k" + ], + "successful_runs": { + "total_time": "83475193", + "count": "1" + } + } +] \ No newline at end of file diff --git a/tezt/records/memory_4k/3.json b/tezt/records/memory_4k/3.json new file mode 100644 index 000000000000..ebb197c7551b --- /dev/null +++ b/tezt/records/memory_4k/3.json @@ -0,0 +1,18 @@ +[ + { + "file": "tezt/tests/storage_snapshots.ml", + "title": "Oxford: storage snapshot export and import", + "tags": [ + "oxford", + "storage", + "snapshot", + "export", + "import", + "memory_4k" + ], + "successful_runs": { + "total_time": "75877320", + "count": "1" + } + } +] \ No newline at end of file diff --git a/tezt/records/memory_4k/4.json b/tezt/records/memory_4k/4.json new file mode 100644 index 000000000000..a4290842bb1b --- /dev/null +++ b/tezt/records/memory_4k/4.json @@ -0,0 +1,75 @@ +[ + { + "file": "tezt/tests/p2p.ml", + "title": "p2p-maintenance-init-expected_connections", + "tags": [ + "p2p", + "node", + "maintenance", + "memory_4k" + ], + "successful_runs": { + "total_time": "9808228", + "count": "1" + } + }, + { + "file": "tezt/tests/p2p.ml", + "title": "p2p-swap-disable", + "tags": [ + "p2p", + "node", + "swap", + "memory_4k" + ], + "successful_runs": { + "total_time": "28973722", + "count": "1" + } + }, + { + "file": "tezt/tests/synchronisation_heuristic.ml", + "title": "Nairobi: bootstrap: many nodes bootstrap", + "tags": [ + "nairobi", + "synchronisation_threshold", + "bootstrap", + "threshold", + "memory_4k" + ], + "successful_runs": { + "total_time": "52823570", + "count": "1" + } + }, + { + "file": "tezt/tests/synchronisation_heuristic.ml", + "title": "Oxford: bootstrap: many nodes bootstrap", + "tags": [ + "oxford", + "synchronisation_threshold", + "bootstrap", + "threshold", + "memory_4k" + ], + "successful_runs": { + "total_time": "52982790", + "count": "1" + } + }, + { + "file": "tezt/tests/synchronisation_heuristic.ml", + "title": "Alpha: bootstrap: many nodes bootstrap", + "tags": [ + "alpha", + "synchronisation_threshold", + "bootstrap", + "threshold", + "memory_4k" + ], + "successful_runs": { + "total_time": "53747957", + "count": "1" + } + } +] \ No newline at end of file diff --git a/tezt/tests/synchronisation_heuristic.ml b/tezt/tests/synchronisation_heuristic.ml index 86c680076aed..676ee2658f67 100644 --- a/tezt/tests/synchronisation_heuristic.ml +++ b/tezt/tests/synchronisation_heuristic.ml @@ -505,14 +505,7 @@ let test_many_nodes_bootstrap = Protocol.register_test ~__FILE__ ~title:"bootstrap: many nodes bootstrap" - ~tags: - [ - Tag.ci_disabled; - "synchronisation_threshold"; - "bootstrap"; - "threshold"; - Tag.memory_4k; - ] + ~tags:["synchronisation_threshold"; "bootstrap"; "threshold"; Tag.memory_4k] @@ fun protocol -> let num_nodes = 8 in let running_time = 10.0 in -- GitLab