diff --git a/.gitlab/ci/jobs/coverage/coverage.yml b/.gitlab/ci/jobs/coverage/coverage.yml index 77d284e25a73e4f38d2ce65adb8b9a07d78bdd5b..a4c049ac3259989e1b07b85d1b06bfedcfbb81af 100644 --- a/.gitlab/ci/jobs/coverage/coverage.yml +++ b/.gitlab/ci/jobs/coverage/coverage.yml @@ -87,6 +87,11 @@ unified_coverage: - "tezt 58/60" - "tezt 59/60" - "tezt 60/60" + - "tezt-greedy-4k 1/4" + - "tezt-greedy-4k 2/4" + - "tezt-greedy-4k 3/4" + - "tezt-greedy-4k 4/4" + - "tezt-greedy-3k 1/1" - "unit:non-proto-x86_64" - "unit:proto-x86_64" - "unit:other-x86_64" diff --git a/.gitlab/ci/jobs/test/tezt.yml b/.gitlab/ci/jobs/test/tezt.yml index b1af9793dd0db926481d91c69bc99d0b843fe858..ac10d3845e26dee1190970875fda7f93e0da6dd3 100644 --- a/.gitlab/ci/jobs/test/tezt.yml +++ b/.gitlab/ci/jobs/test/tezt.yml @@ -9,16 +9,17 @@ include: .gitlab/ci/jobs/test/common.yml .tezt_template: variables: JUNIT: "tezt-junit.xml" - # if we do not set BISECT_FILE here, it will be empty, and include the + # if we do not set BISECT_FILE here, it will be empty, and include the # full CI_PROJECT_DIR in artifacts as per the template .tezt_template. BISECT_FILE: "$CI_PROJECT_DIR/_coverage_output/" + TEZT_VARIANT: "" artifacts: reports: junit: $JUNIT paths: - tezt.log - tezt-*.log - - tezt-results-$CI_NODE_INDEX.json + - tezt-results-${CI_NODE_INDEX}${TEZT_VARIANT}.json - $BISECT_FILE - $JUNIT # The record artifacts (tezt-results-$CI_NODE_INDEX.json) should @@ -36,7 +37,7 @@ include: .gitlab/ci/jobs/test/common.yml expire_in: 3 day when: always script: - - 'echo "TESTS: ${TESTS}, JUNIT: ${JUNIT}, CI_NODE_INDEX: ${CI_NODE_INDEX}, CI_NODE_TOTAL: ${CI_NODE_TOTAL}"' + - 'echo "TESTS: ${TESTS}, JUNIT: ${JUNIT}, CI_NODE_INDEX: ${CI_NODE_INDEX}, CI_NODE_TOTAL: ${CI_NODE_TOTAL}" TEZT_PARALLEL: ${TEZT_PARALLEL} TEZT_VARIANT: ${TEZT_VARIANT}' # For Tezt tests, there are multiple timeouts: # - --global-timeout is the internal timeout of Tezt, which only works if tests # are cooperative; @@ -59,12 +60,12 @@ include: .gitlab/ci/jobs/test/common.yml --junit ${JUNIT} --from-record tezt/records --job ${CI_NODE_INDEX:-1}/${CI_NODE_TOTAL:-1} - --record tezt-results-${CI_NODE_INDEX}.json - --job-count 3 + --record tezt-results-${CI_NODE_INDEX}${TEZT_VARIANT}.json + --job-count ${TEZT_PARALLEL:-3} --retry 1 - ./scripts/ci/merge_coverage.sh -tezt: +.tezt_tests: extends: - .test_template - .image_template__runtime_e2etest_dependencies @@ -89,6 +90,32 @@ tezt: # update test_coverage.yml with the new list of jobs. parallel: 60 +tezt: + extends: [.tezt_tests] + variables: + # Exclude tests with tags 'ci_disabled', 'flaky' and `memory_{3,4}k`. + # See tezt/lib_tezos/tag.mli for more information. + TESTS: "/ci_disabled /flaky /memory_3k /memory_4k" + # the -j option of tezt + TEZT_PARALLEL: 3 + +# the following memory hungry tests are executed with -j 1 +tezt-greedy-4k: + extends: [.tezt_tests] + variables: + TESTS: "memory_4k" + TEZT_PARALLEL: 1 + TEZT_VARIANT: "-memory_4k" + parallel: 4 + +tezt-greedy-3k: + extends: [.tezt_tests] + variables: + TESTS: "memory_3k" + TEZT_PARALLEL: 1 + TEZT_VARIANT: "-memory_3k" + parallel: 1 + # these are tezt tests as above, but run using the static binaries tezt:static-binaries: extends: diff --git a/scripts/ci/update_records/update.ml b/scripts/ci/update_records/update.ml index 4176e3c0702ff27194e44e15968a80ceafd07899..9855b96e7a95e95ac346b969b1b1448160142c23 100644 --- a/scripts/ci/update_records/update.ml +++ b/scripts/ci/update_records/update.ml @@ -31,9 +31,11 @@ let default_branch = let records_directory = "tezt/records" -let fetch_record (uri, index) = +let fetch_record (uri, index, kind) = let local_filename = index ^ ".json" in - let local = records_directory // local_filename in + let local_dir = records_directory // kind in + let local = local_dir // local_filename in + if not @@ Sys.file_exists local_dir then Sys.mkdir local_dir 0o755 ; let* () = Gitlab.get_output uri ~output_path:local in Log.info "Downloaded: %s" local ; match JSON.parse_file local with @@ -60,7 +62,31 @@ let fetch_pipeline_records_from_jobs pipeline = let job_id = JSON.(job |-> "id" |> as_int) in let name = JSON.(job |-> "name" |> as_string) in match name =~* rex "^tezt (\\d+)/\\d+$" with - | None -> None + | None -> ( + match name =~* rex "^tezt-greedy-4k (\\d+)/\\d+$" with + | None -> ( + match name =~* rex "^tezt-greedy-3k (\\d+)/\\d+$" with + | None -> None + | Some index -> + Some + ( Gitlab.project_job_artifact + ~project + ~job_id + ~artifact_path: + ("tezt-results-" ^ index ^ "-memory_3k" ^ ".json") + (), + index, + "memory_3k" )) + | Some index -> + Some + ( Gitlab.project_job_artifact + ~project + ~job_id + ~artifact_path: + ("tezt-results-" ^ index ^ "-memory_4k" ^ ".json") + (), + index, + "memory_4k" )) | Some index -> Some ( Gitlab.project_job_artifact @@ -68,7 +94,8 @@ let fetch_pipeline_records_from_jobs pipeline = ~job_id ~artifact_path:("tezt-results-" ^ index ^ ".json") (), - index ) + index, + "" ) in let records = List.filter_map get_record jobs in Log.info "Found %d Tezt jobs." (List.length records) ; diff --git a/src/lib_test/tag.ml b/src/lib_test/tag.ml index 9a2c94815bc5bd7b72fc167685d6ac0666518cbc..9cdffbf8ad88368e767cb702d821b49243d6634d 100644 --- a/src/lib_test/tag.ml +++ b/src/lib_test/tag.ml @@ -26,3 +26,7 @@ let ci_disabled = "ci_disabled" let flaky = "flaky" + +let memory_3k = "memory_3k" + +let memory_4k = "memory_4k" diff --git a/src/lib_test/tag.mli b/src/lib_test/tag.mli index 2c9f45fe4f8e571f1febaba9d0d6bbe9df1e0652..9e50eaca96035dfbfd8668081eec55bb59251d34 100644 --- a/src/lib_test/tag.mli +++ b/src/lib_test/tag.mli @@ -52,3 +52,9 @@ val flaky : string You must provide a comment to explain why the test is disabled. For flaky tests, {!flaky} should be preferred to [ci_disabled]. *) val ci_disabled : string + +(** ["memory_3k"]: tag memory hungry tests ( >3 GB of memory ). *) +val memory_3k : string + +(** ["memory_4k"]: tag memory hungry tests ( >4 GB of memory ). *) +val memory_4k : string diff --git a/tezt/records/memory_3k/1.json b/tezt/records/memory_3k/1.json new file mode 100644 index 0000000000000000000000000000000000000000..a84d212ae2ed19f467a31f4afd2e578e6641b53c --- /dev/null +++ b/tezt/records/memory_3k/1.json @@ -0,0 +1,205 @@ +[ + { + "file": "tezt/tests/p2p.ml", + "title": "Test [octez-admin-client p2p stat]", + "tags": [ + "p2p", + "connections", + "p2p_stat", + "memory_3k" + ], + "successful_runs": { + "total_time": "10136370", + "count": "1" + } + }, + { + "file": "tezt/tests/nonce_seed_revelation.ml", + "title": "Nairobi: Nonce seed revelation", + "tags": [ + "nairobi", + "nonce", + "seed", + "revelation", + "memory_3k" + ], + "successful_runs": { + "total_time": "20584517", + "count": "1" + } + }, + { + "file": "tezt/tests/nonce_seed_revelation.ml", + "title": "Oxford: Nonce seed revelation", + "tags": [ + "oxford", + "nonce", + "seed", + "revelation", + "memory_3k" + ], + "successful_runs": { + "total_time": "20031038", + "count": "1" + } + }, + { + "file": "tezt/tests/nonce_seed_revelation.ml", + "title": "Alpha: Nonce seed revelation", + "tags": [ + "alpha", + "nonce", + "seed", + "revelation", + "memory_3k" + ], + "successful_runs": { + "total_time": "19991964", + "count": "1" + } + }, + { + "file": "tezt/tests/precheck.ml", + "title": "Nairobi: forge fake block", + "tags": [ + "nairobi", + "precheck", + "fake_block", + "propagation", + "memory_3k" + ], + "successful_runs": { + "total_time": "14106458", + "count": "1" + } + }, + { + "file": "tezt/tests/precheck.ml", + "title": "Oxford: forge fake block", + "tags": [ + "oxford", + "precheck", + "fake_block", + "propagation", + "memory_3k" + ], + "successful_runs": { + "total_time": "13950037", + "count": "1" + } + }, + { + "file": "tezt/tests/precheck.ml", + "title": "Alpha: forge fake block", + "tags": [ + "alpha", + "precheck", + "fake_block", + "propagation", + "memory_3k" + ], + "successful_runs": { + "total_time": "14071660", + "count": "1" + } + }, + { + "file": "tezt/tests/precheck.ml", + "title": "Nairobi: forge block with wrong payload", + "tags": [ + "nairobi", + "precheck", + "fake_block", + "propagation", + "payload", + "memory_3k" + ], + "successful_runs": { + "total_time": "21414777", + "count": "1" + } + }, + { + "file": "tezt/tests/precheck.ml", + "title": "Oxford: forge block with wrong payload", + "tags": [ + "oxford", + "precheck", + "fake_block", + "propagation", + "payload", + "memory_3k" + ], + "successful_runs": { + "total_time": "21365196", + "count": "1" + } + }, + { + "file": "tezt/tests/precheck.ml", + "title": "Alpha: forge block with wrong payload", + "tags": [ + "alpha", + "precheck", + "fake_block", + "propagation", + "payload", + "memory_3k" + ], + "successful_runs": { + "total_time": "26430145", + "count": "1" + } + }, + { + "file": "tezt/tests/synchronisation_heuristic.ml", + "title": "Nairobi: check synchronization state", + "tags": [ + "nairobi", + "ci_disabled", + "synchronisation_threshold", + "bootstrap", + "node", + "sync", + "memory_3k" + ], + "successful_runs": { + "total_time": "16502947", + "count": "1" + } + }, + { + "file": "tezt/tests/synchronisation_heuristic.ml", + "title": "Oxford: check synchronization state", + "tags": [ + "oxford", + "ci_disabled", + "synchronisation_threshold", + "bootstrap", + "node", + "sync", + "memory_3k" + ], + "successful_runs": { + "total_time": "33719892", + "count": "1" + } + }, + { + "file": "tezt/tests/synchronisation_heuristic.ml", + "title": "Alpha: check synchronization state", + "tags": [ + "alpha", + "ci_disabled", + "synchronisation_threshold", + "bootstrap", + "node", + "sync", + "memory_3k" + ], + "successful_runs": { + "total_time": "18405060", + "count": "1" + } + } +] \ No newline at end of file diff --git a/tezt/records/memory_4k/1.json b/tezt/records/memory_4k/1.json new file mode 100644 index 0000000000000000000000000000000000000000..1632aa2850608958e438e6574f0140f4da61e630 --- /dev/null +++ b/tezt/records/memory_4k/1.json @@ -0,0 +1,18 @@ +[ + { + "file": "tezt/tests/storage_snapshots.ml", + "title": "Nairobi: storage snapshot export and import", + "tags": [ + "nairobi", + "storage", + "snapshot", + "export", + "import", + "memory_4k" + ], + "successful_runs": { + "total_time": "81737108", + "count": "1" + } + } +] \ No newline at end of file diff --git a/tezt/records/memory_4k/2.json b/tezt/records/memory_4k/2.json new file mode 100644 index 0000000000000000000000000000000000000000..1ba23c58f992ed6c20a95ad6f2c4cb2372136f86 --- /dev/null +++ b/tezt/records/memory_4k/2.json @@ -0,0 +1,18 @@ +[ + { + "file": "tezt/tests/storage_snapshots.ml", + "title": "Alpha: storage snapshot export and import", + "tags": [ + "alpha", + "storage", + "snapshot", + "export", + "import", + "memory_4k" + ], + "successful_runs": { + "total_time": "83475193", + "count": "1" + } + } +] \ No newline at end of file diff --git a/tezt/records/memory_4k/3.json b/tezt/records/memory_4k/3.json new file mode 100644 index 0000000000000000000000000000000000000000..ebb197c7551bc14f6eff9265fdd25c395ba00ff6 --- /dev/null +++ b/tezt/records/memory_4k/3.json @@ -0,0 +1,18 @@ +[ + { + "file": "tezt/tests/storage_snapshots.ml", + "title": "Oxford: storage snapshot export and import", + "tags": [ + "oxford", + "storage", + "snapshot", + "export", + "import", + "memory_4k" + ], + "successful_runs": { + "total_time": "75877320", + "count": "1" + } + } +] \ No newline at end of file diff --git a/tezt/records/memory_4k/4.json b/tezt/records/memory_4k/4.json new file mode 100644 index 0000000000000000000000000000000000000000..a4290842bb1bfcd9d1008eac8c47c7e123c7ef0e --- /dev/null +++ b/tezt/records/memory_4k/4.json @@ -0,0 +1,75 @@ +[ + { + "file": "tezt/tests/p2p.ml", + "title": "p2p-maintenance-init-expected_connections", + "tags": [ + "p2p", + "node", + "maintenance", + "memory_4k" + ], + "successful_runs": { + "total_time": "9808228", + "count": "1" + } + }, + { + "file": "tezt/tests/p2p.ml", + "title": "p2p-swap-disable", + "tags": [ + "p2p", + "node", + "swap", + "memory_4k" + ], + "successful_runs": { + "total_time": "28973722", + "count": "1" + } + }, + { + "file": "tezt/tests/synchronisation_heuristic.ml", + "title": "Nairobi: bootstrap: many nodes bootstrap", + "tags": [ + "nairobi", + "synchronisation_threshold", + "bootstrap", + "threshold", + "memory_4k" + ], + "successful_runs": { + "total_time": "52823570", + "count": "1" + } + }, + { + "file": "tezt/tests/synchronisation_heuristic.ml", + "title": "Oxford: bootstrap: many nodes bootstrap", + "tags": [ + "oxford", + "synchronisation_threshold", + "bootstrap", + "threshold", + "memory_4k" + ], + "successful_runs": { + "total_time": "52982790", + "count": "1" + } + }, + { + "file": "tezt/tests/synchronisation_heuristic.ml", + "title": "Alpha: bootstrap: many nodes bootstrap", + "tags": [ + "alpha", + "synchronisation_threshold", + "bootstrap", + "threshold", + "memory_4k" + ], + "successful_runs": { + "total_time": "53747957", + "count": "1" + } + } +] \ No newline at end of file diff --git a/tezt/tests/nonce_seed_revelation.ml b/tezt/tests/nonce_seed_revelation.ml index 5097eb19064c05820571945b80c04b2ea3d85feb..884af874cee44152efc2eb400053529a4fbf08de 100644 --- a/tezt/tests/nonce_seed_revelation.ml +++ b/tezt/tests/nonce_seed_revelation.ml @@ -54,7 +54,7 @@ let test_nonce_seed_revelation = Protocol.register_test ~__FILE__ ~title:"Nonce seed revelation" - ~tags:["nonce"; "seed"; "revelation"] + ~tags:["nonce"; "seed"; "revelation"; Tag.memory_3k] @@ fun protocol -> (* Run a node and a baker. The node runs in archive mode to obtain metadata with [RPC.get_chain_block]. *) diff --git a/tezt/tests/p2p.ml b/tezt/tests/p2p.ml index d84c72bcb5825a4bec33eb64b967bd912f856d61..e9623ce368a89667b2030504673b42b4c8342387 100644 --- a/tezt/tests/p2p.ml +++ b/tezt/tests/p2p.ml @@ -321,7 +321,7 @@ module Maintenance = struct Test.register ~__FILE__ ~title:"p2p-maintenance-init-expected_connections" - ~tags:["p2p"; "node"; "maintenance"] + ~tags:["p2p"; "node"; "maintenance"; Tag.memory_4k] @@ fun () -> (* Connections values evaluated from --connections option. *) let min_connections = expected_connections / 2 in @@ -568,7 +568,7 @@ module Swap = struct Test.register ~__FILE__ ~title:"p2p-swap-disable" - ~tags:["p2p"; "node"; "swap"] + ~tags:["p2p"; "node"; "swap"; Tag.memory_4k] @@ fun () -> (* Since we try to verify that something does not happen, we need to find when we consider having waited enough time to consider @@ -1219,7 +1219,7 @@ module P2p_stat = struct Test.register ~__FILE__ ~title:"Test [octez-admin-client p2p stat]" - ~tags:["p2p"; "connections"; "p2p_stat"] + ~tags:["p2p"; "connections"; "p2p_stat"; Tag.memory_3k] @@ fun () -> let num_nodes = 5 in Log.info "Start a clique of %d nodes" num_nodes ; diff --git a/tezt/tests/precheck.ml b/tezt/tests/precheck.ml index fec8c151ac4fb22b8f670c6f4ef1f400a1301739..1136673662340734a2df242224d23d21d4813e32 100644 --- a/tezt/tests/precheck.ml +++ b/tezt/tests/precheck.ml @@ -163,7 +163,7 @@ let propagate_precheckable_bad_block = Protocol.register_test ~__FILE__ ~title:"forge fake block" - ~tags:["precheck"; "fake_block"; "propagation"] + ~tags:["precheck"; "fake_block"; "propagation"; Tag.memory_3k] @@ fun protocol -> (* Expected topology is : N3 @@ -276,7 +276,7 @@ let propagate_precheckable_bad_block_payload = Protocol.register_test ~__FILE__ ~title:"forge block with wrong payload" - ~tags:["precheck"; "fake_block"; "propagation"; "payload"] + ~tags:["precheck"; "fake_block"; "propagation"; "payload"; Tag.memory_3k] @@ fun protocol -> (* Expected topology is : N3 diff --git a/tezt/tests/storage_snapshots.ml b/tezt/tests/storage_snapshots.ml index eab7796fd2ecb4d89056f52eac1d0ebf7d450258..3345fe733217278c57b280808b1782b1535fb213 100644 --- a/tezt/tests/storage_snapshots.ml +++ b/tezt/tests/storage_snapshots.ml @@ -255,7 +255,7 @@ let test_export_import_snapshots = Protocol.register_test ~__FILE__ ~title:"storage snapshot export and import" - ~tags:["storage"; "snapshot"; "export"; "import"] + ~tags:["storage"; "snapshot"; "export"; "import"; Tag.memory_4k] @@ fun protocol -> let archive_node = Node.create diff --git a/tezt/tests/synchronisation_heuristic.ml b/tezt/tests/synchronisation_heuristic.ml index 64815b18f719e26396520aa0362770a3028d543d..676ee2658f6713c4046f26f4ccda138e648d1130 100644 --- a/tezt/tests/synchronisation_heuristic.ml +++ b/tezt/tests/synchronisation_heuristic.ml @@ -77,7 +77,12 @@ let check_node_synchronization_state = ~title:"check synchronization state" ~tags: [ - Tag.ci_disabled; "synchronisation_threshold"; "bootstrap"; "node"; "sync"; + Tag.ci_disabled; + "synchronisation_threshold"; + "bootstrap"; + "node"; + "sync"; + Tag.memory_3k; ] @@ fun protocol -> let* main_node = Node.init ~name:"main_node" [] in @@ -500,8 +505,7 @@ let test_many_nodes_bootstrap = Protocol.register_test ~__FILE__ ~title:"bootstrap: many nodes bootstrap" - ~tags: - [Tag.ci_disabled; "synchronisation_threshold"; "bootstrap"; "threshold"] + ~tags:["synchronisation_threshold"; "bootstrap"; "threshold"; Tag.memory_4k] @@ fun protocol -> let num_nodes = 8 in let running_time = 10.0 in