From 8846d8d1263f0092a08bd785f7a6aa8807db6d24 Mon Sep 17 00:00:00 2001 From: Romain Bardou Date: Wed, 24 Aug 2022 10:54:15 +0200 Subject: [PATCH 1/4] CI: double number of jobs for integration tests --- .gitlab/ci/test/integration.yml | 4 +- .gitlab/ci/test/tezt.yml | 2 +- .gitlab/ci/test_coverage.yml | 74 ++++++++++++++++++++++----------- 3 files changed, 54 insertions(+), 26 deletions(-) diff --git a/.gitlab/ci/test/integration.yml b/.gitlab/ci/test/integration.yml index cbdfe5dd324b..e65114ebbc2c 100644 --- a/.gitlab/ci/test/integration.yml +++ b/.gitlab/ci/test/integration.yml @@ -117,7 +117,9 @@ integration:static-binaries: integration:pytest: extends: - .pytest_integration - parallel: 9 + # WARNING: if you increase the number of parallel jobs, you need to + # update test_coverage.yml with the new list of jobs. + parallel: 20 variables: TESTS: "tests_alpha" diff --git a/.gitlab/ci/test/tezt.yml b/.gitlab/ci/test/tezt.yml index f64cc2bd47f8..33d77783e13f 100644 --- a/.gitlab/ci/test/tezt.yml +++ b/.gitlab/ci/test/tezt.yml @@ -31,7 +31,7 @@ tezt: - .tezt_template # WARNING: if you increase the number of parallel jobs, you need to # update test_coverage.yml with the new list of jobs. - parallel: 15 + parallel: 30 # these are tezt tests as above, but run using the static binaries tezt:static-binaries: diff --git a/.gitlab/ci/test_coverage.yml b/.gitlab/ci/test_coverage.yml index ecac7f8998ae..cbf690766488 100644 --- a/.gitlab/ci/test_coverage.yml +++ b/.gitlab/ci/test_coverage.yml @@ -11,31 +11,57 @@ unified_coverage: # the `dependencies: []` in `.default_settings` with a list of jobs. # Each new job in the stage test needs to be manually added to this list. dependencies: - - "integration:pytest 1/9" - - "integration:pytest 2/9" - - "integration:pytest 3/9" - - "integration:pytest 4/9" - - "integration:pytest 5/9" - - "integration:pytest 6/9" - - "integration:pytest 7/9" - - "integration:pytest 8/9" - - "integration:pytest 9/9" + - "integration:pytest 1/20" + - "integration:pytest 2/20" + - "integration:pytest 3/20" + - "integration:pytest 4/20" + - "integration:pytest 5/20" + - "integration:pytest 6/20" + - "integration:pytest 7/20" + - "integration:pytest 8/20" + - "integration:pytest 9/20" + - "integration:pytest 10/20" + - "integration:pytest 11/20" + - "integration:pytest 12/20" + - "integration:pytest 13/20" + - "integration:pytest 14/20" + - "integration:pytest 15/20" + - "integration:pytest 16/20" + - "integration:pytest 17/20" + - "integration:pytest 18/20" + - "integration:pytest 19/20" + - "integration:pytest 20/20" - "test-liquidity-baking-scripts" - - "tezt 1/15" - - "tezt 2/15" - - "tezt 3/15" - - "tezt 4/15" - - "tezt 5/15" - - "tezt 6/15" - - "tezt 7/15" - - "tezt 8/15" - - "tezt 9/15" - - "tezt 10/15" - - "tezt 11/15" - - "tezt 12/15" - - "tezt 13/15" - - "tezt 14/15" - - "tezt 15/15" + - "tezt 1/30" + - "tezt 2/30" + - "tezt 3/30" + - "tezt 4/30" + - "tezt 5/30" + - "tezt 6/30" + - "tezt 7/30" + - "tezt 8/30" + - "tezt 9/30" + - "tezt 10/30" + - "tezt 11/30" + - "tezt 12/30" + - "tezt 13/30" + - "tezt 14/30" + - "tezt 15/30" + - "tezt 16/30" + - "tezt 17/30" + - "tezt 18/30" + - "tezt 19/30" + - "tezt 20/30" + - "tezt 21/30" + - "tezt 22/30" + - "tezt 23/30" + - "tezt 24/30" + - "tezt 25/30" + - "tezt 26/30" + - "tezt 27/30" + - "tezt 28/30" + - "tezt 29/30" + - "tezt 30/30" - "unit:012_Psithaca: [proto_012_Psithaca]" - "unit:012_Psithaca: [proto_012_Psithaca__lib_protocol__1]" - "unit:012_Psithaca: [proto_012_Psithaca__lib_protocol__2]" -- GitLab From 63f33956950c875c3eb9537c34f27bf0d7d0d9c7 Mon Sep 17 00:00:00 2001 From: Romain Bardou Date: Wed, 24 Aug 2022 18:35:54 +0200 Subject: [PATCH 2/4] CI: split the longest unit:alpha job --- .gitlab/ci/test/unit.yml | 10 +++- .gitlab/ci/test_coverage.yml | 3 + manifest/main.ml | 56 ++++++++++++------- .../lib_protocol/test/pbt/dune | 12 ++++ .../lib_protocol/test/pbt/dune | 24 ++++++++ .../lib_protocol/test/pbt/dune | 24 ++++++++ src/proto_alpha/lib_protocol/test/pbt/dune | 28 ++++++++++ 7 files changed, 137 insertions(+), 20 deletions(-) diff --git a/.gitlab/ci/test/unit.yml b/.gitlab/ci/test/unit.yml index 3c243879edfb..ffe11d48e434 100644 --- a/.gitlab/ci/test/unit.yml +++ b/.gitlab/ci/test/unit.yml @@ -154,9 +154,14 @@ unit:alpha: @src/proto_alpha/lib_protocol/test/integration/operations/runtest @src/proto_alpha/lib_protocol/test/integration/validate/runtest proto_alpha__lib_protocol__3: > - @src/proto_alpha/lib_protocol/test/pbt/runtest @src/proto_alpha/lib_protocol/test/unit/runtest @src/proto_alpha/lib_protocol/runtezt + proto_alpha__lib_protocol__pbt1: > + @src/proto_alpha/lib_protocol/test/pbt/runtest1 + proto_alpha__lib_protocol__pbt2: > + @src/proto_alpha/lib_protocol/test/pbt/runtest2 + proto_alpha__lib_protocol__pbt3: > + @src/proto_alpha/lib_protocol/test/pbt/runtest3 proto_alpha: > @src/proto_alpha/lib_benchmark/runtest @src/proto_alpha/lib_client/runtest @@ -168,6 +173,9 @@ unit:alpha: - proto_alpha__lib_protocol__1 - proto_alpha__lib_protocol__2 - proto_alpha__lib_protocol__3 + - proto_alpha__lib_protocol__pbt1 + - proto_alpha__lib_protocol__pbt2 + - proto_alpha__lib_protocol__pbt3 - proto_alpha script: - scripts/test_wrapper.sh $TEST_TARGETS ${!TEST_TARGETS} diff --git a/.gitlab/ci/test_coverage.yml b/.gitlab/ci/test_coverage.yml index cbf690766488..b962c3144665 100644 --- a/.gitlab/ci/test_coverage.yml +++ b/.gitlab/ci/test_coverage.yml @@ -74,6 +74,9 @@ unified_coverage: - "unit:alpha: [proto_alpha__lib_protocol__1]" - "unit:alpha: [proto_alpha__lib_protocol__2]" - "unit:alpha: [proto_alpha__lib_protocol__3]" + - "unit:alpha: [proto_alpha__lib_protocol__pbt1]" + - "unit:alpha: [proto_alpha__lib_protocol__pbt2]" + - "unit:alpha: [proto_alpha__lib_protocol__pbt3]" - "unit:non-proto-x86_64" script: diff --git a/manifest/main.ml b/manifest/main.ml index b895b0ffbe3e..10b2835f1ca5 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -3471,26 +3471,43 @@ end = struct ~dep_globs:(if N.(number >= 015) then ["wasm_kernel/*.wasm"] else []) in let _pbt = + let list = + (* The first item of each tuple is the index N for the runtestN alias. + Those aliases are used to split into multiple CI jobs. *) + [ + (1, "liquidity_baking_pbt", true); + (1, "saturation_fuzzing", true); + (1, "test_merkle_list", N.(number >= 013)); + (1, "test_gas_properties", true); + (2, "test_sampler", N.(number >= 012)); + (2, "test_script_comparison", true); + (2, "test_tez_repr", true); + (2, "test_tx_rollup_l2_encoding", N.(number >= 013)); + (2, "test_tx_rollup_l2_withdraw_storage", N.(number <= 010)); + (2, "test_bitset", N.(number >= 013)); + (2, "test_sc_rollup_tick_repr", N.(number >= 013)); + (2, "test_sc_rollup_encoding", N.(number >= 015)); + (3, "refutation_game_pbt", N.(number == 013)); + (3, "test_refutation_game", N.(number >= 014)); + (3, "test_carbonated_map", N.(number >= 013)); + (3, "test_zk_rollup_encoding", N.(number >= 015)); + ] + |> List.filter_map (fun (i, n, b) -> if b then Some (i, n) else None) + in + let dune = + (* FIXME: https://gitlab.com/tezos/tezos/-/issues/1265 + Once those tests are ported to Tezt we can remove those aliases + and just use Tezt's auto-balancing. But this requires making QCheck + work with Tezt first. *) + let make_alias (index, test) = + Dune.alias_rule + ~action:(Dune.run_exe test []) + (sf "runtest%d" index) + in + Dune.of_list (List.map make_alias list) + in tests - (conditional_list - [ - ("liquidity_baking_pbt", true); - ("saturation_fuzzing", true); - ("test_merkle_list", N.(number >= 013)); - ("test_gas_properties", true); - ("test_sampler", N.(number >= 012)); - ("test_script_comparison", true); - ("test_tez_repr", true); - ("test_tx_rollup_l2_encoding", N.(number >= 013)); - ("test_tx_rollup_l2_withdraw_storage", N.(number <= 010)); - ("test_bitset", N.(number >= 013)); - ("test_sc_rollup_tick_repr", N.(number >= 013)); - ("test_sc_rollup_encoding", N.(number >= 015)); - ("refutation_game_pbt", N.(number == 013)); - ("test_refutation_game", N.(number >= 014)); - ("test_carbonated_map", N.(number >= 013)); - ("test_zk_rollup_encoding", N.(number >= 015)); - ]) + (List.map snd list) ~synopsis:"Tezos/Protocol: tests for economic-protocol definition" ~path:(path // "lib_protocol/test/pbt") ~opam:(sf "tezos-protocol-%s-tests" name_dash) @@ -3512,6 +3529,7 @@ end = struct benchmark_type_inference |> if_some |> open_; sc_rollup |> if_some |> if_ N.(number >= 015) |> open_; ] + ~dune in let _unit = test diff --git a/src/proto_012_Psithaca/lib_protocol/test/pbt/dune b/src/proto_012_Psithaca/lib_protocol/test/pbt/dune index 94b630142f7c..66d00b0fca32 100644 --- a/src/proto_012_Psithaca/lib_protocol/test/pbt/dune +++ b/src/proto_012_Psithaca/lib_protocol/test/pbt/dune @@ -63,3 +63,15 @@ (alias runtest) (package tezos-protocol-012-Psithaca-tests) (action (run %{dep:./test_tez_repr.exe}))) + +(rule (alias runtest1) (action (run %{exe:liquidity_baking_pbt.exe}))) + +(rule (alias runtest1) (action (run %{exe:saturation_fuzzing.exe}))) + +(rule (alias runtest1) (action (run %{exe:test_gas_properties.exe}))) + +(rule (alias runtest2) (action (run %{exe:test_sampler.exe}))) + +(rule (alias runtest2) (action (run %{exe:test_script_comparison.exe}))) + +(rule (alias runtest2) (action (run %{exe:test_tez_repr.exe}))) diff --git a/src/proto_013_PtJakart/lib_protocol/test/pbt/dune b/src/proto_013_PtJakart/lib_protocol/test/pbt/dune index fb308c7d5b94..b482e797a084 100644 --- a/src/proto_013_PtJakart/lib_protocol/test/pbt/dune +++ b/src/proto_013_PtJakart/lib_protocol/test/pbt/dune @@ -98,3 +98,27 @@ (alias runtest) (package tezos-protocol-013-PtJakart-tests) (action (run %{dep:./test_carbonated_map.exe}))) + +(rule (alias runtest1) (action (run %{exe:liquidity_baking_pbt.exe}))) + +(rule (alias runtest1) (action (run %{exe:saturation_fuzzing.exe}))) + +(rule (alias runtest1) (action (run %{exe:test_merkle_list.exe}))) + +(rule (alias runtest1) (action (run %{exe:test_gas_properties.exe}))) + +(rule (alias runtest2) (action (run %{exe:test_sampler.exe}))) + +(rule (alias runtest2) (action (run %{exe:test_script_comparison.exe}))) + +(rule (alias runtest2) (action (run %{exe:test_tez_repr.exe}))) + +(rule (alias runtest2) (action (run %{exe:test_tx_rollup_l2_encoding.exe}))) + +(rule (alias runtest2) (action (run %{exe:test_bitset.exe}))) + +(rule (alias runtest2) (action (run %{exe:test_sc_rollup_tick_repr.exe}))) + +(rule (alias runtest3) (action (run %{exe:refutation_game_pbt.exe}))) + +(rule (alias runtest3) (action (run %{exe:test_carbonated_map.exe}))) diff --git a/src/proto_014_PtKathma/lib_protocol/test/pbt/dune b/src/proto_014_PtKathma/lib_protocol/test/pbt/dune index 1df4b6426ba3..b91a78686dc0 100644 --- a/src/proto_014_PtKathma/lib_protocol/test/pbt/dune +++ b/src/proto_014_PtKathma/lib_protocol/test/pbt/dune @@ -98,3 +98,27 @@ (alias runtest) (package tezos-protocol-014-PtKathma-tests) (action (run %{dep:./test_carbonated_map.exe}))) + +(rule (alias runtest1) (action (run %{exe:liquidity_baking_pbt.exe}))) + +(rule (alias runtest1) (action (run %{exe:saturation_fuzzing.exe}))) + +(rule (alias runtest1) (action (run %{exe:test_merkle_list.exe}))) + +(rule (alias runtest1) (action (run %{exe:test_gas_properties.exe}))) + +(rule (alias runtest2) (action (run %{exe:test_sampler.exe}))) + +(rule (alias runtest2) (action (run %{exe:test_script_comparison.exe}))) + +(rule (alias runtest2) (action (run %{exe:test_tez_repr.exe}))) + +(rule (alias runtest2) (action (run %{exe:test_tx_rollup_l2_encoding.exe}))) + +(rule (alias runtest2) (action (run %{exe:test_bitset.exe}))) + +(rule (alias runtest2) (action (run %{exe:test_sc_rollup_tick_repr.exe}))) + +(rule (alias runtest3) (action (run %{exe:test_refutation_game.exe}))) + +(rule (alias runtest3) (action (run %{exe:test_carbonated_map.exe}))) diff --git a/src/proto_alpha/lib_protocol/test/pbt/dune b/src/proto_alpha/lib_protocol/test/pbt/dune index 3d161abf23dd..06e66e88e924 100644 --- a/src/proto_alpha/lib_protocol/test/pbt/dune +++ b/src/proto_alpha/lib_protocol/test/pbt/dune @@ -112,3 +112,31 @@ (alias runtest) (package tezos-protocol-alpha-tests) (action (run %{dep:./test_zk_rollup_encoding.exe}))) + +(rule (alias runtest1) (action (run %{exe:liquidity_baking_pbt.exe}))) + +(rule (alias runtest1) (action (run %{exe:saturation_fuzzing.exe}))) + +(rule (alias runtest1) (action (run %{exe:test_merkle_list.exe}))) + +(rule (alias runtest1) (action (run %{exe:test_gas_properties.exe}))) + +(rule (alias runtest2) (action (run %{exe:test_sampler.exe}))) + +(rule (alias runtest2) (action (run %{exe:test_script_comparison.exe}))) + +(rule (alias runtest2) (action (run %{exe:test_tez_repr.exe}))) + +(rule (alias runtest2) (action (run %{exe:test_tx_rollup_l2_encoding.exe}))) + +(rule (alias runtest2) (action (run %{exe:test_bitset.exe}))) + +(rule (alias runtest2) (action (run %{exe:test_sc_rollup_tick_repr.exe}))) + +(rule (alias runtest2) (action (run %{exe:test_sc_rollup_encoding.exe}))) + +(rule (alias runtest3) (action (run %{exe:test_refutation_game.exe}))) + +(rule (alias runtest3) (action (run %{exe:test_carbonated_map.exe}))) + +(rule (alias runtest3) (action (run %{exe:test_zk_rollup_encoding.exe}))) -- GitLab From 218efa343147ff53f18f8d75012fb06f2aa18a3d Mon Sep 17 00:00:00 2001 From: Romain Bardou Date: Fri, 26 Aug 2022 12:12:09 +0200 Subject: [PATCH 3/4] Alpha/PBT: reduce count for test_refutation_game --- src/proto_alpha/lib_protocol/test/pbt/test_refutation_game.ml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/proto_alpha/lib_protocol/test/pbt/test_refutation_game.ml b/src/proto_alpha/lib_protocol/test/pbt/test_refutation_game.ml index 5a9a1b8e1700..f956b24cdc6c 100644 --- a/src/proto_alpha/lib_protocol/test/pbt/test_refutation_game.ml +++ b/src/proto_alpha/lib_protocol/test/pbt/test_refutation_game.ml @@ -428,7 +428,7 @@ module Dissection = struct (** {3 Dissection tests} *) - let count = 1_000 + let count = 300 (** Test the validity of dissection generated by {!gen_dissection} on an initial dissection generated by {!gen_initial_dissection}. @@ -1565,7 +1565,7 @@ let test_game ?nonempty_inputs ~p1_strategy ~p2_strategy () = (if p1_start then "p1" else "p2") pp_levels_and_inputs levels_and_inputs) - ~count:1_000 + ~count:300 ~name ~gen:(gen_game ?nonempty_inputs ~p1_strategy ~p2_strategy ()) (fun ( block, -- GitLab From a3202d018676004a6c8ec9f1efc220c730382adf Mon Sep 17 00:00:00 2001 From: Romain Bardou Date: Fri, 26 Aug 2022 15:44:46 +0200 Subject: [PATCH 4/4] CI: start build sooner --- .gitlab/ci/build.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index ceb8860cdbe9..7acf063f9904 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -48,6 +48,9 @@ build_x86_64: extends: - .build - .rules_template__development_and_extended + # This job is the main bottleneck since a lot of tests depend on it. + # So we start it even before sanity_ci. + - .needs_template__wait_for_trigger_job variables: ARCH: "x86_64" @@ -100,6 +103,10 @@ build:static-x86_64-linux-binaries: extends: - .build_static_binaries_template - .rules_template__not_on_latest_release + # Even though not many tests depend on static executables, some of those that do + # are limiting factors in the total duration of pipelines. + # So we start this job as early as possible, without waiting for sanity_ci. + - .needs_template__wait_for_trigger_job variables: ARCH: "x86_64" -- GitLab