diff --git a/.gitlab/ci/pipelines/before_merging.yml b/.gitlab/ci/pipelines/before_merging.yml index d4f01dd55c49c932547fdd9755c2e9312344e945..9d02ad63d2a3aa484ca2547b88c1c371e02f2811 100644 --- a/.gitlab/ci/pipelines/before_merging.yml +++ b/.gitlab/ci/pipelines/before_merging.yml @@ -3706,7 +3706,7 @@ oc.unit:proto-x86_64: image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - - gcp_high_cpu + - gcp_very_high_cpu rules: - changes: - .gitlab-ci.yml @@ -3913,7 +3913,7 @@ oc.unit:protocol_compiles: image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - - gcp_high_cpu + - gcp_very_high_cpu rules: - changes: - .gitlab-ci.yml diff --git a/.gitlab/ci/pipelines/merge_train.yml b/.gitlab/ci/pipelines/merge_train.yml index 2918b6869d731103d5c9be84464cc17eab66ffe7..24852107385884d6f30d289fb7192200433ecc0a 100644 --- a/.gitlab/ci/pipelines/merge_train.yml +++ b/.gitlab/ci/pipelines/merge_train.yml @@ -3705,7 +3705,7 @@ oc.unit:proto-x86_64: image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - - gcp_high_cpu + - gcp_very_high_cpu rules: - changes: - .gitlab-ci.yml @@ -3912,7 +3912,7 @@ oc.unit:protocol_compiles: image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - - gcp_high_cpu + - gcp_very_high_cpu rules: - changes: - .gitlab-ci.yml diff --git a/.gitlab/ci/pipelines/schedule_extended_test.yml b/.gitlab/ci/pipelines/schedule_extended_test.yml index 984213ac356dd588adc0ab7b358f1d60e8bf71e0..e4bd4511acf37805ed59d1a832fd71ea034d8658 100644 --- a/.gitlab/ci/pipelines/schedule_extended_test.yml +++ b/.gitlab/ci/pipelines/schedule_extended_test.yml @@ -2519,7 +2519,7 @@ oc.unit:proto-x86_64: image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - - gcp_high_cpu + - gcp_very_high_cpu rules: - when: on_success needs: @@ -2651,7 +2651,7 @@ oc.unit:protocol_compiles: image: ${ci_image_name}/build:${ci_image_tag} stage: test tags: - - gcp_high_cpu + - gcp_very_high_cpu rules: - when: on_success needs: diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 3c6635b7f8ff5726b21e1ec6606074787972466e..ba11dc6c1cafbd3f271692b6244d09910aade03a 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -587,7 +587,7 @@ let jobs pipeline_type = job_build_dynamic_binaries ~__POS__ ~arch:Amd64 - ~high_cpu:true + ~cpu:High ~retry: {max = 2; when_ = [Stuck_or_timeout_failure; Runner_system_failure]} ~dependencies:dependencies_needs_start @@ -605,7 +605,7 @@ let jobs pipeline_type = job_build_dynamic_binaries ~__POS__ ~arch:Amd64 - ~high_cpu:true + ~cpu:High ~retry: {max = 2; when_ = [Stuck_or_timeout_failure; Runner_system_failure]} ~dependencies:dependencies_needs_start @@ -693,7 +693,7 @@ let jobs pipeline_type = job ~__POS__ ~name:"ocaml-check" - ~high_cpu:true + ~cpu:High ~image:Images.CI.build ~stage ~retry: @@ -972,7 +972,7 @@ let jobs pipeline_type = make_rules ~changes:changeset_octez ~dependent:true () in let job_unit_test ~__POS__ ?(image = Images.CI.build) ?timeout - ?parallel_vector ?(rules = rules) ~arch ?(high_cpu = false) ~name + ?parallel_vector ?(rules = rules) ~arch ?(cpu = Normal) ~name ~make_targets () : tezos_job = let arch_string = arch_to_string arch in let script = ["make $MAKE_TARGETS"] in @@ -1006,7 +1006,7 @@ let jobs pipeline_type = ~stage:Stages.test ~image ~arch - ~high_cpu + ~cpu ~dependencies ~rules ~variables @@ -1052,7 +1052,7 @@ let jobs pipeline_type = ~__POS__ ~name:"oc.unit:other-x86_64" ~arch:Amd64 - ~high_cpu:true + ~cpu:High ~make_targets:["test-other-unit"] () |> enable_coverage_instrumentation |> enable_coverage_output_artifact @@ -1063,7 +1063,7 @@ let jobs pipeline_type = ~__POS__ ~name:"oc.unit:proto-x86_64" ~arch:Amd64 - ~high_cpu:true + ~cpu:Very_high ~make_targets:["test-proto-unit"] () |> enable_coverage_instrumentation |> enable_coverage_output_artifact @@ -1103,7 +1103,7 @@ let jobs pipeline_type = ~__POS__ ~name:"oc.unit:protocol_compiles" ~arch:Amd64 - ~high_cpu:true + ~cpu:Very_high ~retry: {max = 2; when_ = [Stuck_or_timeout_failure; Runner_system_failure]} ~image:Images.CI.build diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 95473c4a8d5c0650c5dd775695baa77715f25240..6089251d4fa82379d824de31593ea240934793ae 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -616,7 +616,7 @@ let changeset_mir_tzt = (no need to test that we pass the -static flag twice) - released variants exist, that are used in release tag pipelines (they do not build experimental executables) *) -let job_build_static_binaries ~__POS__ ~arch ?(high_cpu = false) +let job_build_static_binaries ~__POS__ ~arch ?(cpu = Normal) ?(executable_files = "script-inputs/released-executables") ?version_executable ?(release = false) ?rules ?dependencies () : tezos_job = let arch_string = arch_to_string arch in @@ -641,7 +641,7 @@ let job_build_static_binaries ~__POS__ ~arch ?(high_cpu = false) ~__POS__ ~stage:Stages.build ~arch - ~high_cpu + ~cpu ~name ~image:Images.CI.build ~before_script:(before_script ~take_ownership:true ~eval_opam:true []) @@ -771,7 +771,7 @@ type bin_package_group = A | B let bin_package_image = Image.mk_external ~image_path:"$DISTRIBUTION" -let job_build_dynamic_binaries ?rules ~__POS__ ~arch ?retry ?high_cpu +let job_build_dynamic_binaries ?rules ~__POS__ ~arch ?retry ?cpu ?(release = false) ?dependencies () = let arch_string = arch_to_string arch in let name = @@ -833,7 +833,7 @@ let job_build_dynamic_binaries ?rules ~__POS__ ~arch ?retry ?high_cpu ~stage:Stages.build ~arch ?retry - ?high_cpu + ?cpu ~name ~image:Images.CI.build ~before_script: diff --git a/ci/bin/master_branch.ml b/ci/bin/master_branch.ml index 4655815f588f5d08a9e40f053387c3e8f6dd3172..4dd6e5b56c1cbe7eb54e6e56948b4c7faec5aaea 100644 --- a/ci/bin/master_branch.ml +++ b/ci/bin/master_branch.ml @@ -56,7 +56,7 @@ let jobs = job_build_static_binaries ~__POS__ ~arch:Amd64 - ~high_cpu:false + ~cpu:Normal ~rules:rules_always () in diff --git a/ci/lib_tezos_ci/tezos_ci.ml b/ci/lib_tezos_ci/tezos_ci.ml index 09afe2e20c5fc5fe5e9d70d718f57b0123372cd1..e4de6a971c9d316539b2cdb34c38bcfcb029d879 100644 --- a/ci/lib_tezos_ci/tezos_ci.ml +++ b/ci/lib_tezos_ci/tezos_ci.ml @@ -739,6 +739,8 @@ type tag = | Gcp_tezt_memory_4k_dev | Gcp_high_cpu | Gcp_high_cpu_dev + | Gcp_very_high_cpu + | Gcp_very_high_cpu_dev | Aws_specific | Dynamic @@ -755,6 +757,8 @@ let string_of_tag = function | Gcp_tezt_memory_4k_dev -> "gcp_tezt_memory_4k_dev" | Gcp_high_cpu -> "gcp_high_cpu" | Gcp_high_cpu_dev -> "gcp_high_cpu_dev" + | Gcp_very_high_cpu -> "gcp_very_high_cpu" + | Gcp_very_high_cpu_dev -> "gcp_very_high_cpu_dev" | Aws_specific -> "aws_specific" | Dynamic -> Gitlab_ci.Var.encode dynamic_tag_var @@ -763,7 +767,8 @@ let arch_of_tag = function | Gcp_arm64 | Gcp_dev_arm64 -> Some Arm64 | Gcp | Gcp_dev | Gcp_tezt | Gcp_tezt_dev | Gcp_tezt_memory_3k | Gcp_tezt_memory_3k_dev | Gcp_tezt_memory_4k | Gcp_tezt_memory_4k_dev - | Gcp_high_cpu | Gcp_high_cpu_dev | Aws_specific -> + | Gcp_high_cpu | Gcp_high_cpu_dev | Gcp_very_high_cpu | Gcp_very_high_cpu_dev + | Aws_specific -> Some Amd64 | Dynamic -> None @@ -819,6 +824,12 @@ let resolve_dependencies job_name dependencies = type git_strategy = Fetch | Clone | No_strategy +(** The list of available CPU profiling tags for runners. *) +type cpu = + | Normal (** Target default Gitlab runner pool. *) + | High (** Target GCP high runner pool. *) + | Very_high (** Target GCP very high runner pool. *) + let enc_git_strategy = function | Fetch -> "fetch" | Clone -> "clone" @@ -827,24 +838,29 @@ let enc_git_strategy = function let job ?arch ?after_script ?allow_failure ?artifacts ?before_script ?cache ?id_tokens ?interruptible ?(dependencies = Staged []) ?(image_dependencies = []) ?services ?variables ?rules - ?(timeout = Gitlab_ci.Types.Minutes 60) ?tag ?(high_cpu = false) - ?git_strategy ?coverage ?retry ?parallel ?description ~__POS__ ?image - ?template ~stage ~name script : tezos_job = + ?(timeout = Gitlab_ci.Types.Minutes 60) ?tag ?(cpu = Normal) ?git_strategy + ?coverage ?retry ?parallel ?description ~__POS__ ?image ?template ~stage + ~name script : tezos_job = (* The tezos/tezos CI uses singleton tags for its runners. *) let tag = - match (arch, tag, high_cpu) with - | Some _, Some _, true - | None, Some _, true - | Some _, None, true - | None, None, true -> + match (arch, tag, cpu) with + | Some _, Some _, High + | None, Some _, High + | Some _, None, High + | None, None, High -> Gcp_high_cpu - | Some arch, None, false -> ( + | Some _, Some _, Very_high + | None, Some _, Very_high + | Some _, None, Very_high + | None, None, Very_high -> + Gcp_very_high_cpu + | Some arch, None, Normal -> ( match arch with Amd64 -> Gcp | Arm64 -> Gcp_arm64) - | None, Some tag, false -> tag - | None, None, false -> + | None, Some tag, Normal -> tag + | None, None, Normal -> (* By default, we assume Amd64 runners as given by the [gcp] tag. *) Gcp - | Some _, Some _, false -> + | Some _, Some _, Normal -> failwith "[job] cannot specify both [arch] and [tags] at the same time in job \ '%s'." diff --git a/ci/lib_tezos_ci/tezos_ci.mli b/ci/lib_tezos_ci/tezos_ci.mli index d479da2598402c8bec8641320964cffe11f33c51..a4cfa419e1a3bf2a36608e8ce3fc7d268ac4bcc5 100644 --- a/ci/lib_tezos_ci/tezos_ci.mli +++ b/ci/lib_tezos_ci/tezos_ci.mli @@ -263,6 +263,10 @@ type tag = (** GCP prod AMD64 runner, suitable for jobs needing high CPU. *) | Gcp_high_cpu_dev (** GCP dev AMD64 runner, suitable for jobs needing high CPU. *) + | Gcp_very_high_cpu + (** GCP prod AMD64 runner, suitable for jobs needing very high CPU. *) + | Gcp_very_high_cpu_dev + (** GCP dev AMD64 runner, suitable for jobs needing very high CPU. *) | Aws_specific (** AWS runners, in cases where a CI is legacy or not suitable for GCP. *) | Dynamic @@ -335,6 +339,11 @@ type git_strategy = CI/CD YAML variable [GIT_STRATEGY]. *) val enc_git_strategy : git_strategy -> string +type cpu = + | Normal (** Target default Gitlab runner pool. *) + | High (** Target GCP high runner pool. *) + | Very_high (** Target GCP very high runner pool. *) + (** Define a job. This smart constructor for {!Gitlab_ci.Types.job} additionally: @@ -366,8 +375,9 @@ val enc_git_strategy : git_strategy -> string architecture for the internal image cannot be statically deduced. - - The [high_cpu] parameter allocates the job to run on top of a GCP GitLab runner with a 1:1 ratio between CPU and RAM. - For more information, see [e2-highcpu-16](https://gcloud-compute.com/e2-highcpu-16.html). *) + - The [cpu] parameter specifies the CPU allocation for the job, + allowing it to run on a GCP GitLab runner with normal, high, + or very high CPU capacity. *) val job : ?arch:arch -> @@ -385,7 +395,7 @@ val job : ?rules:Gitlab_ci.Types.job_rule list -> ?timeout:Gitlab_ci.Types.time_interval -> ?tag:tag -> - ?high_cpu:bool -> + ?cpu:cpu -> ?git_strategy:git_strategy -> ?coverage:string -> ?retry:Gitlab_ci.Types.retry ->