From 19ff28ea66702d6e4c123839b7ad503c1eb657bc Mon Sep 17 00:00:00 2001 From: Pietro Abate Date: Mon, 20 May 2024 12:28:16 +0200 Subject: [PATCH] CIAO: refactor runner tag / architecture handling --- ci/bin/code_verification.ml | 6 ++-- ci/bin/common.ml | 4 +-- ci/bin/debian_repository.ml | 4 +-- ci/bin/tezos_ci.ml | 67 ++++++++++++++++++++++++++++--------- ci/bin/tezos_ci.mli | 47 ++++++++++++++++++++------ ci/lib_gitlab_ci/var.ml | 8 +++-- ci/lib_gitlab_ci/var.mli | 5 +++ 7 files changed, 106 insertions(+), 35 deletions(-) diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index da3859de78ef..df196c7345c8 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -160,7 +160,7 @@ let image_of_distribution = function | Ubuntu_jammy -> Images.ubuntu_jammy | Fedora_37 -> Images.fedora_37 -let job_tezt ~__POS__ ?rules ?parallel ?(tags = ["gcp_tezt"]) ~name +let job_tezt ~__POS__ ?rules ?parallel ?(tag = Gcp_tezt) ~name ~(tezt_tests : Tezt_core.TSL_AST.t) ?(retry = 2) ?(tezt_retry = 1) ?(tezt_parallel = 1) ?(tezt_variant = "") ?(before_script = before_script ~source_version:true ~eval_opam:true []) @@ -217,7 +217,7 @@ let job_tezt ~__POS__ ?rules ?parallel ?(tags = ["gcp_tezt"]) ~name ~image:Images.runtime_e2etest_dependencies ~name ?parallel - ~tags + ~tag ~stage:Stages.test ?rules ~artifacts @@ -1432,7 +1432,7 @@ let jobs pipeline_type = let tezt_static_binaries : tezos_job = job_tezt ~__POS__ - ~tags:["gcp"] + ~tag:Gcp ~name:"tezt:static-binaries" ~tezt_tests:(tezt_tests [Has_tag "cli"; Not (Has_tag "flaky")]) ~tezt_parallel:3 diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 8d9ccf770d1b..e8fac569d656 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -499,7 +499,7 @@ let changeset_test_evm_compatibility = authenticate with Docker Hub provided the environment variable [CI_DOCKER_AUTH] contains the appropriate credentials. *) let job_docker_authenticated ?(skip_docker_initialization = false) - ?ci_docker_hub ?artifacts ?(variables = []) ?rules ?dependencies ?arch ?tags + ?ci_docker_hub ?artifacts ?(variables = []) ?rules ?dependencies ?arch ?tag ?allow_failure ?parallel ~__POS__ ~stage ~name script : tezos_job = let docker_version = "24.0.6" in job @@ -507,7 +507,7 @@ let job_docker_authenticated ?(skip_docker_initialization = false) ?dependencies ?artifacts ?arch - ?tags + ?tag ?allow_failure ?parallel ~__POS__ diff --git a/ci/bin/debian_repository.ml b/ci/bin/debian_repository.ml index 5d6e8022b6ae..01dafee4db69 100644 --- a/ci/bin/debian_repository.ml +++ b/ci/bin/debian_repository.ml @@ -46,7 +46,7 @@ let jobs = ~stage:Stages.build ~variables:(variables [("DISTRIBUTION", distribution)]) ~parallel:(Matrix matrix) - ~tags:["$TAGS"] + ~tag:Dynamic [".gitlab/ci/jobs/packaging/build-debian-packages-dependencies.sh"] in let job_docker_build_debian_dependencies : tezos_job = @@ -71,7 +71,7 @@ let jobs = ~stage:Stages.packaging ~variables:(variables [("DISTRIBUTION", distribution)]) ~parallel:(Matrix matrix) - ~tags:["$TAGS"] + ~tag:Dynamic ~artifacts:(artifacts ["packages/$DISTRIBUTION/$RELEASE"]) [".gitlab/ci/jobs/packaging/build-debian-packages.sh"] in diff --git a/ci/bin/tezos_ci.ml b/ci/bin/tezos_ci.ml index 92cecdbd789d..61687c6e6a5c 100644 --- a/ci/bin/tezos_ci.ml +++ b/ci/bin/tezos_ci.ml @@ -480,6 +480,34 @@ let arch_to_string = function Amd64 -> "x86_64" | Arm64 -> "arm64" let arch_to_string_alt = function Amd64 -> "amd64" | Arm64 -> "arm64" +let dynamic_tag_var = Gitlab_ci.Var.make "TAGS" + +type tag = + | Gcp + | Gcp_arm64 + | Gcp_dev + | Gcp_dev_arm64 + | Gcp_tezt + | Gcp_tezt_dev + | Aws_specific + | Dynamic + +let string_of_tag = function + | Gcp -> "gcp" + | Gcp_arm64 -> "gcp_arm64" + | Gcp_dev -> "gcp_dev" + | Gcp_dev_arm64 -> "gcp_dev_arm64" + | Gcp_tezt -> "gcp_tezt" + | Gcp_tezt_dev -> "gcp_tezt_dev" + | Aws_specific -> "aws_specific" + | Dynamic -> Gitlab_ci.Var.encode dynamic_tag_var + +(** The architecture of the runner associated to a tag . *) +let arch_of_tag = function + | Gcp_arm64 | Gcp_dev_arm64 -> Some Arm64 + | Gcp | Gcp_dev | Gcp_tezt | Gcp_tezt_dev | Aws_specific -> Some Amd64 + | Dynamic -> None + type dependency = | Job of tezos_job | Optional of tezos_job @@ -534,23 +562,23 @@ let enc_git_strategy = function let job ?arch ?after_script ?allow_failure ?artifacts ?before_script ?cache ?interruptible ?(dependencies = Staged []) ?services ?variables ?rules - ?timeout ?tags ?git_strategy ?coverage ?retry ?parallel ~__POS__ ~image + ?timeout ?tag ?git_strategy ?coverage ?retry ?parallel ~__POS__ ~image ~stage ~name script : tezos_job = - let tags = - Some - (match (arch, tags) with - | Some arch, None -> - [(match arch with Amd64 -> "gcp" | Arm64 -> "gcp_arm64")] - | None, Some tags -> tags - | None, None -> - (* By default, we assume Amd64 runners as given by the [gcp] tag. *) - ["gcp"] - | Some _, Some _ -> - failwith - "[job] cannot specify both [arch] and [tags] at the same time in \ - job '%s'." - name) + (* The tezos/tezos CI uses singleton tags for its runners. *) + let tag = + match (arch, tag) with + | Some arch, None -> ( match arch with Amd64 -> Gcp | Arm64 -> Gcp_arm64) + | None, Some tag -> tag + | None, None -> + (* By default, we assume Amd64 runners as given by the [gcp] tag. *) + Gcp + | Some _, Some _ -> + failwith + "[job] cannot specify both [arch] and [tags] at the same time in job \ + '%s'." + name in + let tags = Some [string_of_tag tag] in (match (parallel : Gitlab_ci.Types.parallel option) with | Some (Vector n) when n < 2 -> failwith @@ -602,6 +630,15 @@ let job ?arch ?after_script ?allow_failure ?artifacts ?before_script ?cache retry name | _ -> ()) ; + (match + (Sys.getenv_opt Gitlab_ci.Predefined_vars.(show gitlab_user_login), tag) + with + | Some "nomadic-margebot", (Gcp_dev | Gcp_dev_arm64) -> + failwith + "[job] Attempting to merge a CI configuration using development \ + runners (job: %s)" + name + | _ -> ()) ; let job : Gitlab_ci.Types.job = { name; diff --git a/ci/bin/tezos_ci.mli b/ci/bin/tezos_ci.mli index 11f700837937..4b819bc51ea7 100644 --- a/ci/bin/tezos_ci.mli +++ b/ci/bin/tezos_ci.mli @@ -194,6 +194,33 @@ val arch_to_string : arch -> string (** Alternative string representation of architectures ([Amd64] is ["amd64"]) *) val arch_to_string_alt : arch -> string +(** The list of available runner tags. *) +type tag = + | Gcp (** GCP prod AMD64 runner, general purpose. *) + | Gcp_arm64 (** GCP prod ARM64 runner, general purpose. *) + | Gcp_dev (** GCP dev AMD64 runner, general purpose. *) + | Gcp_dev_arm64 (** GCP dev ARM64 runner, general purpose. *) + | Gcp_tezt + (** GCP prod AMD64 runner, suitable for tezt jobs (more RAM and CPU) *) + | Gcp_tezt_dev + (** GCP dev AMD64 runner, suitable for tezt jobs (more RAM and CPU) *) + | Aws_specific + (** AWS runners, in cases where a CI is legacy or not suitable for GCP. *) + | Dynamic + (** The runner is dynamically set through the CI variable {!dynamic_tag_var}. *) + +(** The variable to set enabling dynamic runner selection. + + To dynamically set the runner of a job through a CI/CD variable, + assign to this variable using [variables:] or [parallel:matrix:]. *) +val dynamic_tag_var : Gitlab_ci.Var.t + +(** The architecture of the runner associated to a tag if statically known. *) +val arch_of_tag : tag -> arch option + +(** The string representation of a tag. *) +val string_of_tag : tag -> string + (** A job dependency. - A job that depends on [Job j] will not start until [j] finishes. @@ -251,15 +278,15 @@ val enc_git_strategy : git_strategy -> string - Translates each {!dependency} to [needs:] and [dependencies:] keywords as detailed in the documentation of {!dependency}. - - Adds [tags:] based on [arch] and [tags]: - - - If only [arch] is set to [Amd64] (resp. [Arm64]) then the tag - ["gcp"] (resp ["gcp_arm64"]) is set. - - If only [tags] is set, then it is passed as is to the job's [tags:] - field. - - Setting both [arch] and [tags] throws an error. - - Omitting both [arch] and [tags] is equivalent to setting - [~arch:Amd64] and omitting [tags]. *) + - Adds [tag:] based on [arch] and [tag]: + + - If only [tag] is set, then it is passed as is to the job's [tags:] + field. The runners of the tezos/tezos CI all use singleton tags, + hence we only allow one tag per job. + - Setting both [arch] and [tag] throws an error. + - Omitting both [arch] and [tag] is equivalent to setting + [~tag:Gcp] or, equivalently, omitting tag and setting + [~arch:Amd64].*) val job : ?arch:arch -> ?after_script:string list -> @@ -273,7 +300,7 @@ val job : ?variables:Gitlab_ci.Types.variables -> ?rules:Gitlab_ci.Types.job_rule list -> ?timeout:Gitlab_ci.Types.time_interval -> - ?tags:string list -> + ?tag:tag -> ?git_strategy:git_strategy -> ?coverage:string -> ?retry:int -> diff --git a/ci/lib_gitlab_ci/var.ml b/ci/lib_gitlab_ci/var.ml index 1a696ab470f6..296b0f961204 100644 --- a/ci/lib_gitlab_ci/var.ml +++ b/ci/lib_gitlab_ci/var.ml @@ -9,8 +9,6 @@ open Base type t = string -let encode = Fun.id - let make variable_name = (* See {{:https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb#L9}string.rb}} @@ -23,4 +21,8 @@ let make variable_name = then raise (Invalid_argument (sf "[Var.t] invalid variable name '%s'" variable_name)) ; - "$" ^ variable_name + variable_name + +let name = Fun.id + +let encode variable = "$" ^ variable diff --git a/ci/lib_gitlab_ci/var.mli b/ci/lib_gitlab_ci/var.mli index b63c1383b7c6..2e7ff1742f5c 100644 --- a/ci/lib_gitlab_ci/var.mli +++ b/ci/lib_gitlab_ci/var.mli @@ -27,3 +27,8 @@ val make : string -> t [encode @@ make "foo"] is ["$foo"]. *) val encode : t -> string + +(** The name of a variable. + + [encode @@ make "foo"] is ["foo"]. *) +val name : t -> string -- GitLab