diff --git a/ci/README.md b/ci/README.md index 513cb09cec45501b720c13541d71895e7bea3699..3b98c6b6bf9855b34b636a67693a3924adabd89f 100644 --- a/ci/README.md +++ b/ci/README.md @@ -7,3 +7,6 @@ This directory is structured like this: - `lib_gitlab_ci`: contains a partial, slightly opiniated, AST of [GitLab CI/CD YAML syntax](https://docs.gitlab.com/ee/ci/yaml/). + - `bin`: contains a set of helpers for creating the Octez-specific + GitLab CI configuration files and the skeleton of an executable that will be used for + for writing `.gitlab-ci.yml` using those helpers. diff --git a/ci/bin/dune b/ci/bin/dune new file mode 100644 index 0000000000000000000000000000000000000000..7438f8014a36eb66a5596a91e2fbb36da38b234e --- /dev/null +++ b/ci/bin/dune @@ -0,0 +1,12 @@ +; This file was automatically generated, do not edit. +; Edit file manifest/main.ml instead. + +(executable + (name main) + (libraries + gitlab_ci + yaml + unix) + (flags + (:standard) + -open Gitlab_ci.Base)) diff --git a/ci/bin/main.ml b/ci/bin/main.ml new file mode 100644 index 0000000000000000000000000000000000000000..e7d4dc6a488d426b26f45c4cf91db795bdc395e4 --- /dev/null +++ b/ci/bin/main.ml @@ -0,0 +1,15 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Nomadic Labs. *) +(* *) +(*****************************************************************************) + +(* Main entrypoint of CI-in-OCaml. + + Currently does nothing. + + Here we will register the set of pipelines, stages and images used to + generate the top-level [.gitlab-ci.yml] file. *) + +let () = () diff --git a/ci/bin/rules.ml b/ci/bin/rules.ml new file mode 100644 index 0000000000000000000000000000000000000000..5b85c8cb6da8a74e6ba72362523312e6aae078fb --- /dev/null +++ b/ci/bin/rules.ml @@ -0,0 +1,53 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Nomadic Labs. *) +(* *) +(*****************************************************************************) + +open Gitlab_ci +open Gitlab_ci.If + +(** The source of a pipeline. *) +type pipeline_source = Schedule | Merge_request_event | Push + +(** Convert at {!pipeline_source} to string. *) +let pipeline_source_to_string = function + | Schedule -> "schedule" + | Merge_request_event -> "merge_request_event" + | Push -> "push" + +let pipeline_source_eq pipeline_source = + Predefined_vars.ci_pipeline_source + == str (pipeline_source_to_string pipeline_source) + +let merge_request = pipeline_source_eq Merge_request_event + +let push = pipeline_source_eq Push + +let scheduled = pipeline_source_eq Schedule + +let schedule_extended_tests = + scheduled && var "TZ_SCHEDULE_KIND" == str "EXTENDED_TESTS" + +let on_master = Predefined_vars.ci_commit_branch == str "master" + +let on_branch branch = Predefined_vars.ci_commit_branch == str branch + +let on_tezos_namespace = Predefined_vars.ci_project_namespace == str "tezos" + +let not_on_tezos_namespace = Predefined_vars.ci_project_namespace != str "tezos" + +let has_tag_match tag = Predefined_vars.ci_commit_tag =~ tag + +let has_tag_not_match tag = + Predefined_vars.(ci_commit_tag != null && ci_commit_tag =~! tag) + +let assigned_to_marge_bot = + Predefined_vars.ci_merge_request_assignees =~ "/nomadic-margebot/" + +let triggered_by_marge_bot = + Predefined_vars.gitlab_user_login == str "nomadic-margebot" + +let has_mr_label label = + Predefined_vars.ci_merge_request_labels =~ "/(?:^|,)" ^ label ^ "(?:$|,)/" diff --git a/ci/bin/rules.mli b/ci/bin/rules.mli new file mode 100644 index 0000000000000000000000000000000000000000..344aa4dd0e326095d4023eebcad4e9655d10aafd --- /dev/null +++ b/ci/bin/rules.mli @@ -0,0 +1,56 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Nomadic Labs. *) +(* *) +(*****************************************************************************) + +open Gitlab_ci + +(** A set of commonly used rules used for defining pipeline types and their inclusion. + + For more info, refer to + {{:https://docs.gitlab.com/ee/ci/variables/predefined_variables.html}Predefined + variables reference}. *) + +(** A rule that is true if [CI_PIPELINE_SOURCE] is [merge_request_event]. *) +val merge_request : If.t + +(** A rule that is true if [CI_PIPELINE_SOURCE] is [push]. *) +val push : If.t + +(** A rule that is true if [CI_PIPELINE_SOURCE] is [scheduled]. *) +val scheduled : If.t + +(** A rule that is true for scheduled extended test pipelines. + + Such pipelines have [CI_PIPELINE_SOURCE] set to [scheduled] and + [TZ_SCHEDULE_KIND] set to [EXTENDED_TESTS]. *) +val schedule_extended_tests : If.t + +(** A rule that is true if [CI_COMMIT_BRANCH] is a given branch. *) +val on_branch : string -> If.t + +(** A rule that is true if [CI_COMMIT_BRANCH] is [master]. *) +val on_master : If.t + +(** A rule that is true if [CI_PROJECT_NAMESPACE] is [tezos]. *) +val on_tezos_namespace : If.t + +(** A rule that is true if [CI_PROJECT_NAMESPACE] is not [tezos]. *) +val not_on_tezos_namespace : If.t + +(** A rule that is true if [CI_COMMIT_TAG] is defined and matches the given regexp. *) +val has_tag_match : string -> If.t + +(** A rule that is true if [CI_COMMIT_TAG] is defined but does not matches the given regexp. *) +val has_tag_not_match : string -> If.t + +(** A rule that is true if the comma-separated list [CI_MERGE_REQUEST_LABELS] contains a given label. *) +val has_mr_label : string -> If.t + +(** A rule that is true if [CI_MERGE_REQUEST_ASSIGNEES] contains [nomadic-margebot]. *) +val assigned_to_marge_bot : If.t + +(** A rule that is true if [CI_USER_LOGIN] equals [nomadic-margebot]. *) +val triggered_by_marge_bot : If.t diff --git a/ci/bin/tezos_ci.ml b/ci/bin/tezos_ci.ml new file mode 100644 index 0000000000000000000000000000000000000000..411658f972f9102d5c086fe55f92435ddaec0103 --- /dev/null +++ b/ci/bin/tezos_ci.ml @@ -0,0 +1,233 @@ +open Gitlab_ci.Util + +let header = + {|# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +|} + +let () = Printexc.register_printer @@ function Failure s -> Some s | _ -> None + +let failwith fmt = Format.kasprintf (fun s -> failwith s) fmt + +module Stage = struct + type t = Stage of string + + let stages : t list ref = ref [] + + let register name = + let stage = Stage name in + if List.mem stage !stages then + failwith "[Stage.register] attempted to register stage %S twice" name + else ( + stages := stage :: !stages ; + stage) + + let name (Stage name) = name + + let to_string_list () = List.map name (List.rev !stages) +end + +module Pipeline = struct + type t = { + name : string; + if_ : Gitlab_ci.If.t; + variables : Gitlab_ci.Types.variables option; + } + + let pipelines : t list ref = ref [] + + let register ?variables name if_ = + let pipeline : t = {variables; if_; name} in + if List.exists (fun {name = name'; _} -> name' = name) !pipelines then + failwith + "[Pipeline.register] attempted to register pipeline %S twice" + name + else pipelines := pipeline :: !pipelines + + let all () = List.rev !pipelines + + let workflow_includes () : + Gitlab_ci.Types.workflow * Gitlab_ci.Types.include_ list = + let workflow_rule_of_pipeline = function + | {name; if_; variables} -> + (* Add [PIPELINE_TYPE] to the variables of the workflow rules, so + that it can be added to the pipeline [name] *) + let variables = + ("PIPELINE_TYPE", name) :: Option.value ~default:[] variables + in + workflow_rule ~if_ ~variables ~when_:Always () + in + let include_of_pipeline = function + | {name; if_; variables = _} -> + (* Note that variables associated to the pipeline are not + set in the include rule, they are set in the workflow + rule *) + let rule = include_rule ~if_ ~when_:Always () in + Gitlab_ci.Types. + {local = sf ".gitlab/ci/pipelines/%s.yml" name; rules = [rule]} + in + let pipelines = all () in + let workflow = + let rules = List.map workflow_rule_of_pipeline pipelines in + Gitlab_ci.Types.{rules; name = Some "[$PIPELINE_TYPE] $CI_COMMIT_TITLE"} + in + let includes = List.map include_of_pipeline pipelines in + (workflow, includes) +end + +module Image = struct + type t = Gitlab_ci.Types.image + + let images : t String_map.t ref = ref String_map.empty + + let register ~name ~image_path = + let image : t = Image image_path in + if String_map.mem name !images then + failwith "[Image.register] attempted to register image %S twice" name + else ( + images := String_map.add name image !images ; + image) + + let name (Gitlab_ci.Types.Image name) = name + + let all () = String_map.bindings !images +end + +type arch = Amd64 | Arm64 + +type dependency = + | Job of Gitlab_ci.Types.job + | Optional of Gitlab_ci.Types.job + | Artifacts of Gitlab_ci.Types.job + +type dependencies = + | Staged of Gitlab_ci.Types.job list + | Dependent of dependency list + +type git_strategy = Fetch | Clone | No_strategy + +let enc_git_strategy = function + | Fetch -> "fetch" + | Clone -> "clone" + | No_strategy -> "none" + +let job ?arch ?after_script ?allow_failure ?artifacts ?before_script ?cache + ?interruptible ?(dependencies = Staged []) ?services ?variables ?rules + ?timeout ?tags ?git_strategy ?when_ ?coverage ?retry ?parallel ~image ~stage + ~name script : Gitlab_ci.Types.job = + (match (rules, when_) with + | Some _, Some _ -> + failwith + "[job] do not use [when_] and [rules] at the same time in job '%s' -- \ + it's confusing." + name + | _ -> ()) ; + let tags = + Some + (match (arch, tags) with + | Some arch, None -> + [(match arch with Amd64 -> "gcp" | Arm64 -> "gcp_arm64")] + | None, Some tags -> tags + | None, None -> + (* By default, we assume Amd64 runners as given by the [gcp] tag. *) + ["gcp"] + | Some _, Some _ -> + failwith + "[job] cannot specify both [arch] and [tags] at the same time in \ + job '%s'." + name) + in + let stage = Some (Stage.name stage) in + (match parallel with + | Some n when n < 2 -> + failwith + "[job] the argument [parallel] must be at least 2 or omitted, in job \ + '%s'." + name + | _ -> ()) ; + let needs, dependencies = + let expand_job = function + | Gitlab_ci.Types.{name; parallel; _} -> ( + match parallel with + | None -> [name] + | Some n -> List.map (fun i -> sf "%s %d/%d" name i n) (range 1 n)) + in + match dependencies with + | Staged dependencies -> (None, List.concat_map expand_job dependencies) + | Dependent dependencies -> + let rec loop (needs, dependencies) = function + | dep :: deps -> + let job_expanded = + match dep with + | Job j | Optional j | Artifacts j -> List.rev (expand_job j) + in + let needs ~optional = + List.map + (fun name -> Gitlab_ci.Types.{job = name; optional}) + job_expanded + @ needs + in + let needs, dependencies = + match dep with + | Job _ -> (needs ~optional:false, dependencies) + | Optional _ -> (needs ~optional:true, dependencies) + | Artifacts _ -> + (needs ~optional:false, job_expanded @ dependencies) + in + loop (needs, dependencies) deps + | [] -> (Some (List.rev needs), List.rev dependencies) + in + loop ([], []) dependencies + in + (* https://docs.gitlab.com/ee/ci/yaml/#needs *) + (match needs with + | Some needs when List.length needs > 50 -> + failwith + "[job] attempted to add %d [needs] to the job '%s' -- GitLab imposes a \ + limit of 50." + (List.length needs) + name + | _ -> ()) ; + let variables = + match git_strategy with + | Some strategy -> + Some + (("GIT_STRATEGY", enc_git_strategy strategy) + :: Option.value ~default:[] variables) + | None -> variables + in + (match retry with + | Some retry when retry < 0 || retry > 2 -> + failwith + "Invalid [retry] value '%d' for job '%s': must be 0, 1 or 2." + retry + name + | _ -> ()) ; + { + name; + after_script; + allow_failure; + artifacts; + before_script; + cache; + image = Some image; + interruptible; + needs; + (* Note that [dependencies] is always filled, because we want to + fetch no dependencies by default ([dependencies = Some + []]), whereas the absence of [dependencies = None] would + fetch all the dependencies of the preceding jobs. *) + dependencies = Some dependencies; + rules; + script; + services; + stage; + variables; + timeout; + tags; + when_; + coverage; + retry; + parallel; + } diff --git a/ci/bin/tezos_ci.mli b/ci/bin/tezos_ci.mli new file mode 100644 index 0000000000000000000000000000000000000000..d8e261685e310ed157a1d196896b42e8e375ef2e --- /dev/null +++ b/ci/bin/tezos_ci.mli @@ -0,0 +1,174 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Nomadic Labs. *) +(* *) +(*****************************************************************************) + +(** A string that should be prepended to all generated files. + + Warns not to modify the generated files, and refers to the generator. *) +val header : string + +(** A facility for registering pipeline stages. *) +module Stage : sig + (* Represents a pipeline stage *) + type t + + (** Register a stage. + + Fails if a stage of the same name has already been registered. *) + val register : string -> t + + (** Name of a stage *) + val name : t -> string + + (** Returns the list of registered stages, in order of registration, as a list of strings. + + This is appropriate to use with the [Stages] constructor of + {!Gitlab_ci.Types.config_element} generating a [stages:] + element. *) + val to_string_list : unit -> string list +end + +(** A facility for registering pipelines. *) +module Pipeline : sig + (* Register a pipeline. + + [register ?variables name rule] will register a pipeline [name] + that runs when [rule] is true. The pipeline is expected to be + defined in [.gitlab/ci/pipelines/NAME.yml] which will be included + from the top-level [.gitlab-ci.yml]. + + If [variables] is set, then these variables will be added to the + [workflow:] clause for this pipeline in the top-level [.gitlab-ci.yml]. *) + val register : + ?variables:Gitlab_ci.Types.variables -> string -> Gitlab_ci.If.t -> unit + + (** Splits the set of registered pipelines into workflow rules and includes. + + The result of this function is used in the top-level + [.gitlab-ci.yml] to filter pipelines (using [workflow:] rules) + and to include the select pipeline (using [include:]). *) + val workflow_includes : + unit -> Gitlab_ci.Types.workflow * Gitlab_ci.Types.include_ list +end + +(** A facility for registering images for [image:] keywords. + + During the transition from hand-written [.gitlab-ci.yml] to + CI-in-OCaml, we write a set of templates corresponding to the + registered images, to make them available for hand-written jobs. *) +module Image : sig + (** Represents an image *) + type t = Gitlab_ci.Types.image + + (** Register an image of the given [name] and [image_path]. *) + val register : name:string -> image_path:string -> t + + (** The name of an image *) + val name : t -> string + + (** Returns the set of registered images as [name, image] tuples. *) + val all : unit -> (string * t) list +end + +(** Represents architectures. *) +type arch = Amd64 | Arm64 + +(** A job dependency. + + - A job that depends on [Job j] will not start until [j] finishes. + + - A job that depends on [Optional j] will not start until [j] + finishes, if it is present in the pipeline. For more information, + see + {{:https://docs.gitlab.com/ee/ci/yaml/#needsoptional}needs:optional}. + + - A job that depends on [Artefacts j] will not start until [j] finishes + and will also have the artefacts of [j] available. *) +type dependency = + | Job of Gitlab_ci.Types.job + | Optional of Gitlab_ci.Types.job + | Artifacts of Gitlab_ci.Types.job + +(** Job dependencies. + + - A [Staged artifact_deps] job implements the default GitLab CI behavior of + running once all jobs in the previous stage have terminated. Artifacts are + downloaded from the list of jobs in [artifact_deps] (by default, no + artifacts are downloaded). + - An [Dependent deps] job runs once all the jobs in [deps] have terminated. + To have a job run immediately, set [Dependent []] + + In practice, prefer using [Dependent]. Only use [Staged + artifact_deps] when the number of dependencies exceed the GitLab + imposed limit of 50 [needs:] per job. *) +type dependencies = + | Staged of Gitlab_ci.Types.job list + | Dependent of dependency list + +(** Values for the [GIT_STRATEGY] variable. + + This can be used to specify whether a job should [Fetch] or [Clone] + the git repository, or not get it at all with [No_strategy]. + + For more information, see + {{:https://docs.gitlab.com/ee/ci/runners/configure_runners.html#git-strategy}GIT_STRATEGY} *) +type git_strategy = + | Fetch (** Translates to [fetch]. *) + | Clone (** Translates to [clone]. *) + | No_strategy + (** Translates to []. + + Renamed to avoid clashes with {!Option.None}. *) + +(** GitLab CI/CD YAML representation of [git_strategy]. + + Translates {!git_strategy} to values of accepted by the GitLab + CI/CD YAML variable [GIT_STRATEGY]. *) +val enc_git_strategy : git_strategy -> string + +(** Define a job. + + This smart constructor for {!Gitlab_ci.Types.job} additionally: + + - Translates each {!dependency} to [needs:] and [dependencies:] + keywords as detailed in the documentation of {!dependency}. + - Adds [tags:] based on [arch] and [tags]: + + - If only [arch] is set to [Amd64] (resp. [Arm64]) then the tag + ["gcp"] (resp ["gcp_arm64"]) is set. + - If only [tags] is set, then it is passed as is to the job's [tags:] + field. + - Setting both [arch] and [tags] throws an error. + - Omitting both [arch] and [tags] is equivalent to setting + [~arch:Amd64] and omitting [tags]. + + - Throws a run-time error if both [rules] and [when_] are passed. A + [when_] field can always be represented by [rules] instead, so use + the latter for more complex conditions. *) +val job : + ?arch:arch -> + ?after_script:string list -> + ?allow_failure:Gitlab_ci.Types.allow_failure_job -> + ?artifacts:Gitlab_ci.Types.artifacts -> + ?before_script:string list -> + ?cache:Gitlab_ci.Types.cache list -> + ?interruptible:bool -> + ?dependencies:dependencies -> + ?services:Gitlab_ci.Types.service list -> + ?variables:Gitlab_ci.Types.variables -> + ?rules:Gitlab_ci.Types.job_rule list -> + ?timeout:Gitlab_ci.Types.time_interval -> + ?tags:string list -> + ?git_strategy:git_strategy -> + ?when_:Gitlab_ci.Types.when_job -> + ?coverage:string -> + ?retry:int -> + ?parallel:int -> + image:Image.t -> + stage:Stage.t -> + name:string -> + string list -> + Gitlab_ci.Types.job diff --git a/manifest/main.ml b/manifest/main.ml index 6b3ca36215992dcf7752a79d052dbe7999c11d63..f3e30a653fbbc157f4e97ab62598383c772dac71 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -8764,7 +8764,7 @@ let _docs_doc_gen_errors = Protocol.(client_exn alpha) |> open_; ] -let _ci_lib_gitlab_ci_main = +let ci_lib_gitlab_ci_main = public_lib "gitlab_ci" ~synopsis:"OCaml library for generating GitLab CI YAML configuration files" @@ -8774,6 +8774,15 @@ let _ci_lib_gitlab_ci_main = ~inline_tests:ppx_expect ~release_status:Unreleased +let _ci_bin_main = + private_exe + "main" + ~opam:"" + ~path:"ci/bin" + ~bisect_ppx:No + ~deps:[ci_lib_gitlab_ci_main |> open_ ~m:"Base"; yaml; unix] + ~release_status:Unreleased + (* Add entries to this function to declare that some dune and .opam files are not generated by the manifest on purpose.