From e5d517b4821ae47aeb372af23685b10fad09a315 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 5 Jan 2024 18:19:39 +0100 Subject: [PATCH 1/3] CI-in-OCaml: can write generated pipelines with simple dependency check --- ci/bin/main.ml | 1 + ci/bin/tezos_ci.ml | 107 +++++++++++++++++++++++++++++++++++++++++--- ci/bin/tezos_ci.mli | 24 +++++++--- 3 files changed, 121 insertions(+), 11 deletions(-) diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 53adfc848612..ec4a290a296b 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -261,6 +261,7 @@ let config = :: {local = ".gitlab/ci/jobs/shared/templates.yml"; rules = []} :: includes in + Pipeline.write () ; [ Workflow workflow; Default default; diff --git a/ci/bin/tezos_ci.ml b/ci/bin/tezos_ci.ml index 411658f972f9..5fe34f097aa2 100644 --- a/ci/bin/tezos_ci.ml +++ b/ci/bin/tezos_ci.ml @@ -33,12 +33,18 @@ module Pipeline = struct name : string; if_ : Gitlab_ci.If.t; variables : Gitlab_ci.Types.variables option; + jobs : Gitlab_ci.Types.job list; } let pipelines : t list ref = ref [] - let register ?variables name if_ = - let pipeline : t = {variables; if_; name} in + let filename : name:string -> string = + fun ~name -> sf ".gitlab/ci/pipelines/%s.yml" name + + let register ?variables ?(jobs = []) name if_ = + let pipeline : t = {variables; if_; name; jobs} in + (* TODO: https://gitlab.com/tezos/tezos/-/issues/7015 + check that stages have not been crossed. *) if List.exists (fun {name = name'; _} -> name' = name) !pipelines then failwith "[Pipeline.register] attempted to register pipeline %S twice" @@ -47,10 +53,100 @@ module Pipeline = struct let all () = List.rev !pipelines + (* Perform a set of static checks on the full pipeline before writing it. *) + let precheck {name = pipeline_name; jobs; _} = + let job_by_name : (string, Gitlab_ci.Types.job) Hashtbl.t = + Hashtbl.create 5 + in + (* Populate [job_by_name] and check that no two different jobs have the same name. *) + List.iter + (fun (job : Gitlab_ci.Types.job) -> + match Hashtbl.find_opt job_by_name job.name with + | None -> Hashtbl.add job_by_name job.name job + | Some _ -> + failwith + "[%s] the job '%s' is included twice" + pipeline_name + job.name) + jobs ; + (* Check usage of [needs:] & [depends:] *) + Fun.flip List.iter jobs @@ fun job -> + (* Get the [needs:] / [dependencies:] of job *) + let opt_set l = String_set.of_list (Option.value ~default:[] l) in + let needs = + (* The mandatory set of needs *) + job.needs |> Option.value ~default:[] + |> List.filter_map (fun Gitlab_ci.Types.{job; optional} -> + if not optional then Some job else None) + |> String_set.of_list + in + let dependencies = opt_set job.dependencies in + (* Check that dependencies are a subset of needs. + Note: this is already enforced by the smart constructor {!job} + defined below. Is it redundant? Nothing enforces the usage of + this smart constructor at this point.*) + String_set.iter + (fun dependency -> + if not (String_set.mem dependency needs) then + failwith + "[%s] the job '%s' has a [dependency:] on '%s' which is not \ + included in it's [need:]" + pipeline_name + job.name + dependency) + dependencies ; + (* Check that needed jobs (which thus includes dependencies) are defined *) + ( Fun.flip String_set.iter needs @@ fun need -> + match Hashtbl.find_opt job_by_name need with + | Some _needed_job -> + (* TODO: https://gitlab.com/tezos/tezos/-/issues/7015 + check rule implication *) + () + | None -> + (* TODO: https://gitlab.com/tezos/tezos/-/issues/7015 + handle optional needs *) + failwith + "[%s] job '%s' has a need on '%s' which is not defined in this \ + pipeline." + pipeline_name + job.name + need ) ; + (* Check that all [dependencies:] are on jobs that produce artifacts *) + ( Fun.flip String_set.iter dependencies @@ fun dependency -> + match Hashtbl.find_opt job_by_name dependency with + | Some {artifacts = Some {paths = _ :: _; _}; _} + | Some {artifacts = Some {reports = Some {dotenv = Some _; _}; _}; _} -> + (* This is fine: we depend on a job that define non-report artifacts, or a dotenv file. *) + () + | Some _ -> + failwith + "[%s] the job '%s' has a [dependency:] on '%s' which produces \ + neither regular, [paths:] artifacts or a dotenv report." + pipeline_name + job.name + dependency + | None -> + (* This case is precluded by the dependency analysis above. *) + assert false ) ; + + () + + let write () = + all () + |> List.iter @@ fun ({name; jobs; _} as pipeline) -> + if not (Sys.getenv_opt "CI_DISABLE_PRECHECK" = Some "true") then + precheck pipeline ; + match jobs with + | [] -> () + | _ :: _ -> + let filename = filename ~name in + let config = List.map (fun j -> Gitlab_ci.Types.Job j) jobs in + Gitlab_ci.To_yaml.to_file ~header ~filename config + let workflow_includes () : Gitlab_ci.Types.workflow * Gitlab_ci.Types.include_ list = let workflow_rule_of_pipeline = function - | {name; if_; variables} -> + | {name; if_; variables; jobs = _} -> (* Add [PIPELINE_TYPE] to the variables of the workflow rules, so that it can be added to the pipeline [name] *) let variables = @@ -59,13 +155,12 @@ module Pipeline = struct workflow_rule ~if_ ~variables ~when_:Always () in let include_of_pipeline = function - | {name; if_; variables = _} -> + | {name; if_; variables = _; jobs = _} -> (* Note that variables associated to the pipeline are not set in the include rule, they are set in the workflow rule *) let rule = include_rule ~if_ ~when_:Always () in - Gitlab_ci.Types. - {local = sf ".gitlab/ci/pipelines/%s.yml" name; rules = [rule]} + Gitlab_ci.Types.{local = filename ~name; rules = [rule]} in let pipelines = all () in let workflow = diff --git a/ci/bin/tezos_ci.mli b/ci/bin/tezos_ci.mli index d8e261685e31..50721847630d 100644 --- a/ci/bin/tezos_ci.mli +++ b/ci/bin/tezos_ci.mli @@ -36,14 +36,23 @@ module Pipeline : sig (* Register a pipeline. [register ?variables name rule] will register a pipeline [name] - that runs when [rule] is true. The pipeline is expected to be - defined in [.gitlab/ci/pipelines/NAME.yml] which will be included - from the top-level [.gitlab-ci.yml]. + that runs when [rule] is true. If [variables] is set, then these variables will be added to the - [workflow:] clause for this pipeline in the top-level [.gitlab-ci.yml]. *) + [workflow:] clause for this pipeline in the top-level [.gitlab-ci.yml]. + + If [jobs] is not set, then the pipeline is a legacy, hand-written + .yml file, expected to be defined in + [.gitlab/ci/pipelines/NAME.yml]. If [jobs] is set, then the those + jobs will be generated to the same file when {!write} is + called. In both cases, this file will be included from the + top-level [.gitlab-ci.yml]. *) val register : - ?variables:Gitlab_ci.Types.variables -> string -> Gitlab_ci.If.t -> unit + ?variables:Gitlab_ci.Types.variables -> + ?jobs:Gitlab_ci.Types.job list -> + string -> + Gitlab_ci.If.t -> + unit (** Splits the set of registered pipelines into workflow rules and includes. @@ -52,6 +61,11 @@ module Pipeline : sig and to include the select pipeline (using [include:]). *) val workflow_includes : unit -> Gitlab_ci.Types.workflow * Gitlab_ci.Types.include_ list + + (** Writes the set of non-legacy registered pipelines. + + The string {!header} will be prepended to each written file. *) + val write : unit -> unit end (** A facility for registering images for [image:] keywords. -- GitLab From e1ca4352ec2975e7905ec46609837a25d3c177ef Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Tue, 27 Feb 2024 13:08:12 +0100 Subject: [PATCH 2/3] CI-in-OCaml: generate [{before_,after_,}script] in logical order Makes it easier to read generated YAML. --- ci/lib_gitlab_ci/to_yaml.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/lib_gitlab_ci/to_yaml.ml b/ci/lib_gitlab_ci/to_yaml.ml index 9340e99d813b..97cdd86df5a1 100644 --- a/ci/lib_gitlab_ci/to_yaml.ml +++ b/ci/lib_gitlab_ci/to_yaml.ml @@ -226,9 +226,9 @@ let enc_job : job -> value = opt "timeout" enc_time_interval timeout; opt "cache" (array1 enc_cache) cache; opt "interruptible" bool interruptible; + opt "before_script" strings before_script; key "script" strings script; opt "after_script" strings after_script; - opt "before_script" strings before_script; opt "services" enc_services services; opt "variables" enc_variables variables; opt "artifacts" enc_artifacts artifacts; -- GitLab From 1a89ac6fea4e4f005910c449ea702c6bb3975f54 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 5 Jan 2024 18:20:01 +0100 Subject: [PATCH 3/3] CI: generate [latest_release] and [latest_release_test] pipelines --- .../docker:promote_to_latest-release.yml | 9 ----- .../publish/docker:promote_to_latest-test.yml | 9 ----- .gitlab/ci/pipelines/latest_release.yml | 21 ++++++++++-- .gitlab/ci/pipelines/latest_release_test.yml | 21 ++++++++++-- ci/bin/main.ml | 33 +++++++++++++++++-- 5 files changed, 66 insertions(+), 27 deletions(-) delete mode 100644 .gitlab/ci/jobs/publish/docker:promote_to_latest-release.yml delete mode 100644 .gitlab/ci/jobs/publish/docker:promote_to_latest-test.yml diff --git a/.gitlab/ci/jobs/publish/docker:promote_to_latest-release.yml b/.gitlab/ci/jobs/publish/docker:promote_to_latest-release.yml deleted file mode 100644 index f89939c10ddb..000000000000 --- a/.gitlab/ci/jobs/publish/docker:promote_to_latest-release.yml +++ /dev/null @@ -1,9 +0,0 @@ -docker:promote_to_latest: - extends: - - .docker_auth_template - - .image_template__docker - stage: publish_release - variables: - CI_DOCKER_HUB: "true" - script: - - ./scripts/ci/docker_promote_to_latest.sh diff --git a/.gitlab/ci/jobs/publish/docker:promote_to_latest-test.yml b/.gitlab/ci/jobs/publish/docker:promote_to_latest-test.yml deleted file mode 100644 index bc9b55848b63..000000000000 --- a/.gitlab/ci/jobs/publish/docker:promote_to_latest-test.yml +++ /dev/null @@ -1,9 +0,0 @@ -docker:promote_to_latest: - extends: - - .docker_auth_template - - .image_template__docker - stage: publish_release - variables: - CI_DOCKER_HUB: "false" - script: - - ./scripts/ci/docker_promote_to_latest.sh diff --git a/.gitlab/ci/pipelines/latest_release.yml b/.gitlab/ci/pipelines/latest_release.yml index 31411020d226..b685d917addd 100644 --- a/.gitlab/ci/pipelines/latest_release.yml +++ b/.gitlab/ci/pipelines/latest_release.yml @@ -1,3 +1,18 @@ -include: - # Stage: publish_release - - .gitlab/ci/jobs/publish/docker:promote_to_latest-release.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +docker:promote_to_latest: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: publish_release + tags: + - gcp + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_promote_to_latest.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "true" diff --git a/.gitlab/ci/pipelines/latest_release_test.yml b/.gitlab/ci/pipelines/latest_release_test.yml index 6e17a2fd4374..274e0428053e 100644 --- a/.gitlab/ci/pipelines/latest_release_test.yml +++ b/.gitlab/ci/pipelines/latest_release_test.yml @@ -1,3 +1,18 @@ -include: - # Stage: publish_release - - .gitlab/ci/jobs/publish/docker:promote_to_latest-test.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +docker:promote_to_latest: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: publish_release + tags: + - gcp + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_promote_to_latest.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" diff --git a/ci/bin/main.ml b/ci/bin/main.ml index ec4a290a296b..1a8ad4d7d7ab 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -41,7 +41,7 @@ module Stages = struct let _publish_release_gitlab = Stage.register "publish_release_gitlab" - let _publish_release = Stage.register "publish_release" + let publish_release = Stage.register "publish_release" let _publish_package_gitlab = Stage.register "publish_package_gitlab" @@ -166,7 +166,7 @@ module Images = struct For more info, see {{:https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-socket-binding}} here. This image is defined in {{:https://gitlab.com/tezos/docker-images/ci-docker}tezos/docker-images/ci-docker}. *) - let _docker = + let docker = Image.register ~name:"docker" ~image_path:"${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0" @@ -193,6 +193,31 @@ let job_dummy : job = ~script:[{|echo "This job will never execute"|}] () +(** Helper to create jobs that uses the docker deamon. + + It: + - Sets the appropriate image. + - Activates the Docker daemon as a service. + - It sets up authentification with docker registries *) +let job_docker_authenticated ?variables ~stage ~name script : job = + let docker_version = "24.0.6" in + job + ~image:Images.docker + ~variables: + ([("DOCKER_VERSION", docker_version)] @ Option.value ~default:[] variables) + ~before_script:["./scripts/ci/docker_initialize.sh"] + ~services:[{name = "docker:${DOCKER_VERSION}-dind"}] + ~stage + ~name + script + +let job_docker_promote_to_latest ~ci_docker_hub : job = + job_docker_authenticated + ~stage:Stages.publish_release + ~name:"docker:promote_to_latest" + ~variables:[("CI_DOCKER_HUB", Bool.to_string ci_docker_hub)] + ["./scripts/ci/docker_promote_to_latest.sh"] + (* Register pipelines types. Pipelines types are used to generate workflow rules and includes of the files where the jobs of the pipeline is defined. At the moment, all these pipelines are defined @@ -215,10 +240,12 @@ let () = register "before_merging" If.(on_tezos_namespace && merge_request) ; register "latest_release" + ~jobs:[job_docker_promote_to_latest ~ci_docker_hub:true] If.(on_tezos_namespace && push && on_branch "latest-release") ; register "latest_release_test" - If.(not_on_tezos_namespace && push && on_branch "latest-release-test") ; + If.(not_on_tezos_namespace && push && on_branch "latest-release-test") + ~jobs:[job_docker_promote_to_latest ~ci_docker_hub:false] ; register "master_branch" If.(on_tezos_namespace && push && on_branch "master") ; register "release_tag" -- GitLab