diff --git a/.gitlab/ci/jobs/publish/docker:promote_to_latest-release.yml b/.gitlab/ci/jobs/publish/docker:promote_to_latest-release.yml deleted file mode 100644 index f89939c10ddbf124d992b8082c1deb4929e3a140..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/publish/docker:promote_to_latest-release.yml +++ /dev/null @@ -1,9 +0,0 @@ -docker:promote_to_latest: - extends: - - .docker_auth_template - - .image_template__docker - stage: publish_release - variables: - CI_DOCKER_HUB: "true" - script: - - ./scripts/ci/docker_promote_to_latest.sh diff --git a/.gitlab/ci/jobs/publish/docker:promote_to_latest-test.yml b/.gitlab/ci/jobs/publish/docker:promote_to_latest-test.yml deleted file mode 100644 index bc9b55848b63a51e84d86f2f8fc64749f3b26025..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/publish/docker:promote_to_latest-test.yml +++ /dev/null @@ -1,9 +0,0 @@ -docker:promote_to_latest: - extends: - - .docker_auth_template - - .image_template__docker - stage: publish_release - variables: - CI_DOCKER_HUB: "false" - script: - - ./scripts/ci/docker_promote_to_latest.sh diff --git a/.gitlab/ci/pipelines/latest_release.yml b/.gitlab/ci/pipelines/latest_release.yml index 31411020d226ea0941da4054d98867ed7b9b4180..b685d917addd86cf503ea7a1562ead1c18be2dad 100644 --- a/.gitlab/ci/pipelines/latest_release.yml +++ b/.gitlab/ci/pipelines/latest_release.yml @@ -1,3 +1,18 @@ -include: - # Stage: publish_release - - .gitlab/ci/jobs/publish/docker:promote_to_latest-release.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +docker:promote_to_latest: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: publish_release + tags: + - gcp + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_promote_to_latest.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "true" diff --git a/.gitlab/ci/pipelines/latest_release_test.yml b/.gitlab/ci/pipelines/latest_release_test.yml index 6e17a2fd4374ef687c4202e8f7fc8ba5e4877cb7..274e0428053ee2b566bde1b4fa09916775b0b556 100644 --- a/.gitlab/ci/pipelines/latest_release_test.yml +++ b/.gitlab/ci/pipelines/latest_release_test.yml @@ -1,3 +1,18 @@ -include: - # Stage: publish_release - - .gitlab/ci/jobs/publish/docker:promote_to_latest-test.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. + +docker:promote_to_latest: + image: ${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0 + stage: publish_release + tags: + - gcp + dependencies: [] + before_script: + - ./scripts/ci/docker_initialize.sh + script: + - ./scripts/ci/docker_promote_to_latest.sh + services: + - docker:${DOCKER_VERSION}-dind + variables: + DOCKER_VERSION: 24.0.6 + CI_DOCKER_HUB: "false" diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 53adfc8486121817db0de3cb3137b64f77bbb4ec..1a8ad4d7d7ab436091238f78466e2e7d0246ae4b 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -41,7 +41,7 @@ module Stages = struct let _publish_release_gitlab = Stage.register "publish_release_gitlab" - let _publish_release = Stage.register "publish_release" + let publish_release = Stage.register "publish_release" let _publish_package_gitlab = Stage.register "publish_package_gitlab" @@ -166,7 +166,7 @@ module Images = struct For more info, see {{:https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-socket-binding}} here. This image is defined in {{:https://gitlab.com/tezos/docker-images/ci-docker}tezos/docker-images/ci-docker}. *) - let _docker = + let docker = Image.register ~name:"docker" ~image_path:"${CI_REGISTRY}/tezos/docker-images/ci-docker:v1.9.0" @@ -193,6 +193,31 @@ let job_dummy : job = ~script:[{|echo "This job will never execute"|}] () +(** Helper to create jobs that uses the docker deamon. + + It: + - Sets the appropriate image. + - Activates the Docker daemon as a service. + - It sets up authentification with docker registries *) +let job_docker_authenticated ?variables ~stage ~name script : job = + let docker_version = "24.0.6" in + job + ~image:Images.docker + ~variables: + ([("DOCKER_VERSION", docker_version)] @ Option.value ~default:[] variables) + ~before_script:["./scripts/ci/docker_initialize.sh"] + ~services:[{name = "docker:${DOCKER_VERSION}-dind"}] + ~stage + ~name + script + +let job_docker_promote_to_latest ~ci_docker_hub : job = + job_docker_authenticated + ~stage:Stages.publish_release + ~name:"docker:promote_to_latest" + ~variables:[("CI_DOCKER_HUB", Bool.to_string ci_docker_hub)] + ["./scripts/ci/docker_promote_to_latest.sh"] + (* Register pipelines types. Pipelines types are used to generate workflow rules and includes of the files where the jobs of the pipeline is defined. At the moment, all these pipelines are defined @@ -215,10 +240,12 @@ let () = register "before_merging" If.(on_tezos_namespace && merge_request) ; register "latest_release" + ~jobs:[job_docker_promote_to_latest ~ci_docker_hub:true] If.(on_tezos_namespace && push && on_branch "latest-release") ; register "latest_release_test" - If.(not_on_tezos_namespace && push && on_branch "latest-release-test") ; + If.(not_on_tezos_namespace && push && on_branch "latest-release-test") + ~jobs:[job_docker_promote_to_latest ~ci_docker_hub:false] ; register "master_branch" If.(on_tezos_namespace && push && on_branch "master") ; register "release_tag" @@ -261,6 +288,7 @@ let config = :: {local = ".gitlab/ci/jobs/shared/templates.yml"; rules = []} :: includes in + Pipeline.write () ; [ Workflow workflow; Default default; diff --git a/ci/bin/tezos_ci.ml b/ci/bin/tezos_ci.ml index 411658f972f9102d5c086fe55f92435ddaec0103..5fe34f097aa2b8367faf091258d734fd504c21f4 100644 --- a/ci/bin/tezos_ci.ml +++ b/ci/bin/tezos_ci.ml @@ -33,12 +33,18 @@ module Pipeline = struct name : string; if_ : Gitlab_ci.If.t; variables : Gitlab_ci.Types.variables option; + jobs : Gitlab_ci.Types.job list; } let pipelines : t list ref = ref [] - let register ?variables name if_ = - let pipeline : t = {variables; if_; name} in + let filename : name:string -> string = + fun ~name -> sf ".gitlab/ci/pipelines/%s.yml" name + + let register ?variables ?(jobs = []) name if_ = + let pipeline : t = {variables; if_; name; jobs} in + (* TODO: https://gitlab.com/tezos/tezos/-/issues/7015 + check that stages have not been crossed. *) if List.exists (fun {name = name'; _} -> name' = name) !pipelines then failwith "[Pipeline.register] attempted to register pipeline %S twice" @@ -47,10 +53,100 @@ module Pipeline = struct let all () = List.rev !pipelines + (* Perform a set of static checks on the full pipeline before writing it. *) + let precheck {name = pipeline_name; jobs; _} = + let job_by_name : (string, Gitlab_ci.Types.job) Hashtbl.t = + Hashtbl.create 5 + in + (* Populate [job_by_name] and check that no two different jobs have the same name. *) + List.iter + (fun (job : Gitlab_ci.Types.job) -> + match Hashtbl.find_opt job_by_name job.name with + | None -> Hashtbl.add job_by_name job.name job + | Some _ -> + failwith + "[%s] the job '%s' is included twice" + pipeline_name + job.name) + jobs ; + (* Check usage of [needs:] & [depends:] *) + Fun.flip List.iter jobs @@ fun job -> + (* Get the [needs:] / [dependencies:] of job *) + let opt_set l = String_set.of_list (Option.value ~default:[] l) in + let needs = + (* The mandatory set of needs *) + job.needs |> Option.value ~default:[] + |> List.filter_map (fun Gitlab_ci.Types.{job; optional} -> + if not optional then Some job else None) + |> String_set.of_list + in + let dependencies = opt_set job.dependencies in + (* Check that dependencies are a subset of needs. + Note: this is already enforced by the smart constructor {!job} + defined below. Is it redundant? Nothing enforces the usage of + this smart constructor at this point.*) + String_set.iter + (fun dependency -> + if not (String_set.mem dependency needs) then + failwith + "[%s] the job '%s' has a [dependency:] on '%s' which is not \ + included in it's [need:]" + pipeline_name + job.name + dependency) + dependencies ; + (* Check that needed jobs (which thus includes dependencies) are defined *) + ( Fun.flip String_set.iter needs @@ fun need -> + match Hashtbl.find_opt job_by_name need with + | Some _needed_job -> + (* TODO: https://gitlab.com/tezos/tezos/-/issues/7015 + check rule implication *) + () + | None -> + (* TODO: https://gitlab.com/tezos/tezos/-/issues/7015 + handle optional needs *) + failwith + "[%s] job '%s' has a need on '%s' which is not defined in this \ + pipeline." + pipeline_name + job.name + need ) ; + (* Check that all [dependencies:] are on jobs that produce artifacts *) + ( Fun.flip String_set.iter dependencies @@ fun dependency -> + match Hashtbl.find_opt job_by_name dependency with + | Some {artifacts = Some {paths = _ :: _; _}; _} + | Some {artifacts = Some {reports = Some {dotenv = Some _; _}; _}; _} -> + (* This is fine: we depend on a job that define non-report artifacts, or a dotenv file. *) + () + | Some _ -> + failwith + "[%s] the job '%s' has a [dependency:] on '%s' which produces \ + neither regular, [paths:] artifacts or a dotenv report." + pipeline_name + job.name + dependency + | None -> + (* This case is precluded by the dependency analysis above. *) + assert false ) ; + + () + + let write () = + all () + |> List.iter @@ fun ({name; jobs; _} as pipeline) -> + if not (Sys.getenv_opt "CI_DISABLE_PRECHECK" = Some "true") then + precheck pipeline ; + match jobs with + | [] -> () + | _ :: _ -> + let filename = filename ~name in + let config = List.map (fun j -> Gitlab_ci.Types.Job j) jobs in + Gitlab_ci.To_yaml.to_file ~header ~filename config + let workflow_includes () : Gitlab_ci.Types.workflow * Gitlab_ci.Types.include_ list = let workflow_rule_of_pipeline = function - | {name; if_; variables} -> + | {name; if_; variables; jobs = _} -> (* Add [PIPELINE_TYPE] to the variables of the workflow rules, so that it can be added to the pipeline [name] *) let variables = @@ -59,13 +155,12 @@ module Pipeline = struct workflow_rule ~if_ ~variables ~when_:Always () in let include_of_pipeline = function - | {name; if_; variables = _} -> + | {name; if_; variables = _; jobs = _} -> (* Note that variables associated to the pipeline are not set in the include rule, they are set in the workflow rule *) let rule = include_rule ~if_ ~when_:Always () in - Gitlab_ci.Types. - {local = sf ".gitlab/ci/pipelines/%s.yml" name; rules = [rule]} + Gitlab_ci.Types.{local = filename ~name; rules = [rule]} in let pipelines = all () in let workflow = diff --git a/ci/bin/tezos_ci.mli b/ci/bin/tezos_ci.mli index d8e261685e310ed157a1d196896b42e8e375ef2e..50721847630d3c054c6a74fdcf54294d1d6cde13 100644 --- a/ci/bin/tezos_ci.mli +++ b/ci/bin/tezos_ci.mli @@ -36,14 +36,23 @@ module Pipeline : sig (* Register a pipeline. [register ?variables name rule] will register a pipeline [name] - that runs when [rule] is true. The pipeline is expected to be - defined in [.gitlab/ci/pipelines/NAME.yml] which will be included - from the top-level [.gitlab-ci.yml]. + that runs when [rule] is true. If [variables] is set, then these variables will be added to the - [workflow:] clause for this pipeline in the top-level [.gitlab-ci.yml]. *) + [workflow:] clause for this pipeline in the top-level [.gitlab-ci.yml]. + + If [jobs] is not set, then the pipeline is a legacy, hand-written + .yml file, expected to be defined in + [.gitlab/ci/pipelines/NAME.yml]. If [jobs] is set, then the those + jobs will be generated to the same file when {!write} is + called. In both cases, this file will be included from the + top-level [.gitlab-ci.yml]. *) val register : - ?variables:Gitlab_ci.Types.variables -> string -> Gitlab_ci.If.t -> unit + ?variables:Gitlab_ci.Types.variables -> + ?jobs:Gitlab_ci.Types.job list -> + string -> + Gitlab_ci.If.t -> + unit (** Splits the set of registered pipelines into workflow rules and includes. @@ -52,6 +61,11 @@ module Pipeline : sig and to include the select pipeline (using [include:]). *) val workflow_includes : unit -> Gitlab_ci.Types.workflow * Gitlab_ci.Types.include_ list + + (** Writes the set of non-legacy registered pipelines. + + The string {!header} will be prepended to each written file. *) + val write : unit -> unit end (** A facility for registering images for [image:] keywords. diff --git a/ci/lib_gitlab_ci/to_yaml.ml b/ci/lib_gitlab_ci/to_yaml.ml index 9340e99d813b9008fc0223372ca0fe36b2b7101b..97cdd86df5a162c9b980c39daa0d5010d1877472 100644 --- a/ci/lib_gitlab_ci/to_yaml.ml +++ b/ci/lib_gitlab_ci/to_yaml.ml @@ -226,9 +226,9 @@ let enc_job : job -> value = opt "timeout" enc_time_interval timeout; opt "cache" (array1 enc_cache) cache; opt "interruptible" bool interruptible; + opt "before_script" strings before_script; key "script" strings script; opt "after_script" strings after_script; - opt "before_script" strings before_script; opt "services" enc_services services; opt "variables" enc_variables variables; opt "artifacts" enc_artifacts artifacts;