diff --git a/ci/bin/release_tag.ml b/ci/bin/release_tag.ml index ae781bda534b287903597e857a291bad8c710420..fb78f72c5e99f8f8b95c9549fe87c5a19d983720 100644 --- a/ci/bin/release_tag.ml +++ b/ci/bin/release_tag.ml @@ -287,11 +287,25 @@ let octez_jobs ?(test = false) ?(major = true) release_tag_pipeline_type = ] @ jobs_debian_repository @ jobs_dnf_repository (* Include components release jobs only if this is a major release. *) - @ (if major then - let dry_run = test && release_tag_pipeline_type == Schedule_test in - Grafazos.Release.jobs ~test ~dry_run () - @ Teztale.Release.jobs ~test ~dry_run () - else []) + @ (if not major then [] + else + match (test, release_tag_pipeline_type) with + | false, (Release_tag | Beta_release_tag | Non_release_tag) -> + !Tezos_ci.Hooks.global_release + @ Grafazos.Release.jobs ~test:false ~dry_run:false () + @ Teztale.Release.jobs ~test:false ~dry_run:false () + | true, (Release_tag | Beta_release_tag | Non_release_tag) -> + !Tezos_ci.Hooks.global_test_release + @ Grafazos.Release.jobs ~test:true ~dry_run:false () + @ Teztale.Release.jobs ~test:true ~dry_run:false () + | true, Schedule_test -> + !Tezos_ci.Hooks.global_scheduled_test_release + @ Grafazos.Release.jobs ~test:true ~dry_run:true () + @ Teztale.Release.jobs ~test:true ~dry_run:true () + | false, Schedule_test -> + failwith + "test = false is inconsistent with release_tag_pipeline_type = \ + Schedule_test") @ match (test, release_tag_pipeline_type) with (* for the moment the apt repository are not official, so we do not add to the release diff --git a/ci/lib_cacio/cacio.ml b/ci/lib_cacio/cacio.ml new file mode 100644 index 0000000000000000000000000000000000000000..ace8f2c0fa1151fe0035a6ee9c292f87df5ea477 --- /dev/null +++ b/ci/lib_cacio/cacio.ml @@ -0,0 +1,570 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2025 Nomadic Labs. *) +(* *) +(*****************************************************************************) + +open Gitlab_ci.Base + +type stage = Build | Test | Publish + +(* Should actually be equivalent to [Stdlib.compare] + if stages are defined in the right order. + But this function is used to check job dependencies, + so the order cannot be arbitrary. + It is thus safer to be explicit instead of relying on the compiler's internals. + Also, if one adds a stage, the compiler will attract the attention to this function, + and thus this comment. *) +let compare_stages a b = + match (a, b) with + | Build, Build -> 0 + | Build, (Test | Publish) -> -1 + | Test, Build -> 1 + | Test, Test -> 0 + | Test, Publish -> -1 + | Publish, (Build | Test) -> 1 + | Publish, Publish -> 0 + +let show_stage = function + | Build -> "build" + | Test -> "test" + | Publish -> "publish" + +type need = Job | Artifacts + +type job = { + uid : int; + source_location : string * int * int * int; + name : string; + stage : stage; + description : string; + image : Tezos_ci.Image.t; + needs : (need * job) list; + needs_legacy : (need * Tezos_ci.tezos_job) list; + only_if_changed : Tezos_ci.Changeset.t; + variables : Gitlab_ci.Types.variables option; + script : string list; + artifacts : Gitlab_ci.Types.artifacts option; +} + +type trigger = Auto | Manual + +let fresh_uid = + let last = ref (-1) in + fun () -> + incr last ; + !last + +module UID_set = Set.Make (Int) +module UID_map = Map.Make (Int) + +(* GRAPH TRANSFORMATIONS + + Step 1: the user specifies a list of jobs, as well as how they want those jobs + to be triggered. This list thus have type [(trigger * job) list]. + + Step 2: the [make_graph] function converts this list into a graph, of type [job_graph]. + The main differences with the original list are: + - a [job_graph] is not a list, but a map from job UID, + making it more efficient to work with; + - a [job_graph] is transitively closed, i.e. all dependencies are explicitly part + of the graph even if they were not explicitly given in the original list. + Additionally, [make_graph] checks that the same job does not appear twice + in the original list with different triggers. + + Step 3: the [fix_graph] function converts the [job_graph] into a [fixed_job_graph]. + This function fixes triggers and changesets. + By "fix" we mean not only that we set them to corrected values, + but also that these corrected values are a fixpoint. + More precisely: + - triggers are fixed so that if a job has trigger [Auto], + then all of its dependencies also have trigger [Auto]; + - changesets are fixed so that if changing a file causes a job to be included, + then changing this file also causes all dependencies of this job to be included. + + At this point, we took the *intent* of the user (given in step 1), + and corrected it to something that actually makes sense. + In particular this prevents some cases of invalid pipelines. + + Step 4: the [convert_graph] function converts the [fixed_job_graph] + into a [tezos_job_graph]. This just converts all jobs to [Tezos_ci.tezos_job], + that can be used by CIAO. + + Those successive transformations are driven by the [convert_jobs] function, + which is used by all functions that take Cacio jobs and register them as CIAO jobs. *) + +type job_graph_node = { + job : job; + trigger : trigger option; + rev_deps : UID_set.t; (* UIDs of jobs that directly depend on [job]. *) +} + +type job_graph = job_graph_node UID_map.t + +let error_s (file, line, start_char, end_char) message = + (* We start with [\n] because Dune's progress bar messes with the output. *) + Printf.eprintf + "\nFile %S, line %d, characters %d-%d:\nError: %s\n%!" + file + line + start_char + end_char + message ; + exit 1 + +let error pos x = Printf.ksprintf (error_s pos) x + +(* Compute the transitive closure of [jobs] as a [job_graph]. + Also makes sure that the same job is not requested twice with different triggers. + See GRAPH TRANSFORMATIONS above (step 2). *) +let make_graph (jobs : (trigger * job) list) : job_graph = + (* [add_job] adds [job] to [acc], as well as its dependencies, recursively. + [acc] is the resulting graph being built. + [add_job] also adds [rev_deps] to the list of reverse dependencies of [job]. + [trigger] is the trigger to use for [job]; it can be: + - [Some _] if the trigger was specified by the user, from the [jobs] list; + - [None] if the job was added automatically as a dependency.*) + let rec add_job ~trigger ~rev_deps acc job = + (* Add dependencies of [job], with at least [job] as reverse dependency. *) + let acc : job_graph_node UID_map.t = + List.fold_left + (add_job ~trigger:None ~rev_deps:(UID_set.singleton job.uid)) + acc + (List.map snd job.needs) + in + (* Add [job], with at least [rev_deps] as reverse dependency. *) + let update = function + | None -> + (* [job] is not in the graph yet, just add it. *) + Some {job; trigger; rev_deps} + | Some {job; trigger = old_trigger; rev_deps = old_rev_deps} -> + (* [job] is already in the graph, update it. *) + (* Make sure the triggers are compatible. *) + let trigger = + match (old_trigger, trigger) with + | x, None | None, x -> + (* If both triggers are [None], it means the job has been added twice + as a dependency; we don't know the triggers of its reverse dependencies + yet so we keep [None]. + If one of the trigger is [None] and the other is [Some _], + it means the job was both specified by the user (with a trigger), + and added automatically. In this case we keep the trigger specified + by the user. *) + x + | Some old_trigger, Some trigger -> + (* The job was specified twice in [jobs]. + This is suspicious but not a problem + as long as the triggers are the same. *) + if old_trigger <> trigger then + error + job.source_location + "job %S is listed twice in the same pipeline but with \ + different triggers" + job.name ; + Some trigger + in + (* A job can be added multiple times because it is the dependency + of multiple reverse dependencies. + Make sure all those reverse dependencies are stored. *) + let rev_deps = UID_set.union rev_deps old_rev_deps in + Some {job; trigger; rev_deps} + in + UID_map.update job.uid update acc + in + (* Start from the empty graph and use [add_jobs] to add all [jobs]. *) + List.fold_left + (fun acc (trigger, job) -> + add_job ~trigger:(Some trigger) ~rev_deps:UID_set.empty acc job) + UID_map.empty + jobs + +type fixed_job_graph_node = { + job : job; + trigger : trigger; + only_if_changed : Tezos_ci.Changeset.t; +} + +type fixed_job_graph = fixed_job_graph_node UID_map.t + +(* Compute the final values for [only_if_changed] and [trigger]s. + Assumes there are no cycles. + See GRAPH TRANSFORMATIONS above (step 3). *) +let fix_graph (graph : job_graph) : fixed_job_graph = + (* To build the graph, we take all jobs from [graph], fix them, + and add them to [result]. + But before we fix a job, we need the [trigger] and [only_if_changed] + of its reverse dependencies. + So we need to fix those reverse dependencies first. + [fix_uid] takes the UID of a job, fixes and adds its reverse dependencies recursively, + then fixes this job and adds it to [result]. *) + let result : fixed_job_graph ref = ref UID_map.empty in + let rec fix_uid uid = + match UID_map.find_opt uid !result with + | Some result_node -> + (* Job already visited as dependency of another job. *) + result_node + | None -> + let result_node = + match UID_map.find_opt uid graph with + | None -> + (* Not supposed to happen. *) + assert false + | Some {job; trigger; rev_deps} -> + (* Fix and add reverse dependencies recursively. *) + let rev_deps = rev_deps |> UID_set.elements |> List.map fix_uid in + (* Fix [trigger]. *) + let trigger = + let merge_triggers a b = + match (a, b) with + | Manual, Manual -> Manual + | Auto, (Auto | Manual) | Manual, Auto -> + (* If a job is supposed to run automatically, + its dependencies must run automatically as well. *) + Auto + in + let initial_trigger = + match trigger with + | None -> + (* We don't know yet. + [Manual] will be upgraded to [Auto] if necessary. *) + Manual + | Some trigger -> trigger + in + rev_deps + |> List.map (fun node -> node.trigger) + |> List.fold_left merge_triggers initial_trigger + in + (* Fix the changeset, i.e. add the union of the changesets + of reverse dependencies. *) + let only_if_changed = + rev_deps + |> List.map (fun node -> node.only_if_changed) + |> List.fold_left Tezos_ci.Changeset.union job.only_if_changed + in + {job; trigger; only_if_changed} + in + result := UID_map.add uid result_node !result ; + result_node + in + (* Make sure all jobs from the input [graph] are visited at least once. *) + UID_map.iter + (fun uid _ -> + let _ : fixed_job_graph_node = fix_uid uid in + ()) + graph ; + !result + +let convert_stage (stage : stage) : Tezos_ci.Stage.t = + match stage with + | Build -> Tezos_ci.Stages.build + | Test -> Tezos_ci.Stages.test + | Publish -> Tezos_ci.Stages.publish + +type tezos_job_graph = Tezos_ci.tezos_job UID_map.t + +(* Convert jobs to [Tezos_ci] jobs. + See GRAPH TRANSFORMATIONS above (step 4). + + If [with_changes] is [true], the job [rules] will contain a [changes] clause. + If it is [false], [only_if_changed] is ignored. + [changes] clauses are typically only used in merge request pipelines + such as [before_merging]. *) +let convert_graph ~with_changes (graph : fixed_job_graph) : tezos_job_graph = + (* To build the graph, we take all jobs from [graph], convert them, + and add them to [result]. + But before we convert a job, we need the converted version of its dependencies. + So we need to fix those dependencies first. + [convert_uid] takes the UID of a job, converts its dependencies recursively, + then converts this job and adds it to [result]. *) + let result = ref UID_map.empty in + let rec convert_uid uid = + match UID_map.find_opt uid !result with + | Some result_node -> + (* Job already visited as dependency of another job. *) + result_node + | None -> + let result_node = + match UID_map.find_opt uid graph with + | None -> + (* Not supposed to happen. *) + assert false + | Some + { + job = + { + uid = _; + source_location; + name; + stage; + description; + image; + needs; + needs_legacy; + only_if_changed = _; + variables; + script; + artifacts; + }; + trigger; + only_if_changed; + } -> + (* Convert dependencies recursively. *) + let dependencies = + let needs = + needs_legacy + @ List.map + (fun (need, dep) -> (need, convert_uid dep.uid)) + needs + in + Fun.flip List.map needs @@ fun (need, dep) -> + match need with + | Job -> Tezos_ci.Job dep + | Artifacts -> Tezos_ci.Artifacts dep + in + (* Compute [rules] from on the job's fixed changeset, + whether we actually want the [changes] clause ([with_changes]), + and the [trigger]. *) + let rules = + if with_changes then + [ + Gitlab_ci.Util.job_rule + ~changes:(Tezos_ci.Changeset.encode only_if_changed) + ~when_: + (match trigger with + | Auto -> On_success + | Manual -> Manual) + (); + ] + else + match trigger with + | Auto -> [] + | Manual -> [Gitlab_ci.Util.job_rule ~when_:Manual ()] + in + let interruptible = + match stage with Build | Test -> true | Publish -> false + in + Tezos_ci.job + ~__POS__:source_location + ~name + ~stage:(convert_stage stage) + ~description + ~image + ~dependencies:(Dependent dependencies) + ~rules + ~interruptible + ?variables + ?artifacts + script + in + result := UID_map.add uid result_node !result ; + result_node + in + (* Make sure all jobs from the input [graph] are visited at least once. *) + UID_map.iter + (fun uid _ -> + let _ : Tezos_ci.tezos_job = convert_uid uid in + ()) + graph ; + !result + +(* Convert user-specified jobs into [Tezos_ci] jobs. + See GRAPH TRANSFORMATIONS. *) +let convert_jobs ~with_changes (jobs : (trigger * job) list) : + Tezos_ci.tezos_job list = + jobs |> make_graph |> fix_graph + |> convert_graph ~with_changes + |> UID_map.bindings |> List.map snd + +let parameterize make = + let table = Hashtbl.create 8 in + fun value -> + match Hashtbl.find_opt table value with + | None -> + let result = make value in + Hashtbl.add table value result ; + result + | Some result -> result + +module type COMPONENT = sig + val name : string + + val paths : string list +end + +module type COMPONENT_API = sig + val job : + __POS__:string * int * int * int -> + stage:stage -> + description:string -> + image:Tezos_ci.Image.t -> + ?needs:(need * job) list -> + ?needs_legacy:(need * Tezos_ci.tezos_job) list -> + ?variables:Gitlab_ci.Types.variables -> + ?artifacts:Gitlab_ci.Types.artifacts -> + string -> + string list -> + job + + val register_before_merging_jobs : (trigger * job) list -> unit + + val register_master_jobs : (trigger * job) list -> unit + + val register_scheduled_pipeline : + description:string -> string -> (trigger * job) list -> unit + + val register_global_release_jobs : (trigger * job) list -> unit + + val register_global_test_release_jobs : (trigger * job) list -> unit + + val register_global_scheduled_test_release_jobs : (trigger * job) list -> unit + + val register_dedicated_release_pipeline : (trigger * job) list -> unit + + val register_dedicated_test_release_pipeline : (trigger * job) list -> unit +end + +(* We could avoid using a functor if we required the user of this module + to pass the component's [name] and [paths] to the functions that need them, + but in practice this would be less convenient since all functions need at least + one of them. *) +module Make (Component : COMPONENT) : COMPONENT_API = struct + let only_if_changed = Tezos_ci.Changeset.make Component.paths + + let job ~__POS__:source_location ~stage ~description ~image ?(needs = []) + ?(needs_legacy = []) ?variables ?artifacts name script = + let name = Component.name ^ "." ^ name in + (* Check that no dependency is in an ulterior stage. *) + ( Fun.flip List.iter needs @@ fun (_, dep) -> + if compare_stages dep.stage stage > 0 then + error + source_location + "job %s, which is in stage '%s', cannot depend on %s, which is in \ + ulterior stage '%s'" + name + (show_stage stage) + dep.name + (show_stage dep.stage) ) ; + { + uid = fresh_uid (); + source_location; + name; + stage; + description; + image; + needs; + needs_legacy; + only_if_changed; + variables; + script; + artifacts; + } + + let register_before_merging_jobs jobs = + (* Add [trigger] as a dependency of all [jobs]. *) + let jobs = + (* The actual [trigger] job is defined deep inside [code_verification.ml] + (look for [job_start]). CIAO only really cares about the name of the job, + so we hackishly redefine it here. *) + let job_trigger = + Tezos_ci.job ~__POS__ ~stage:Tezos_ci.Stages.start ~name:"trigger" [] + in + (* Here we re-allocate the jobs with the same UID to override [needs_legacy]. + But only these re-allocated jobs will be in the pipeline, + so there is no risk of actually duplicating them. *) + Fun.flip List.map jobs @@ fun (need, job) -> + let job = + {job with needs_legacy = (Job, job_trigger) :: job.needs_legacy} + in + (need, job) + in + let jobs = convert_jobs ~with_changes:true jobs in + Tezos_ci.Hooks.before_merging := jobs @ !Tezos_ci.Hooks.before_merging + + let register_master_jobs jobs = + let jobs = convert_jobs ~with_changes:false jobs in + Tezos_ci.Hooks.master := jobs @ !Tezos_ci.Hooks.master + + let full_pipeline_name name = sf "%s.%s" Component.name name + + (* Helper for other functions below, that is not exposed to users of this module. + It is responsible for: + - prefixing the name of the pipeline with the name of the component; + - including the DataDog job. + Returns the pipeline name. *) + let register_pipeline ~description ~jobs name rules = + Tezos_ci.Pipeline.register + ~description + ~jobs:(Tezos_ci.job_datadog_pipeline_trace :: jobs) + (full_pipeline_name name) + rules + + let register_scheduled_pipeline ~description name jobs = + register_pipeline + name + ~description + ~jobs:(convert_jobs ~with_changes:false jobs) + Tezos_ci.Rules.( + Gitlab_ci.If.( + scheduled && var "TZ_SCHEDULE_KIND" == str (full_pipeline_name name))) + + let register_global_release_jobs jobs = + let jobs = convert_jobs ~with_changes:false jobs in + Tezos_ci.Hooks.global_release := jobs @ !Tezos_ci.Hooks.global_release + + let register_global_test_release_jobs jobs = + let jobs = convert_jobs ~with_changes:false jobs in + Tezos_ci.Hooks.global_test_release := + jobs @ !Tezos_ci.Hooks.global_test_release + + let register_global_scheduled_test_release_jobs jobs = + let jobs = convert_jobs ~with_changes:false jobs in + Tezos_ci.Hooks.global_scheduled_test_release := + jobs @ !Tezos_ci.Hooks.global_scheduled_test_release + + (* Use this function to get the release tag regular expression, + as it makes sure that it is registered with [Hooks.release_tags] only once. *) + let get_release_tag_rex = + let tag = ref None in + fun () -> + match !tag with + | Some tag -> tag + | None -> + let result = + "/^" ^ String.lowercase_ascii Component.name ^ "-v\\d+\\.\\d+$/" + in + tag := Some result ; + Tezos_ci.Hooks.release_tags := result :: !Tezos_ci.Hooks.release_tags ; + result + + (* Wrap a function with this function to make sure it is called only once. *) + let only_once name f = + let already_called = ref false in + fun x -> + if !already_called then + error __POS__ "component %s called %s twice" Component.name name ; + already_called := true ; + f x + + let register_dedicated_release_pipeline = + only_once "register_dedicated_release_pipeline" @@ fun jobs -> + let release_tag_rex = get_release_tag_rex () in + register_pipeline + "release" + ~description:(sf "Release %s." Component.name) + ~jobs:(convert_jobs ~with_changes:false jobs) + Tezos_ci.Rules.( + Gitlab_ci.If.( + on_tezos_namespace && push && has_tag_match release_tag_rex)) + + let register_dedicated_test_release_pipeline = + only_once "register_dedicated_test_release_pipeline" @@ fun jobs -> + let release_tag_rex = get_release_tag_rex () in + register_pipeline + "test_release" + ~description:(sf "Release %s (test)." Component.name) + ~jobs:(convert_jobs ~with_changes:false jobs) + Tezos_ci.Rules.( + Gitlab_ci.If.( + not_on_tezos_namespace && push && has_tag_match release_tag_rex)) +end diff --git a/ci/lib_cacio/cacio.mli b/ci/lib_cacio/cacio.mli new file mode 100644 index 0000000000000000000000000000000000000000..8b734cd75f4879d888e647dff6f920ca1cfb5a4f --- /dev/null +++ b/ci/lib_cacio/cacio.mli @@ -0,0 +1,270 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2025 Nomadic Labs. *) +(* *) +(*****************************************************************************) + +(** Pipeline stages. + + The purpose of [Build] jobs is to produce artifacts for other jobs. + They can depend on other build jobs. + If there is no job that uses the artifacts of a build job, + the build job should not be included. + If you intend to include a build job as a test, + consider having it in the [Test] stage instead, + or add some actual tests that use the artifacts from the build job. + + The purpose of [Test] jobs is to make sure that pipelines fail if something looks wrong. + They typically use artifacts from build jobs. + Test jobs should only depend on build jobs that build the artifacts that they are testing. + Test jobs usually do not depend on other test jobs, + with the exception of jobs that aggregate debug artifacts. + Test jobs typically produce debug artifacts such as reports and logs, + but not artifacts that are used by other jobs, + except jobs that aggregate debug artifacts. + + The purpose of [Publish] jobs is to publish artifacts from build jobs + somewhere more convenient than GitLab artifacts. + Publish jobs should only depend on build jobs that build the artifacts + that they are publishing. They can also depend on test jobs to prevent + publishing broken artifacts, and on other publish jobs if order matters. + + There is no need to include [Build] jobs explicitly in pipelines. + One should only list [Test] and [Publish] jobs and let Cacio automatically + include the relevant build jobs that they depend on. + This reflects the fact that the purpose of build jobs is only to + make artifacts for other jobs. *) +type stage = Build | Test | Publish + +(** Dependency relationships. + + - [Job]: do not download artifacts of the dependency. + A typical use case is to prevent [Publish] jobs from running + before some [Test] jobs succeed, without losing time downloading test reports. + + - [Artifacts]: download artifacts of the dependency. *) +type need = Job | Artifacts + +(** Pipeline jobs. + + Jobs are basically code to run in a given pipeline stage. *) +type job + +(** How a job is triggered. + + - [Auto]: the job is triggered automatically once all of its dependencies succeed. + This includes the [trigger] job for pipelines that have one. + - [Manual]: the job can be triggered manually once all of its dependencies succeed. + + If a job [dep] is needed by a job with trigger [Auto], + the trigger of [dep] is automatically forced to [Auto]. + + If a job [dep] is added to a pipeline automatically + because it is the dependency of other jobs, + and if all those other jobs have trigger [Manual], + [dep] is added with trigger [Manual]. + Otherwise it is added with trigger [Auto]. *) +type trigger = Auto | Manual + +(** Memoize a function. + + The intended use case is to parameterize jobs without duplicating them by mistake. + For instance: + {[ + let my_job = + parameterize @@ fun stage -> + parameterize @@ fun image -> + job ~stage ~image ... + ]} + + [my_job] always returns the same job when given the same parameters. + This means that you can apply [my_job] in multiple places with the same parameters + without duplicating the job in the CI. + + If you instead defined [my_job] as a function directly, like this: + {[ + let my_job stage image = + job ~stage ~image ... + ]} + then the risk of duplicating jobs by mistake would be really high. + For instance, if you wrote [~needs: [my_job Build fedora_37]] + in two places to include this job as a dependency of two jobs in the same pipeline, + this dependency would be included twice, since each application of [my_job] + would return a different job (even if those different jobs had the same name). + + Note that [parameterize] does not automatically parameterize a job's name. + If you intend to include multiple instances of [my_job] in the same pipeline, + you should compute a name that differs according to the parameters. + [parameterize] knows nothing about jobs. + + The parameter type ['a] must be hashable by [Hashtbl]. *) +val parameterize : ('a -> 'b) -> 'a -> 'b + +module type COMPONENT = sig + (** Description of a component (argument of the {!Make} functor). *) + + (** Component name. + + This is added as a prefix to jobs and pipelines of this component. + It should thus be short. + + Release tags are prefixed by the lowercase version of [name]. + So names must be valid identifiers. *) + val name : string + + (** Files that belong to the component. + + This is added as [changes] clauses for jobs of this component + in the [before_merging] pipeline. *) + val paths : string list +end + +module type COMPONENT_API = sig + (** Functions that components can use to define their CI (result of the {!Make} functor). *) + + (** Define a job. + + This function must only be called once per job. See {!parameterize}. + + Usage: + {[ + job name + ~__POS__ + ~stage: ... + ~description: ... + ~image: ... + [ + (* script *) + ] + ]} + + This defines a job named [name] prefixed by the name of the component. + The job can then be added to multiple pipelines using other functions + from this module. + + Jobs themselves do not carry their [!trigger]. + Instead, each pipeline can decide to add jobs with a different trigger. + + When added to the [before_merging] pipeline, + jobs are given a [changes] clause which is the union of: + - the component [paths]; + - the [changes] clauses of reverse dependencies. + + The job will not start before all [needs] and [needs_legacy] jobs succeed. + Additionally, [needs] are automatically added to pipelines in which the job + is present if they are not already present. + [needs_legacy] allows to make an incremental transition to Cacio, + but [needs_legacy] jobs are not automatically added to pipelines, + contrary to [needs] jobs. + + See {!Tezos_ci.job} for information about other arguments. *) + val job : + __POS__:string * int * int * int -> + stage:stage -> + description:string -> + image:Tezos_ci.Image.t -> + ?needs:(need * job) list -> + ?needs_legacy:(need * Tezos_ci.tezos_job) list -> + ?variables:Gitlab_ci.Types.variables -> + ?artifacts:Gitlab_ci.Types.artifacts -> + string -> + string list -> + job + + (** Register jobs to be included in [before_merging] and [merge_train] pipelines. *) + val register_before_merging_jobs : (trigger * job) list -> unit + + (** Register jobs to be included in [master_branch] pipelines. *) + val register_master_jobs : (trigger * job) list -> unit + + (** Register a scheduled pipeline for this component. + + Usage: [register_scheduled_pipeline ~description name jobs] + + [name] can typically be ["daily"] or ["weekly"]. + The actual name of the pipeline will automatically be prefixed + by the name of the component. + + This does not actually register the pipeline in GitLab. + The pipeline must manually be set up in GitLab + to run with variable [TZ_SCHEDULE_KIND] equal to the name of the pipeline + (including the prefix, which is the component's name). *) + val register_scheduled_pipeline : + description:string -> string -> (trigger * job) list -> unit + + (** {2 Releases} *) + + (** This section contains functions to define jobs for release pipelines. + In the future, these functions may be replaced by something more abstract. + (See the future work section at the end of the file.) *) + + (** Register jobs to be included in the global release pipeline. + + This pipeline is for major releases and includes all components. + It runs in [tezos/tezos]. *) + val register_global_release_jobs : (trigger * job) list -> unit + + (** Register jobs to be included in the global test release pipeline. + + This pipeline is supposed to be very close to the global release pipeline. + It is not run in [tezos/tezos] but in forks. *) + val register_global_test_release_jobs : (trigger * job) list -> unit + + (** Register jobs to be included in the global scheduled test release pipeline. + + This pipeline is supposed to be close to the global release pipeline, + except that it should not actually create the release. + It runs in [tezos/tezos]. *) + val register_global_scheduled_test_release_jobs : (trigger * job) list -> unit + + (** Register jobs to be included in the release pipeline of the current component. + + This pipeline is for releasing this component separately. + It runs in [tezos/tezos]. + + This function must be called only once per component. *) + val register_dedicated_release_pipeline : (trigger * job) list -> unit + + (** Register jobs to be included in the test release pipeline of the current component. + + This pipeline is for testing the release of this component separately. + It runs in [tezos/tezos]. + + This function must be called only once per component. *) + val register_dedicated_test_release_pipeline : (trigger * job) list -> unit +end + +(** The main functor of Cacio. *) +module Make (_ : COMPONENT) : COMPONENT_API + +(** {2 Future work} *) + +(** One idea would be to have Cacio provide a dedicated function to create Tezt jobs. + This would automatically cause the relevant pipelines with those jobs + to have a [select_tezts] job. + It could assume that the [main.ml] is located in [component/tezt] + (although this requires to change [COMPONENT] to be able to identify the main path). *) + +(** Ideally, all components would only have a single path (their toplevel directory). + For now, we choose to allow multiple paths, so that: + - one can migrate old components more easily; + - one can continue to run tests of a component if one of its dependencies is modified + (by including the paths of these dependencies in the list of paths for + this component). *) + +(** Instead of having one [register_*_release_jobs] function per release pipeline, + it would be better if components only declared WHAT they want to release, not HOW. + Components could give a list of build jobs, and for each of them they could specify + which artifacts are meant to be published, as well as which Docker images. + It should be possible to generate everything else just from this information. + This abstraction could guarantee important invariants like: + - a test release must not create a GitLab release on [tezos/tezos]; + - there is only one job that creates a GitLab release in each pipeline + (unless we actually want one per component as well, in which case + the invariant becomes: there is exactly one job per component, plus a global one); + - our release page is only updated by actual release pipelines + (other pipelines can update the test release page instead); + - Docker images are only pushed to Docker Hub in actual release pipelines, + not test release pipelines (those can use the GitLab registry instead for instance); + - ... *) diff --git a/ci/lib_cacio/dune b/ci/lib_cacio/dune new file mode 100644 index 0000000000000000000000000000000000000000..5e9a4dae78754afb1747bf5859410afa29c49392 --- /dev/null +++ b/ci/lib_cacio/dune @@ -0,0 +1,8 @@ +; This file was automatically generated, do not edit. +; Edit file manifest/main.ml instead. + +(library + (name cacio) + (libraries + gitlab_ci + tezos_ci)) diff --git a/ci/lib_tezos_ci/tezos_ci.ml b/ci/lib_tezos_ci/tezos_ci.ml index db3e8e9c5c19d36ab87466c0f2c9ee226ada0422..c93a82b5c1cda82f049a1fc50e8348d8c24a8ffc 100644 --- a/ci/lib_tezos_ci/tezos_ci.ml +++ b/ci/lib_tezos_ci/tezos_ci.ml @@ -1738,6 +1738,12 @@ module Hooks = struct let master = ref [] + let global_release = ref [] + + let global_test_release = ref [] + + let global_scheduled_test_release = ref [] + let release_tags = ref [] end diff --git a/ci/lib_tezos_ci/tezos_ci.mli b/ci/lib_tezos_ci/tezos_ci.mli index 36c7a9096404d986e984cd050c6e3d79a36fee36..a983cac13a5fc6458f434f205a4e4b26446d47e8 100644 --- a/ci/lib_tezos_ci/tezos_ci.mli +++ b/ci/lib_tezos_ci/tezos_ci.mli @@ -715,6 +715,15 @@ module Hooks : sig (** Jobs to add to [master] branch pipelines. *) val master : tezos_job list ref + (** Jobs to add to the global release pipeline. *) + val global_release : tezos_job list ref + + (** Jobs to add to the global test release pipeline. *) + val global_test_release : tezos_job list ref + + (** Jobs to add to the global scheduled test release pipeline. *) + val global_scheduled_test_release : tezos_job list ref + (** Regular expressions that match release tags. Used by [ci/bin/main.ml] to define the [non_release_tag] diff --git a/manifest/product_ciao.ml b/manifest/product_ciao.ml index 464fa09817fe6eaf5c2c629cfc10b42c1bb95b4c..838a009037c7519b314446a1ca32e656feb11768 100644 --- a/manifest/product_ciao.ml +++ b/manifest/product_ciao.ml @@ -35,6 +35,15 @@ let ci_lib_tezos_ci = ~deps:[ci_lib_gitlab_ci_main |> open_ ~m:"Base"] ~release_status:Unreleased +let _ci_lib_cacio = + private_lib + "cacio" + ~opam:"" + ~path:"ci/lib_cacio" + ~bisect_ppx:No + ~deps:[ci_lib_gitlab_ci_main; ci_lib_tezos_ci] + ~release_status:Unreleased + let ci_grafazos = private_lib "grafazos"