diff --git a/ci/bin/common.ml b/ci/bin/common.ml index ed962b89dc15e57229816de4b5b0208646e8b1f9..02e379875bc5441faa97327b72e045bc247a1109 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -754,8 +754,8 @@ let changeset_mir_tzt = [CI_DOCKER_AUTH] contains the appropriate credentials. *) let job_docker_authenticated ?(skip_docker_initialization = false) ?ci_docker_hub ?artifacts ?(variables = []) ?rules ?dependencies - ?image_dependencies ?arch ?tag ?allow_failure ?parallel ?retry ~__POS__ - ~stage ~name script : tezos_job = + ?image_dependencies ?arch ?tag ?allow_failure ?parallel ?retry ?description + ~__POS__ ~stage ~name script : tezos_job = let docker_version = "24.0.7" in job ?rules @@ -767,6 +767,7 @@ let job_docker_authenticated ?(skip_docker_initialization = false) ?allow_failure ?parallel ?retry + ?description ~__POS__ ~image:Images_external.docker ~variables: @@ -807,6 +808,7 @@ module Images = struct ~__POS__ ~stage ~name:"oc.docker:client-libs-dependencies" + ~description:"Build internal client-libs-dependencies images" (* This image is not built for external use. *) ~ci_docker_hub:false (* Handle docker initialization, if necessary, in [./scripts/ci/docker_client_libs_dependencies_build.sh]. *) @@ -834,6 +836,8 @@ module Images = struct ~skip_docker_initialization:true ~stage ~name:("oc.docker:rust-toolchain:" ^ arch_to_string_alt arch) + ~description: + ("Build internal rust-toolchain images for " ^ arch_to_string_alt arch) ~ci_docker_hub:false ~artifacts: (artifacts @@ -885,6 +889,7 @@ module Images = struct ~skip_docker_initialization:true ~stage ~name:("oc.docker:ci:" ^ arch_to_string_alt arch) + ~description:("Build internal CI images for " ^ arch_to_string_alt arch) ~ci_docker_hub:false ~artifacts: (artifacts ~reports:(reports ~dotenv:"ci_image_tag.env" ()) []) diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 7f44f3cfbfc1879b525a51e906a8ccc6171e202a..0c7f84f65b88a0c96274c50ed82636178706048d 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -323,3 +323,5 @@ let () = ~exclude:exclude_fun () | List_pipelines -> Pipeline.list_pipelines () + | Overview_pipelines -> Pipeline.overview_pipelines () + | Describe_pipeline {name} -> Pipeline.describe_pipeline name diff --git a/ci/bin/release_tag.ml b/ci/bin/release_tag.ml index 7a39808e656351685b7be6b3440aa3f23e0e28d7..ff475f736530ad4fee82be36b78a01499e667efd 100644 --- a/ci/bin/release_tag.ml +++ b/ci/bin/release_tag.ml @@ -198,6 +198,13 @@ let octez_jobs ?(test = false) release_tag_pipeline_type = ~__POS__ ~image:Images.CI.prebuild ~stage:Stages.publish_release + ~description: + "Update opam package descriptions on tezos/tezos opam-repository fork.\n\n\ + This job does preliminary work for releasing Octez opam packages on \ + opam repository, by pushing a branch with updated package \ + descriptions (.opam files) to \ + https://github.com/tezos/opam-repository. It _does not_ automatically \ + create a corresponding pull request on the official opam repository." ~interruptible:false ?variables ~name:"opam:release" @@ -302,6 +309,7 @@ let octez_evm_node_jobs ?(test = false) () = ~interruptible:false ~dependencies ~name:"gitlab:octez-evm-node-release" + ~description:"Create a GitLab release for Etherlink" ["./scripts/ci/create_gitlab_octez_evm_node_release.sh"] in [ diff --git a/ci/bin/tezos_ci.ml b/ci/bin/tezos_ci.ml index 0f9de57b3c9805da24389686a8295d06b6e4ee77..a58c989bda0db75d0e85dee06ebc8051d78d5d90 100644 --- a/ci/bin/tezos_ci.ml +++ b/ci/bin/tezos_ci.ml @@ -3,7 +3,11 @@ open Gitlab_ci.Util let failwith fmt = Format.kasprintf (fun s -> failwith s) fmt module Cli = struct - type action = Write | List_pipelines + type action = + | Write + | Overview_pipelines + | List_pipelines + | Describe_pipeline of {name : string} type config = { mutable verbose : bool; @@ -48,9 +52,15 @@ module Cli = struct ( "--remove-extra-files", Arg.Unit (fun () -> config.remove_extra_files <- true), " Remove files that are neither generated nor excluded." ); + ( "--overview-pipelines", + Arg.Unit (fun () -> config.action <- Overview_pipelines), + " List registered pipelines." ); ( "--list-pipelines", Arg.Unit (fun () -> config.action <- List_pipelines), " List registered pipelines." ); + ( "--describe-pipeline", + Arg.String (fun name -> config.action <- Describe_pipeline {name}), + "NAME Describe a pipeline." ); ] in Arg.parse @@ -93,6 +103,7 @@ type tezos_image = and tezos_job = { job : Gitlab_ci.Types.generic_job; source_position : string * int * int * int; + description : string option; stage : Stage.t; image_builders : tezos_job list; } @@ -376,6 +387,17 @@ module Pipeline = struct let includes = List.map include_of_pipeline pipelines in (workflow, includes) + let stages pipeline = + let jobs = jobs pipeline in + let stages : (string, int) Hashtbl.t = Hashtbl.create 5 in + List.iter + (fun job -> + Hashtbl.replace stages (Stage.name job.stage) (Stage.index job.stage)) + jobs ; + Hashtbl.to_seq stages |> List.of_seq + |> List.sort (fun (_n1, idx1) (_n2, idx2) -> Int.compare idx1 idx2) + |> List.map fst + let write ?default ?variables ~filename () = (* Write all registered pipelines *) ( Fun.flip List.iter (all ()) @@ fun pipeline -> @@ -399,19 +421,6 @@ module Pipeline = struct name filename) jobs ; - let stages = - let stages : (string, int) Hashtbl.t = Hashtbl.create 5 in - List.iter - (fun job -> - Hashtbl.replace - stages - (Stage.name job.stage) - (Stage.index job.stage)) - jobs ; - Hashtbl.to_seq stages |> List.of_seq - |> List.sort (fun (_n1, idx1) (_n2, idx2) -> Int.compare idx1 idx2) - |> List.map fst - in let prepend_config = (match pipeline with | Pipeline _ -> [] @@ -426,7 +435,7 @@ module Pipeline = struct }; Variables [("PIPELINE_TYPE", name)]; ]) - @ [Stages stages] + @ [Stages (stages pipeline)] in let config = prepend_config @ List.concat_map tezos_job_to_config_elements jobs @@ -566,6 +575,103 @@ module Pipeline = struct printf "@,@[<2> %a@]@,@," pp_print_text (description pipeline) ) ; close_box () ; print_flush () + + let markdown_table fmt ~headers ~rows = + let cells = List.length headers in + ( Fun.flip List.iteri rows @@ fun idx row -> + if List.length row <> cells then + raise + (Invalid_argument + (sf + "Row %d should have exactly as many cells as the header row: \ + %d, but has %d." + (idx + 1) + (List.length row) + cells)) ) ; + let cell_sizes = + List.fold_left + (fun max_sizes row -> + let row_sizes = List.map String.length row in + List.map2 Int.max max_sizes row_sizes) + (List.map String.length headers) + rows + in + let open Printf in + let print_row row = + fprintf fmt "|" ; + ( Fun.flip List.iteri row @@ fun idx cell -> + fprintf + fmt + " %s%s |" + cell + (String.make (List.nth cell_sizes idx - String.length cell) ' ') ) ; + fprintf fmt "\n" + in + (* make header *) + print_row headers ; + (* make separator between header and rows *) + print_row (Fun.flip List.map cell_sizes @@ fun size -> String.make size '-') ; + (* print rows *) + Fun.flip List.iter rows print_row + + let shorten_description description = + description |> String.split_on_char '\n' |> function + | [] -> description + | l :: _ -> l + + let overview_pipelines () = + markdown_table + stdout + ~headers:["PIPELINE"; "DESCRIPTION"; "JOBS"] + ~rows: + ( Fun.flip List.map (all ()) @@ fun pipeline -> + let description = shorten_description @@ description pipeline in + [ + name pipeline; + description; + List.length (jobs pipeline) |> string_of_int; + ] ) + + let describe_pipeline pipeline_name = + let pipelines = all () in + match + List.find_opt (fun pipeline -> name pipeline = pipeline_name) pipelines + with + | Some pipeline -> + let pipeline = add_image_builders pipeline in + let jobs = jobs pipeline in + let name = name pipeline in + let filename = path ~name in + let open Format in + open_vbox 0 ; + printf "%s (%s): %d jobs@," name filename (List.length jobs) ; + printf "@,@[%a@]@,@," pp_print_text (description pipeline) ; + close_box () ; + print_flush () ; + print_endline "Jobs in this pipeline:\n" ; + markdown_table + stdout + ~headers:["STAGE"; "JOB"; "DESCRIPTION"] + ~rows: + ( Fun.flip List.concat_map (stages pipeline) @@ fun stage -> + let jobs_of_stage = + List.filter (fun job -> Stage.name job.stage = stage) jobs + in + Fun.flip List.map jobs_of_stage @@ fun job -> + let description = + match job.description with + | None -> "" + | Some d -> shorten_description d + in + [stage; name_of_tezos_job job; description] ) + | None -> + Printf.eprintf + "Pipeline '%s' does not exist.\n\nTry one of:\n" + pipeline_name ; + List.iter + (fun pipeline -> Printf.eprintf "- %s\n" (name pipeline)) + pipelines ; + exit 1 end module Image = struct @@ -733,8 +839,8 @@ let enc_git_strategy = function let job ?arch ?after_script ?allow_failure ?artifacts ?before_script ?cache ?interruptible ?(dependencies = Staged []) ?(image_dependencies = []) ?services ?variables ?rules ?(timeout = Gitlab_ci.Types.Minutes 60) ?tag - ?git_strategy ?coverage ?retry ?parallel ~__POS__ ~image ~stage ~name script - : tezos_job = + ?git_strategy ?coverage ?retry ?parallel ?description ~__POS__ ~image ~stage + ~name script : tezos_job = (* The tezos/tezos CI uses singleton tags for its runners. *) let tag = match (arch, tag) with @@ -865,9 +971,9 @@ let job ?arch ?after_script ?allow_failure ?artifacts ?before_script ?cache parallel; } in - {job = Job job; source_position = __POS__; stage; image_builders} + {job = Job job; source_position = __POS__; description; stage; image_builders} -let trigger_job ?(dependencies = Staged []) ?rules ~__POS__ ~stage +let trigger_job ?(dependencies = Staged []) ?rules ?description ~__POS__ ~stage Pipeline. { name = child_pipeline_name; @@ -894,6 +1000,7 @@ let trigger_job ?(dependencies = Staged []) ?rules ~__POS__ ~stage in { job = Trigger_job trigger_job; + description; source_position = __POS__; stage; image_builders = []; diff --git a/ci/bin/tezos_ci.mli b/ci/bin/tezos_ci.mli index 6130ce40919cfa884848d01d3d65a9f87ce10356..fac10fc19410f3636e68efaa2fa1c33d7269fe55 100644 --- a/ci/bin/tezos_ci.mli +++ b/ci/bin/tezos_ci.mli @@ -42,7 +42,10 @@ module Cli : sig (** Action the binary should perform. *) type action = | Write (** Write the CI configuration *) + | Overview_pipelines (** Print pipelines as table. *) | List_pipelines (** List registered pipelines. *) + | Describe_pipeline of {name : string} + (** Describe a registered pipeline. *) (** Type of the command-line configuration for the generator binary. *) type config = { @@ -152,8 +155,17 @@ module Pipeline : sig unit -> unit - (** Pretty prints the set of registered pipelines. *) + (** Outputs the set of registered pipelines with their description. *) val list_pipelines : unit -> unit + + (** Pretty prints the set of registered pipelines as a table. *) + val overview_pipelines : unit -> unit + + (** Describe the registered pipeline of a given name. + + If no such pipeline is registered, [exit 1] is called and an + error message is printed. *) + val describe_pipeline : string -> unit end (** A facility for registering images for [image:] keywords. @@ -362,6 +374,7 @@ val job : ?coverage:string -> ?retry:Gitlab_ci.Types.retry -> ?parallel:Gitlab_ci.Types.parallel -> + ?description:string -> __POS__:string * int * int * int -> image:Image.t -> stage:Stage.t -> @@ -375,6 +388,7 @@ val job : val trigger_job : ?dependencies:dependencies -> ?rules:Gitlab_ci.Types.job_rule list -> + ?description:string -> __POS__:string * int * int * int -> stage:Stage.t -> Pipeline.child_pipeline ->