From 4afff060805bcf7cb416f3ba3b13e885e29133e8 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 13 Sep 2024 16:27:27 +0200 Subject: [PATCH 1/4] CIAO: add description to jobs, add [--describe-pipeline] --- ci/bin/main.ml | 1 + ci/bin/tezos_ci.ml | 125 +++++++++++++++++++++++++++++++++++++------- ci/bin/tezos_ci.mli | 10 ++++ 3 files changed, 117 insertions(+), 19 deletions(-) diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 7f44f3cfbfc1..b9e208bcb5c0 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -323,3 +323,4 @@ let () = ~exclude:exclude_fun () | List_pipelines -> Pipeline.list_pipelines () + | Describe_pipeline {name} -> Pipeline.describe_pipeline name diff --git a/ci/bin/tezos_ci.ml b/ci/bin/tezos_ci.ml index 0f9de57b3c98..72cb29b13b8d 100644 --- a/ci/bin/tezos_ci.ml +++ b/ci/bin/tezos_ci.ml @@ -3,7 +3,7 @@ open Gitlab_ci.Util let failwith fmt = Format.kasprintf (fun s -> failwith s) fmt module Cli = struct - type action = Write | List_pipelines + type action = Write | List_pipelines | Describe_pipeline of {name : string} type config = { mutable verbose : bool; @@ -51,6 +51,9 @@ module Cli = struct ( "--list-pipelines", Arg.Unit (fun () -> config.action <- List_pipelines), " List registered pipelines." ); + ( "--describe-pipeline", + Arg.String (fun name -> config.action <- Describe_pipeline {name}), + "NAME Describe a pipeline." ); ] in Arg.parse @@ -93,6 +96,7 @@ type tezos_image = and tezos_job = { job : Gitlab_ci.Types.generic_job; source_position : string * int * int * int; + description : string option; stage : Stage.t; image_builders : tezos_job list; } @@ -376,6 +380,17 @@ module Pipeline = struct let includes = List.map include_of_pipeline pipelines in (workflow, includes) + let stages pipeline = + let jobs = jobs pipeline in + let stages : (string, int) Hashtbl.t = Hashtbl.create 5 in + List.iter + (fun job -> + Hashtbl.replace stages (Stage.name job.stage) (Stage.index job.stage)) + jobs ; + Hashtbl.to_seq stages |> List.of_seq + |> List.sort (fun (_n1, idx1) (_n2, idx2) -> Int.compare idx1 idx2) + |> List.map fst + let write ?default ?variables ~filename () = (* Write all registered pipelines *) ( Fun.flip List.iter (all ()) @@ fun pipeline -> @@ -399,19 +414,6 @@ module Pipeline = struct name filename) jobs ; - let stages = - let stages : (string, int) Hashtbl.t = Hashtbl.create 5 in - List.iter - (fun job -> - Hashtbl.replace - stages - (Stage.name job.stage) - (Stage.index job.stage)) - jobs ; - Hashtbl.to_seq stages |> List.of_seq - |> List.sort (fun (_n1, idx1) (_n2, idx2) -> Int.compare idx1 idx2) - |> List.map fst - in let prepend_config = (match pipeline with | Pipeline _ -> [] @@ -426,7 +428,7 @@ module Pipeline = struct }; Variables [("PIPELINE_TYPE", name)]; ]) - @ [Stages stages] + @ [Stages (stages pipeline)] in let config = prepend_config @ List.concat_map tezos_job_to_config_elements jobs @@ -566,6 +568,90 @@ module Pipeline = struct printf "@,@[<2> %a@]@,@," pp_print_text (description pipeline) ) ; close_box () ; print_flush () + + let markdown_table fmt ~headers ~rows = + let cells = List.length headers in + ( Fun.flip List.iteri rows @@ fun idx row -> + if List.length row <> cells then + raise + (Invalid_argument + (sf + "Row %d should have exactly as many cells as the header row: \ + %d, but has %d." + (idx + 1) + (List.length row) + cells)) ) ; + let cell_sizes = + List.fold_left + (fun max_sizes row -> + let row_sizes = List.map String.length row in + List.map2 Int.max max_sizes row_sizes) + (List.map String.length headers) + rows + in + let open Printf in + let print_row row = + fprintf fmt "|" ; + ( Fun.flip List.iteri row @@ fun idx cell -> + fprintf + fmt + " %s%s |" + cell + (String.make (List.nth cell_sizes idx - String.length cell) ' ') ) ; + fprintf fmt "\n" + in + (* make header *) + print_row headers ; + (* make separator between header and rows *) + print_row (Fun.flip List.map cell_sizes @@ fun size -> String.make size '-') ; + (* print rows *) + Fun.flip List.iter rows print_row + + let shorten_description description = + description |> String.split_on_char '\n' |> function + | [] -> description + | l :: _ -> l + + let describe_pipeline pipeline_name = + let pipelines = all () in + match + List.find_opt (fun pipeline -> name pipeline = pipeline_name) pipelines + with + | Some pipeline -> + let pipeline = add_image_builders pipeline in + let jobs = jobs pipeline in + let name = name pipeline in + let filename = path ~name in + let open Format in + open_vbox 0 ; + printf "%s (%s): %d jobs@," name filename (List.length jobs) ; + printf "@,@[%a@]@,@," pp_print_text (description pipeline) ; + close_box () ; + print_flush () ; + print_endline "Jobs in this pipeline:\n" ; + markdown_table + stdout + ~headers:["STAGE"; "JOB"; "DESCRIPTION"] + ~rows: + ( Fun.flip List.concat_map (stages pipeline) @@ fun stage -> + let jobs_of_stage = + List.filter (fun job -> Stage.name job.stage = stage) jobs + in + Fun.flip List.map jobs_of_stage @@ fun job -> + let description = + match job.description with + | None -> "" + | Some d -> shorten_description d + in + [stage; name_of_tezos_job job; description] ) + | None -> + Printf.eprintf + "Pipeline '%s' does not exist.\n\nTry one of:\n" + pipeline_name ; + List.iter + (fun pipeline -> Printf.eprintf "- %s\n" (name pipeline)) + pipelines ; + exit 1 end module Image = struct @@ -733,8 +819,8 @@ let enc_git_strategy = function let job ?arch ?after_script ?allow_failure ?artifacts ?before_script ?cache ?interruptible ?(dependencies = Staged []) ?(image_dependencies = []) ?services ?variables ?rules ?(timeout = Gitlab_ci.Types.Minutes 60) ?tag - ?git_strategy ?coverage ?retry ?parallel ~__POS__ ~image ~stage ~name script - : tezos_job = + ?git_strategy ?coverage ?retry ?parallel ?description ~__POS__ ~image ~stage + ~name script : tezos_job = (* The tezos/tezos CI uses singleton tags for its runners. *) let tag = match (arch, tag) with @@ -865,9 +951,9 @@ let job ?arch ?after_script ?allow_failure ?artifacts ?before_script ?cache parallel; } in - {job = Job job; source_position = __POS__; stage; image_builders} + {job = Job job; source_position = __POS__; description; stage; image_builders} -let trigger_job ?(dependencies = Staged []) ?rules ~__POS__ ~stage +let trigger_job ?(dependencies = Staged []) ?rules ?description ~__POS__ ~stage Pipeline. { name = child_pipeline_name; @@ -894,6 +980,7 @@ let trigger_job ?(dependencies = Staged []) ?rules ~__POS__ ~stage in { job = Trigger_job trigger_job; + description; source_position = __POS__; stage; image_builders = []; diff --git a/ci/bin/tezos_ci.mli b/ci/bin/tezos_ci.mli index 6130ce40919c..2ea947788a6d 100644 --- a/ci/bin/tezos_ci.mli +++ b/ci/bin/tezos_ci.mli @@ -43,6 +43,8 @@ module Cli : sig type action = | Write (** Write the CI configuration *) | List_pipelines (** List registered pipelines. *) + | Describe_pipeline of {name : string} + (** Describe a registered pipeline. *) (** Type of the command-line configuration for the generator binary. *) type config = { @@ -154,6 +156,12 @@ module Pipeline : sig (** Pretty prints the set of registered pipelines. *) val list_pipelines : unit -> unit + + (** Describe the registered pipeline of a given name. + + If no such pipeline is registered, [exit 1] is called and an + error message is printed. *) + val describe_pipeline : string -> unit end (** A facility for registering images for [image:] keywords. @@ -362,6 +370,7 @@ val job : ?coverage:string -> ?retry:Gitlab_ci.Types.retry -> ?parallel:Gitlab_ci.Types.parallel -> + ?description:string -> __POS__:string * int * int * int -> image:Image.t -> stage:Stage.t -> @@ -375,6 +384,7 @@ val job : val trigger_job : ?dependencies:dependencies -> ?rules:Gitlab_ci.Types.job_rule list -> + ?description:string -> __POS__:string * int * int * int -> stage:Stage.t -> Pipeline.child_pipeline -> -- GitLab From 08fb69cca4a73077d7dde3d4972d95b52ec39333 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Tue, 10 Sep 2024 12:03:02 +0200 Subject: [PATCH 2/4] CIAO: pipeline table, add [--overview-pipelines] --- ci/bin/main.ml | 1 + ci/bin/tezos_ci.ml | 22 +++++++++++++++++++++- ci/bin/tezos_ci.mli | 6 +++++- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/ci/bin/main.ml b/ci/bin/main.ml index b9e208bcb5c0..0c7f84f65b88 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -323,4 +323,5 @@ let () = ~exclude:exclude_fun () | List_pipelines -> Pipeline.list_pipelines () + | Overview_pipelines -> Pipeline.overview_pipelines () | Describe_pipeline {name} -> Pipeline.describe_pipeline name diff --git a/ci/bin/tezos_ci.ml b/ci/bin/tezos_ci.ml index 72cb29b13b8d..a58c989bda0d 100644 --- a/ci/bin/tezos_ci.ml +++ b/ci/bin/tezos_ci.ml @@ -3,7 +3,11 @@ open Gitlab_ci.Util let failwith fmt = Format.kasprintf (fun s -> failwith s) fmt module Cli = struct - type action = Write | List_pipelines | Describe_pipeline of {name : string} + type action = + | Write + | Overview_pipelines + | List_pipelines + | Describe_pipeline of {name : string} type config = { mutable verbose : bool; @@ -48,6 +52,9 @@ module Cli = struct ( "--remove-extra-files", Arg.Unit (fun () -> config.remove_extra_files <- true), " Remove files that are neither generated nor excluded." ); + ( "--overview-pipelines", + Arg.Unit (fun () -> config.action <- Overview_pipelines), + " List registered pipelines." ); ( "--list-pipelines", Arg.Unit (fun () -> config.action <- List_pipelines), " List registered pipelines." ); @@ -612,6 +619,19 @@ module Pipeline = struct | [] -> description | l :: _ -> l + let overview_pipelines () = + markdown_table + stdout + ~headers:["PIPELINE"; "DESCRIPTION"; "JOBS"] + ~rows: + ( Fun.flip List.map (all ()) @@ fun pipeline -> + let description = shorten_description @@ description pipeline in + [ + name pipeline; + description; + List.length (jobs pipeline) |> string_of_int; + ] ) + let describe_pipeline pipeline_name = let pipelines = all () in match diff --git a/ci/bin/tezos_ci.mli b/ci/bin/tezos_ci.mli index 2ea947788a6d..fac10fc19410 100644 --- a/ci/bin/tezos_ci.mli +++ b/ci/bin/tezos_ci.mli @@ -42,6 +42,7 @@ module Cli : sig (** Action the binary should perform. *) type action = | Write (** Write the CI configuration *) + | Overview_pipelines (** Print pipelines as table. *) | List_pipelines (** List registered pipelines. *) | Describe_pipeline of {name : string} (** Describe a registered pipeline. *) @@ -154,9 +155,12 @@ module Pipeline : sig unit -> unit - (** Pretty prints the set of registered pipelines. *) + (** Outputs the set of registered pipelines with their description. *) val list_pipelines : unit -> unit + (** Pretty prints the set of registered pipelines as a table. *) + val overview_pipelines : unit -> unit + (** Describe the registered pipeline of a given name. If no such pipeline is registered, [exit 1] is called and an -- GitLab From fb7df70658a71edea768f1bf7c7b7a2f11415080 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 13 Sep 2024 16:27:36 +0200 Subject: [PATCH 3/4] CI: set [description] of [opam:release, gitlab:octez-evm-node-release] --- ci/bin/common.ml | 6 ++++-- ci/bin/release_tag.ml | 8 ++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ci/bin/common.ml b/ci/bin/common.ml index ed962b89dc15..68485920df66 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -754,8 +754,8 @@ let changeset_mir_tzt = [CI_DOCKER_AUTH] contains the appropriate credentials. *) let job_docker_authenticated ?(skip_docker_initialization = false) ?ci_docker_hub ?artifacts ?(variables = []) ?rules ?dependencies - ?image_dependencies ?arch ?tag ?allow_failure ?parallel ?retry ~__POS__ - ~stage ~name script : tezos_job = + ?image_dependencies ?arch ?tag ?allow_failure ?parallel ?retry ?description + ~__POS__ ~stage ~name script : tezos_job = let docker_version = "24.0.7" in job ?rules @@ -767,6 +767,7 @@ let job_docker_authenticated ?(skip_docker_initialization = false) ?allow_failure ?parallel ?retry + ?description ~__POS__ ~image:Images_external.docker ~variables: @@ -885,6 +886,7 @@ module Images = struct ~skip_docker_initialization:true ~stage ~name:("oc.docker:ci:" ^ arch_to_string_alt arch) + ~description:("Build internal CI images for " ^ arch_to_string_alt arch) ~ci_docker_hub:false ~artifacts: (artifacts ~reports:(reports ~dotenv:"ci_image_tag.env" ()) []) diff --git a/ci/bin/release_tag.ml b/ci/bin/release_tag.ml index 7a39808e6563..ff475f736530 100644 --- a/ci/bin/release_tag.ml +++ b/ci/bin/release_tag.ml @@ -198,6 +198,13 @@ let octez_jobs ?(test = false) release_tag_pipeline_type = ~__POS__ ~image:Images.CI.prebuild ~stage:Stages.publish_release + ~description: + "Update opam package descriptions on tezos/tezos opam-repository fork.\n\n\ + This job does preliminary work for releasing Octez opam packages on \ + opam repository, by pushing a branch with updated package \ + descriptions (.opam files) to \ + https://github.com/tezos/opam-repository. It _does not_ automatically \ + create a corresponding pull request on the official opam repository." ~interruptible:false ?variables ~name:"opam:release" @@ -302,6 +309,7 @@ let octez_evm_node_jobs ?(test = false) () = ~interruptible:false ~dependencies ~name:"gitlab:octez-evm-node-release" + ~description:"Create a GitLab release for Etherlink" ["./scripts/ci/create_gitlab_octez_evm_node_release.sh"] in [ -- GitLab From a9c364535da97a72a6b00194088916e1847c3741 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 13 Sep 2024 16:12:29 +0200 Subject: [PATCH 4/4] CI: set [description] of [images] jobs --- ci/bin/common.ml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 68485920df66..02e379875bc5 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -808,6 +808,7 @@ module Images = struct ~__POS__ ~stage ~name:"oc.docker:client-libs-dependencies" + ~description:"Build internal client-libs-dependencies images" (* This image is not built for external use. *) ~ci_docker_hub:false (* Handle docker initialization, if necessary, in [./scripts/ci/docker_client_libs_dependencies_build.sh]. *) @@ -835,6 +836,8 @@ module Images = struct ~skip_docker_initialization:true ~stage ~name:("oc.docker:rust-toolchain:" ^ arch_to_string_alt arch) + ~description: + ("Build internal rust-toolchain images for " ^ arch_to_string_alt arch) ~ci_docker_hub:false ~artifacts: (artifacts -- GitLab