From b00b1ac3640bb52bf2c2ea450cfee4617ea4056e Mon Sep 17 00:00:00 2001
From: Arvid Jakobsson <arvid.jakobsson@nomadic-labs.com>
Date: Mon, 9 Sep 2024 11:34:10 +0200
Subject: [PATCH 1/9] ci/README.md: more details

---
 ci/README.md | 35 +++++++++++++++++++++++++++++------
 1 file changed, 29 insertions(+), 6 deletions(-)

diff --git a/ci/README.md b/ci/README.md
index 11abfb4c028c..ec7ef2e775e5 100644
--- a/ci/README.md
+++ b/ci/README.md
@@ -1,17 +1,40 @@
-# CI-in-OCaml
+# CIAO: CI-in-OCaml
 
 This directory contains an OCaml library for writing generators of
-GitLab CI configuration files (i.e. `.gitlab-ci.yml`).
+GitLab CI configuration files (i.e. `.gitlab-ci.yml`) and includes a
+generator for the tezos/tezos CI configuration.
 
-This directory is structured like this:
+If you need to update the CI configuration of the
+[tezos/tezos](https://gitlab.com/tezos/tezos) project, this is the
+directory to investigate.
+
+CIAO and the generator for tezos/tezos is structured like this:
 
  - `lib_gitlab_ci`: contains a partial, slightly opiniated, AST of
    [GitLab CI/CD YAML syntax](https://docs.gitlab.com/ee/ci/yaml/).
  - `bin`: contains a set of helpers for creating the Octez-specific
-   GitLab CI configuration files and an executable that generates part
-   of the CI configuration using those helpers.
+   GitLab CI configuration files (`Tezos_ci`, `Rules`) and an
+   executable that generates the tezos/tezos CI configuration using
+   those helpers.
+
+## The tezos/tezos CI configuration
+
+The tezos/tezos CI configuration consists of a set of *pipeline
+types*.  Pipelines of a given type run under a certain *condition* and
+perform a given set of automated checks or tasks. For instance:
+
+ - "before merging" pipelines run on merge requests and verify code
+   quality.
+ - "master branch" pipelines runs after the merge of a merge request
+   and publishes bleeding-edge artifacts (documentation, static
+   binaries, etc.)
+
+For the full set of pipelines, when they run and what they do, refer
+to `bin/main.ml`, where in they are all registered. The jobs of a
+given pipeline type is registered in separate modules (for instance,
+`master_branch.ml` for master branch pipelines).
 
-## Usage
+## CIAO usage
 
 To regenerate the GitLab CI configuration, run (from the root of the repo):
 
-- 
GitLab


From 61b2158464b54d2ef851a0532f1c15400b68fc6f Mon Sep 17 00:00:00 2001
From: Arvid Jakobsson <arvid.jakobsson@nomadic-labs.com>
Date: Wed, 24 Jul 2024 15:43:10 +0200
Subject: [PATCH 2/9] CIAO: refactor [ci/makefile] by extracting [manifest]
 target

---
 ci/makefile | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/ci/makefile b/ci/makefile
index 9430f45a4800..7aeef9b23954 100644
--- a/ci/makefile
+++ b/ci/makefile
@@ -1,18 +1,22 @@
 .PHONY: all
-all:  # Regenerate GitLab CI configuration.
-	cd .. && ${MAKE} -C manifest && . ./scripts/version.sh && dune exec ci/bin/main.exe
+all: manifest # Regenerate GitLab CI configuration.
+	cd .. && . ./scripts/version.sh && dune exec ci/bin/main.exe
 
 .PHONY: verbose
-verbose: # Regenerate GitLab CI configuration with verbose output.
-	cd .. && ${MAKE} -C manifest && . ./scripts/version.sh && dune exec ci/bin/main.exe -- --verbose
+verbose: manifest # Regenerate GitLab CI configuration with verbose output.
+	cd .. && . ./scripts/version.sh && dune exec ci/bin/main.exe -- --verbose
 
 .PHONY: inline-source-info
-inline-source-info: # Regenerate GitLab CI configuration, inlining source information in generated files.
-	cd .. && ${MAKE} -C manifest && . ./scripts/version.sh && dune exec ci/bin/main.exe -- --inline-source-info
+inline-source-info: manifest # Regenerate GitLab CI configuration, inlining source information in generated files.
+	cd .. && . ./scripts/version.sh && dune exec ci/bin/main.exe -- --inline-source-info
 
 .PHONY: remove-extra-files
-remove-extra-files: # Regenerate GitLab CI configuration and remove any non-generated files.
-	cd .. && ${MAKE} -C manifest && . ./scripts/version.sh && dune exec ci/bin/main.exe -- --remove-extra-files
+remove-extra-files: manifest # Regenerate GitLab CI configuration and remove any non-generated files.
+	cd .. && . ./scripts/version.sh && dune exec ci/bin/main.exe -- --remove-extra-files
+
+.PHONY: manifest
+manifest:
+	cd .. && ${MAKE} -C manifest
 
 .PHONY: docker-do-%
 docker-do-%:
-- 
GitLab


From e3bc949c3b68975570bfe7742cce2a8f6c5b8bb3 Mon Sep 17 00:00:00 2001
From: Arvid Jakobsson <arvid.jakobsson@nomadic-labs.com>
Date: Wed, 24 Jul 2024 15:43:32 +0200
Subject: [PATCH 3/9] CIAO: make manifest build [--silent] in [ci/makefile]

---
 ci/makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/makefile b/ci/makefile
index 7aeef9b23954..3b5e26c6f0b2 100644
--- a/ci/makefile
+++ b/ci/makefile
@@ -16,7 +16,7 @@ remove-extra-files: manifest # Regenerate GitLab CI configuration and remove any
 
 .PHONY: manifest
 manifest:
-	cd .. && ${MAKE} -C manifest
+	cd .. && ${MAKE} --silent -C manifest
 
 .PHONY: docker-do-%
 docker-do-%:
-- 
GitLab


From c8cde8b55f1e9b77ce19843e330f5faf1f95e0a2 Mon Sep 17 00:00:00 2001
From: Arvid Jakobsson <arvid.jakobsson@nomadic-labs.com>
Date: Tue, 10 Sep 2024 16:23:26 +0200
Subject: [PATCH 4/9] CIAO: add [--list-pipelines]

---
 ci/README.md        |  7 +++++++
 ci/bin/main.ml      |  7 +++++--
 ci/bin/tezos_ci.ml  | 49 +++++++++++++++++++++++++++++++++++++--------
 ci/bin/tezos_ci.mli | 17 +++++++++++++++-
 ci/makefile         |  7 +++++++
 5 files changed, 76 insertions(+), 11 deletions(-)

diff --git a/ci/README.md b/ci/README.md
index ec7ef2e775e5..e14ac7c0b382 100644
--- a/ci/README.md
+++ b/ci/README.md
@@ -65,3 +65,10 @@ required prerequisites installed.
 For more information, see
 
     dune exec bin/ci/main.exe -- --help
+
+## Inline documentation
+
+The set of registered pipelines types, and their documentation, can be
+accessed by:
+
+    make -s -C ci list-pipelines
diff --git a/ci/bin/main.ml b/ci/bin/main.ml
index d7aff42c4124..d19d2bed38ce 100644
--- a/ci/bin/main.ml
+++ b/ci/bin/main.ml
@@ -158,5 +158,8 @@ let () =
   (* If argument --verbose is set, then log generation info.
      If argument --inline-source, then print generation info in yml files. *)
   let filename = ".gitlab-ci.yml" in
-  Pipeline.write ~default ~variables ~filename () ;
-  Tezos_ci.check_files ~remove_extra_files:Cli.config.remove_extra_files ()
+  match Cli.config.action with
+  | Write ->
+      Pipeline.write ~default ~variables ~filename () ;
+      Tezos_ci.check_files ~remove_extra_files:Cli.config.remove_extra_files ()
+  | List_pipelines -> Pipeline.list_pipelines ()
diff --git a/ci/bin/tezos_ci.ml b/ci/bin/tezos_ci.ml
index 2d568afe6bd9..42c423dd2173 100644
--- a/ci/bin/tezos_ci.ml
+++ b/ci/bin/tezos_ci.ml
@@ -3,14 +3,22 @@ open Gitlab_ci.Util
 let failwith fmt = Format.kasprintf (fun s -> failwith s) fmt
 
 module Cli = struct
+  type action = Write | List_pipelines
+
   type config = {
     mutable verbose : bool;
     mutable inline_source_info : bool;
     mutable remove_extra_files : bool;
+    mutable action : action;
   }
 
   let config =
-    {verbose = false; inline_source_info = false; remove_extra_files = false}
+    {
+      verbose = false;
+      inline_source_info = false;
+      remove_extra_files = false;
+      action = Write;
+    }
 
   let verbose fmt =
     Format.kasprintf (if config.verbose then print_endline else fun _ -> ()) fmt
@@ -32,7 +40,7 @@ module Cli = struct
         [
           ( "--verbose",
             Arg.Unit (fun () -> config.verbose <- true),
-            " Show debug output, including the location of each generated job.."
+            " Show debug output, including the location of each generated job."
           );
           ( "--inline-source-info",
             Arg.Unit (fun () -> config.inline_source_info <- true),
@@ -40,6 +48,9 @@ module Cli = struct
           ( "--remove-extra-files",
             Arg.Unit (fun () -> config.remove_extra_files <- true),
             " Remove files that are neither generated nor excluded." );
+          ( "--list-pipelines",
+            Arg.Unit (fun () -> config.action <- List_pipelines),
+            " List registered pipelines." );
         ]
     in
     Arg.parse
@@ -121,6 +132,7 @@ let () = Printexc.register_printer @@ function Failure s -> Some s | _ -> None
 module Pipeline = struct
   type pipeline = {
     name : string;
+    description : string;
     if_ : Gitlab_ci.If.t;
     variables : Gitlab_ci.Types.variables option;
     auto_cancel : Gitlab_ci.Types.auto_cancel option;
@@ -129,6 +141,7 @@ module Pipeline = struct
 
   type child_pipeline = {
     name : string;
+    description : string;
     auto_cancel : Gitlab_ci.Types.auto_cancel option;
     jobs : tezos_job list;
   }
@@ -139,6 +152,9 @@ module Pipeline = struct
 
   let name = function Pipeline {name; _} | Child_pipeline {name; _} -> name
 
+  let description = function
+    | Pipeline {description; _} | Child_pipeline {description; _} -> description
+
   let jobs = function
     | Child_pipeline {jobs; _} -> jobs
     | Pipeline {jobs; _} -> jobs
@@ -158,11 +174,12 @@ module Pipeline = struct
         (name pipeline)
     else pipelines := pipeline :: !pipelines
 
-  let register ?variables ?auto_cancel ~jobs name if_ =
-    register_raw (Pipeline {variables; if_; name; jobs; auto_cancel})
+  let register ?variables ?auto_cancel ~description ~jobs name if_ =
+    register_raw
+      (Pipeline {variables; if_; name; jobs; auto_cancel; description})
 
-  let register_child ?auto_cancel ~jobs name =
-    let child_pipeline = {name; jobs; auto_cancel} in
+  let register_child ?auto_cancel ~description ~jobs name =
+    let child_pipeline = {name; jobs; auto_cancel; description} in
     register_raw (Child_pipeline child_pipeline) ;
     child_pipeline
 
@@ -447,6 +464,21 @@ module Pipeline = struct
     in
     to_file ~filename config ;
     ()
+
+  let list_pipelines () =
+    let pipelines = all () in
+    let open Format in
+    open_vbox 0 ;
+    Printf.printf "Registered pipeline types:\n\n" ;
+    ( Fun.flip List.iter pipelines @@ fun pipeline ->
+      let pipeline = add_image_builders pipeline in
+      let jobs = jobs pipeline in
+      let name = name pipeline in
+      let filename = path ~name in
+      printf " - '%s' (%s): %d job(s)@," name filename (List.length jobs) ;
+      printf "@,@[<2>  %a@]@,@," pp_print_text (description pipeline) ) ;
+    close_box () ;
+    print_flush ()
 end
 
 module Image = struct
@@ -738,8 +770,9 @@ let job ?arch ?after_script ?allow_failure ?artifacts ?before_script ?cache
   {job = Job job; source_position = __POS__; stage; image_builders}
 
 let trigger_job ?(dependencies = Staged []) ?rules ~__POS__ ~stage
-    Pipeline.{name = child_pipeline_name; jobs = _; auto_cancel = _} : tezos_job
-    =
+    Pipeline.
+      {name = child_pipeline_name; jobs = _; auto_cancel = _; description = _} :
+    tezos_job =
   let job_name = "trigger:" ^ child_pipeline_name in
   let needs, dependencies = resolve_dependencies job_name dependencies in
   if dependencies != [] then
diff --git a/ci/bin/tezos_ci.mli b/ci/bin/tezos_ci.mli
index 6833054bff5b..8fb691181643 100644
--- a/ci/bin/tezos_ci.mli
+++ b/ci/bin/tezos_ci.mli
@@ -39,6 +39,11 @@ val check_files :
 
 (** Run-time configuration and command-line processing. *)
 module Cli : sig
+  (** Action the binary should perform. *)
+  type action =
+    | Write  (** Write the CI configuration *)
+    | List_pipelines  (** List registered pipelines. *)
+
   (** Type of the command-line configuration for the generator binary. *)
   type config = {
     mutable verbose : bool;
@@ -47,6 +52,7 @@ module Cli : sig
         (** Enable the emission of source information in generated configuration. *)
     mutable remove_extra_files : bool;
         (** Remove files that are neither generated nor excluded. *)
+    mutable action : action;  (** Action to perform *)
   }
 
   (** Populate  {!config} from command-line arguments.
@@ -74,18 +80,21 @@ end
 module Pipeline : sig
   (** Register a pipeline.
 
-      [register ?variables name rule] will register a pipeline [name]
+      [register ~description ?variables name rule] will register a pipeline [name]
       that runs when [rule] is true.
 
       If [variables] is set, then these variables will be added to the
       [workflow:] clause for this pipeline in the top-level [.gitlab-ci.yml].
       Similarly, an [auto_cancel] clause can be specified.
 
+      The [description] is printed in {!list_pipelines}.
+
       The [jobs] of the pipeline are generated to the file
       [.gitlab/ci/pipelines/NAME.yml] when {!write} is called. *)
   val register :
     ?variables:Gitlab_ci.Types.variables ->
     ?auto_cancel:Gitlab_ci.Types.auto_cancel ->
+    description:string ->
     jobs:tezos_job list ->
     string ->
     Gitlab_ci.If.t ->
@@ -104,10 +113,13 @@ module Pipeline : sig
       [.gitlab/ci/pipelines/NAME.yml] when {!write} is called. See
       {!register} for info on [auto_cancel].
 
+      The [description] is printed in {!list_pipelines}.
+
       Child pipelines cannot be launched without a trigger job that is
       included in a regular pipeline (see {!trigger_job}). *)
   val register_child :
     ?auto_cancel:Gitlab_ci.Types.auto_cancel ->
+    description:string ->
     jobs:tezos_job list ->
     string ->
     child_pipeline
@@ -138,6 +150,9 @@ module Pipeline : sig
     filename:string ->
     unit ->
     unit
+
+  (** Pretty prints the set of registered pipelines. *)
+  val list_pipelines : unit -> unit
 end
 
 (** A facility for registering images for [image:] keywords.
diff --git a/ci/makefile b/ci/makefile
index 3b5e26c6f0b2..5aa696fcee80 100644
--- a/ci/makefile
+++ b/ci/makefile
@@ -14,6 +14,10 @@ inline-source-info: manifest # Regenerate GitLab CI configuration, inlining sour
 remove-extra-files: manifest # Regenerate GitLab CI configuration and remove any non-generated files.
 	cd .. && . ./scripts/version.sh && dune exec ci/bin/main.exe -- --remove-extra-files
 
+.PHONY: list-pipelines
+list-pipelines: manifest # List and describe the set of registered pipelines
+	cd .. && . ./scripts/version.sh && dune exec ci/bin/main.exe -- --list-pipelines
+
 .PHONY: manifest
 manifest:
 	cd .. && ${MAKE} --silent -C manifest
@@ -38,6 +42,9 @@ docker-inline-source-info: docker-do-inline-source-info # Build the target 'inli
 .PHONY: docker-remove-extra-files
 docker-remove-extra-files: docker-do-remove-extra-files # Build the target 'remove-extra-files' using the 'build' Docker image.
 
+.PHONY: docker-list-pipelines
+docker-list-pipelines: docker-do-list-pipelines # Build the target 'list-pipelines' using the 'build' Docker image.
+
 .PHONY: check
 check: all # Used in the CI to verify that [.gitlab-ci.yml] is up to date.
 	@if [ $$(git status --porcelain | wc -l) -ne 0 ]; then ( \
-- 
GitLab


From 9ee464e51fdd33b2254f79bd09e948efffe0f2a5 Mon Sep 17 00:00:00 2001
From: Arvid Jakobsson <arvid.jakobsson@nomadic-labs.com>
Date: Tue, 10 Sep 2024 16:23:42 +0200
Subject: [PATCH 5/9] CI: add [description] to all registered pipelines

---
 .gitlab-ci.yml              |  20 ++---
 ci/bin/code_verification.ml |   2 +-
 ci/bin/debian_repository.ml |  19 +++--
 ci/bin/main.ml              | 152 +++++++++++++++++++++++++++++++-----
 ci/bin/master_branch.ml     |   4 +-
 ci/bin/release_tag.ml       |   2 +-
 6 files changed, 159 insertions(+), 40 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0d99c30ad3b9..e8cf52798d61 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -16,6 +16,11 @@ workflow:
     when: always
     auto_cancel:
       on_job_failure: all
+  - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH
+      == "master"
+    variables:
+      PIPELINE_TYPE: master_branch
+    when: always
   - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH
       == "latest-release"
     variables:
@@ -26,11 +31,6 @@ workflow:
     variables:
       PIPELINE_TYPE: octez_latest_release_test
     when: always
-  - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH
-      == "master"
-    variables:
-      PIPELINE_TYPE: master_branch
-    when: always
   - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG
       =~ /^octez-v\d+\.\d+(?:\-rc\d+)?$/
     variables:
@@ -112,6 +112,11 @@ include:
   - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "merge_request_event"
       && $CI_MERGE_REQUEST_EVENT_TYPE == "merge_train"
     when: always
+- local: .gitlab/ci/pipelines/master_branch.yml
+  rules:
+  - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH
+      == "master"
+    when: always
 - local: .gitlab/ci/pipelines/octez_latest_release.yml
   rules:
   - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH
@@ -122,11 +127,6 @@ include:
   - if: $CI_PROJECT_NAMESPACE != "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH
       == "latest-release-test"
     when: always
-- local: .gitlab/ci/pipelines/master_branch.yml
-  rules:
-  - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH
-      == "master"
-    when: always
 - local: .gitlab/ci/pipelines/octez_release_tag.yml
   rules:
   - if: $CI_PROJECT_NAMESPACE == "tezos" && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_TAG
diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml
index ae018a330407..3748e656f6d3 100644
--- a/ci/bin/code_verification.ml
+++ b/ci/bin/code_verification.ml
@@ -1693,7 +1693,7 @@ let jobs pipeline_type =
     jobs_install_python @ jobs_documentation
   in
 
-  (*Manual jobs *)
+  (* Manual jobs *)
   let manual =
     (* On scheduled pipelines we build and test the full test matrix.
        On [Before_merging] pipelines only a subset of the packages are built
diff --git a/ci/bin/debian_repository.ml b/ci/bin/debian_repository.ml
index 52bcd7497066..b66edc917cc6 100644
--- a/ci/bin/debian_repository.ml
+++ b/ci/bin/debian_repository.ml
@@ -426,14 +426,23 @@ let jobs pipeline_type =
         job_build_debian_package_current_b )
 
 let register pipeline_type =
-  let pipeline_name =
+  let pipeline_name, description =
     match pipeline_type with
-    | Partial -> "debian_repository_partial"
-    | Full -> "debian_repository_full"
-    | Release -> "debian_repository_release"
+    | Partial ->
+        ( "debian_repository_partial",
+          "A child pipeline of 'before_merging' testing the build of .deb \
+           packages for Debian stable." )
+    | Full ->
+        ( "debian_repository_full",
+          "A child pipeline of 'schedule_extended_test' testing the build of \
+           all .deb packages." )
+    | Release ->
+        ( "debian_repository_release",
+          "A child pipeline testing the builds of and publishes all the .deb \
+           packages." )
   in
   let jobs, _, _, _, _ = jobs pipeline_type in
-  Pipeline.register_child pipeline_name ~jobs
+  Pipeline.register_child pipeline_name ~description ~jobs
 
 let child_pipeline_partial = register Partial
 
diff --git a/ci/bin/main.ml b/ci/bin/main.ml
index d19d2bed38ce..b2581d0678e9 100644
--- a/ci/bin/main.ml
+++ b/ci/bin/main.ml
@@ -80,7 +80,7 @@ let () =
   let open Rules in
   let open Pipeline in
   (* Matches either Octez release tags or Octez beta release tags,
-     e.g. [octez-v1.2.3], [octez-v1.2.3-rc4] or [octez-v1.2.3-beta5]. *)
+     e.g. [octez-v1.2], [octez-v1.2-rc4] or [octez-v1.2-beta5]. *)
   let has_any_octez_release_tag =
     If.(
       has_tag_match octez_release_tag_re
@@ -92,67 +92,177 @@ let () =
       && (not has_any_octez_release_tag)
       && not (has_tag_match octez_evm_node_release_tag_re))
   in
+  let release_description =
+    "\n\n\
+     For more information on Octez' release system, see: \
+     https://tezos.gitlab.io/releases/releases.html"
+  in
+  (* Please add a [~description] to each pipeline.
+
+     The first sentence of the description should be short (<=80
+     characters), and be terminated by two new-lines. It should
+     describe _what_ the pipeline does.
+
+     The remainder of the description should detail:
+      - what the pipeline does;
+      - why we do it;
+      - when it happens;
+      - how;
+      - and by whom it is triggered (a developer? a release manager? some automated system?). *)
   register
     "before_merging"
     If.(on_tezos_namespace && merge_request && not merge_train)
-    ~jobs:(Code_verification.jobs Before_merging) ;
-  (* The [merge_train] pipeline has the exact same jobs as the
-     [Before_merging] pipeline, but it auto-cancels on job failure. *)
+    ~jobs:(Code_verification.jobs Before_merging)
+    ~description:
+      "Lints code in merge requests, checks that it compiles and runs tests.\n\n\
+       This pipeline is created on each push to a branch with an associated \
+       open merge request, typically by the developer. It runs sanity checks, \
+       linters and checks that code of the MR compiles and that the tests \
+       pass. Must be manually started through the job 'trigger'." ;
   register
     "merge_train"
     ~auto_cancel:{on_job_failure = true; on_new_commit = false}
     If.(on_tezos_namespace && merge_request && merge_train)
-    ~jobs:(Code_verification.jobs Before_merging) ;
+    ~jobs:(Code_verification.jobs Before_merging)
+    ~description:
+      "A merge-train-specific version of 'before_merging'.\n\n\
+       This pipeline contains the same set of jobs as 'before_merging' but \
+       with auto-cancelling enabled on job failures. That is, if one job in \
+       the pipeline fails, the full pipeline is cancelled. This ensures that \
+       pipelines running in a merge train, that are bound to fail (due to some \
+       failing job), does so as early as possible. This prevents unneccessary \
+       delays in merging MRs further down the train.\n\n\
+       The merge train pipeline is created by GitLab when a merge request is \
+       added to the merge train (typically by marge-bot)." ;
+  register
+    "master_branch"
+    If.(on_tezos_namespace && push && on_branch "master")
+    ~jobs:Master_branch.jobs
+    ~description:
+      "Publishes artifacts (docs, static binaries) from master on each merge.\n\n\
+       This pipeline publishes the documentation at tezos.gitlab.io, builds \
+       static binaries, and the 'master' tag of the Octez Docker distribution. \
+       This pipeline is created automatically by GitLab on each push, \
+       typically resulting from the merge of a merge request, to the 'master' \
+       branch on tezos/tezos." ;
+  (* TODO: rename 'octez_docker_latest_release' ?? *)
   register
     "octez_latest_release"
     ~jobs:(Octez_latest_release.jobs ())
-    If.(on_tezos_namespace && push && on_branch "latest-release") ;
+    If.(on_tezos_namespace && push && on_branch "latest-release")
+    ~description:
+      ("Updates 'latest' tag of the Octez Docker distribution on Docker Hub.\n\n\
+        This pipeline is created on each push to the 'latest-release' branch \
+        of 'tezos/tezos', typically performed by the release manager. On each \
+        release, the 'latest-release' branch is updated to point to the git \
+        tag of the release. This resulting pipeline then updates the Docker \
+        tag 'latest' of the Octez Docker distribution published to Docker hub \
+        (https://hub.docker.com/r/tezos/tezos) to point to the Docker release \
+        associated with the git tag pushed to the 'latest-release' branch."
+     ^ release_description) ;
   register
     "octez_latest_release_test"
     If.(not_on_tezos_namespace && push && on_branch "latest-release-test")
-    ~jobs:(Octez_latest_release.jobs ~test:true ()) ;
-  register
-    "master_branch"
-    If.(on_tezos_namespace && push && on_branch "master")
-    ~jobs:Master_branch.jobs ;
+    ~jobs:(Octez_latest_release.jobs ~test:true ())
+    ~description:
+      "Dry-run pipeline for 'octez_latest_release' pipelines.\n\n\
+       This pipeline is used to dry run the 'octez_latest_release' pipeline, \
+       checking that it works as intended, without updating any Docker tags. \
+       Developers or release managers trigger it manually by pushing to the \
+       branch 'latest-release-test' of a fork of 'tezos/tezos', e.g. to the \
+       'nomadic-labs/tezos' project." ;
+  (* TODO: simplify dry run pipelines by having them all be on tezos/tezos? *)
   register
     "octez_release_tag"
     If.(on_tezos_namespace && push && has_tag_match octez_release_tag_re)
-    ~jobs:(Release_tag.octez_jobs Release_tag) ;
+    ~jobs:(Release_tag.octez_jobs Release_tag)
+    ~description:
+      ("Release tag pipelines for Octez.\n\n\
+        This pipeline is created when the release manager pushes a tag in the \
+        format octez-vX.Y(-rcN). It creates and publishes release on GitLab \
+        with the associated artifacts (static binaries, Docker images, .deb \
+        packages, etc.). It also prepares for a new opam package release by \
+        updating https://github.com/tezos/opam-repository."
+     ^ release_description) ;
   register
     "octez_beta_release_tag"
     If.(on_tezos_namespace && push && has_tag_match octez_beta_release_tag_re)
-    ~jobs:(Release_tag.octez_jobs Beta_release_tag) ;
+    ~jobs:(Release_tag.octez_jobs Beta_release_tag)
+    ~description:
+      ("Beta release tag pipelines for Octez.\n\n\
+        This pipeline is created when the release manager pushes a tag in the \
+        format octez-vX.Y(-betaN). It is as Octez release tag pipelines, but \
+        does not publish to opam." ^ release_description) ;
   register
     "octez_release_tag_test"
     If.(not_on_tezos_namespace && push && has_any_octez_release_tag)
-    ~jobs:(Release_tag.octez_jobs ~test:true Release_tag) ;
-  (* To test this type of release, push a tag to a fork of [tezos/tezos]
-     e.g. [nomadic-labs/tezos] project. *)
+    ~jobs:(Release_tag.octez_jobs ~test:true Release_tag)
+    ~description:
+      "Dry-run pipeline for 'octez_release_tag'.\n\n\
+       This pipeline checks that 'octez_release_tag' pipelines work as \
+       intended, without publishing any release. Developers or release \
+       managers can create this pipeline by pushing a tag to a fork of \
+       'tezos/tezos', e.g. to the 'nomadic-labs/tezos' project." ;
   register
     "octez_evm_node_release_tag"
     If.(push && has_tag_match octez_evm_node_release_tag_re)
-    ~jobs:(Release_tag.octez_evm_node_jobs ()) ;
+    ~jobs:(Release_tag.octez_evm_node_jobs ())
+    ~description:
+      ("Release tag pipelines for Etherlink.\n\n\
+        Created when the release manager pushes a tag in the format \
+        octez-evm-node-vX.Y(-rcN). Creates and publishes a release on GitLab \
+        with associated etherlink artifacts (static binaries and Docker \
+        image)." ^ release_description) ;
   register
     "non_release_tag"
     If.(on_tezos_namespace && push && has_non_release_tag)
-    ~jobs:(Release_tag.octez_jobs Non_release_tag) ;
+    ~jobs:(Release_tag.octez_jobs Non_release_tag)
+    ~description:
+      ("Tag pipeline for non-release tags.\n\n\
+        Created on each push of a tag that does not match e.g. \
+        octez(-evm-node)-vX.Y(-rcN). This pipeline creates a release on GitLab \
+        and associated artifacts, like 'octez_release_tag' pipelines, but does \
+        not publish it." ^ release_description) ;
   register
     "non_release_tag_test"
     If.(not_on_tezos_namespace && push && has_non_release_tag)
-    ~jobs:(Release_tag.octez_jobs ~test:true Non_release_tag) ;
+    ~jobs:(Release_tag.octez_jobs ~test:true Non_release_tag)
+    ~description:
+      "Dry-run pipeline for 'non_release_tag'.\n\n\
+       This pipeline checks that 'non_release_tag' pipelines work as intended, \
+       without publishing any release. Developers, or release managers, can \
+       create this pipeline by pushing a tag to a fork of 'tezos/tezos', e.g. \
+       to the 'nomadic-labs/tezos' project." ;
+  (* Scheduled pipelines *)
   register
     "schedule_extended_test"
     schedule_extended_tests
-    ~jobs:(Code_verification.jobs Schedule_extended_test) ;
+    ~jobs:(Code_verification.jobs Schedule_extended_test)
+    ~description:
+      "Scheduled, full version of 'before_merging', daily on 'master'.\n\n\
+       This pipeline unconditionally executes all jobs in 'before_merging' \
+       pipelines, daily on the 'master_branch'. Regular 'before_merging' \
+       pipelines run only subset of all jobs depending on files modified by \
+       the MR. This \"safety net\"-pipeline ensures that all jobs run at least \
+       daily." ;
   register
     "schedule_extended_rpc_test"
     schedule_extended_rpc_tests
-    ~jobs:Custom_extended_test_pipeline.jobs ;
+    ~jobs:Custom_extended_test_pipeline.jobs
+    ~description:
+      "Scheduled run of all tezt tests with external RPC servers, weekly on \
+       'master'.\n\n\
+       This scheduled pipeline exercices the full tezt tests suites, but with \
+       Octez nodes configured to use external RPC servers." ;
   register
     "schedule_extended_validation_test"
     schedule_extended_validation_tests
     ~jobs:Custom_extended_test_pipeline.jobs
+    ~description:
+      "Scheduled run of all tezt tests with single-process validation, weekly \
+       on 'master'.\n\n\
+       This scheduled pipeline exercices the full tezt tests suites, but with \
+       Octez nodes configured to use single-process validation."
 
 let () =
   (* If argument --verbose is set, then log generation info.
diff --git a/ci/bin/master_branch.ml b/ci/bin/master_branch.ml
index e6954b330a2b..4553f498269a 100644
--- a/ci/bin/master_branch.ml
+++ b/ci/bin/master_branch.ml
@@ -8,8 +8,8 @@
 (* This module defines the jobs of the [master_branch] pipeline.
 
    This pipeline runs for each merge on the [master] branch. To goal
-   of this pipeline is to publish artifacts for the development
-   version of Octez, including:
+   of this pipeline is to publish artifacts for the latest development
+   version of Octez (e.g. the latest version on 'master'), including:
 
    - docker images,
    - static binaries, and
diff --git a/ci/bin/release_tag.ml b/ci/bin/release_tag.ml
index fa335b44ec9d..e92338680b73 100644
--- a/ci/bin/release_tag.ml
+++ b/ci/bin/release_tag.ml
@@ -32,7 +32,7 @@ open Common
     - All release tag pipelines types publish [Release] type Docker builds.
     - No release tag pipelines include experimental binaries.
     - [Release_tag] and [Beta_release_tag] pipelines creates GitLab
-    and publishes releases. [Non_release_tag] pipelines creates the
+    and publishes releases. [Non_release_tag] pipelines create the
     GitLab release but do not publish them.
     - Only [Release_tag] pipelines publish to opam. *)
 type release_tag_pipeline_type =
-- 
GitLab


From f58e1286143c74d7f7bb2ab2c053d31066ec921c Mon Sep 17 00:00:00 2001
From: Arvid Jakobsson <arvid.jakobsson@nomadic-labs.com>
Date: Tue, 10 Sep 2024 16:05:07 +0200
Subject: [PATCH 6/9] CI: Split pipeline registration

---
 ci/bin/main.ml | 103 +++++++++++++++++++++++++++++--------------------
 1 file changed, 61 insertions(+), 42 deletions(-)

diff --git a/ci/bin/main.ml b/ci/bin/main.ml
index b2581d0678e9..4af5e2286e92 100644
--- a/ci/bin/main.ml
+++ b/ci/bin/main.ml
@@ -65,50 +65,30 @@ let variables : variables =
     ("CARGO_TERM_QUIET", "true");
   ]
 
+(** {2 Pipeline types} *)
+
 (* Register pipelines types. Pipelines types are used to generate
    workflow rules and includes of the files where the jobs of the
-   pipeline is defined. *)
+   pipeline is defined.
+
+   Please add a [~description] to each pipeline.
+
+   The first sentence of the description should be short (<=80
+   characters), and be terminated by two new-lines. It should
+   describe _what_ the pipeline does.
+
+   The remainder of the description should detail:
+   - what the pipeline does;
+   - why we do it;
+   - when it happens;
+   - how;
+   - and by whom it is triggered (a developer? a release manager? some automated system?). *)
+
+(** {3 General pipelines} *)
+
 let () =
-  (* Matches Octez release tags, e.g. [octez-v1.2.3] or [octez-v1.2.3-rc4]. *)
-  let octez_release_tag_re = "/^octez-v\\d+\\.\\d+(?:\\-rc\\d+)?$/" in
-  (* Matches Octez beta release tags, e.g. [octez-v1.2.3-beta5]. *)
-  let octez_beta_release_tag_re = "/^octez-v\\d+\\.\\d+\\-beta\\d*$/" in
-  (* Matches Etherlink release tags, e.g. [etherlink-v1.2.3] or [etherlink-v1.2.3-rc4]. *)
-  let octez_evm_node_release_tag_re =
-    "/^octez-evm-node-v\\d+\\.\\d+(?:\\-rc\\d+)?$/"
-  in
   let open Rules in
   let open Pipeline in
-  (* Matches either Octez release tags or Octez beta release tags,
-     e.g. [octez-v1.2], [octez-v1.2-rc4] or [octez-v1.2-beta5]. *)
-  let has_any_octez_release_tag =
-    If.(
-      has_tag_match octez_release_tag_re
-      || has_tag_match octez_beta_release_tag_re)
-  in
-  let has_non_release_tag =
-    If.(
-      Predefined_vars.ci_commit_tag != null
-      && (not has_any_octez_release_tag)
-      && not (has_tag_match octez_evm_node_release_tag_re))
-  in
-  let release_description =
-    "\n\n\
-     For more information on Octez' release system, see: \
-     https://tezos.gitlab.io/releases/releases.html"
-  in
-  (* Please add a [~description] to each pipeline.
-
-     The first sentence of the description should be short (<=80
-     characters), and be terminated by two new-lines. It should
-     describe _what_ the pipeline does.
-
-     The remainder of the description should detail:
-      - what the pipeline does;
-      - why we do it;
-      - when it happens;
-      - how;
-      - and by whom it is triggered (a developer? a release manager? some automated system?). *)
   register
     "before_merging"
     If.(on_tezos_namespace && merge_request && not merge_train)
@@ -144,7 +124,39 @@ let () =
        static binaries, and the 'master' tag of the Octez Docker distribution. \
        This pipeline is created automatically by GitLab on each push, \
        typically resulting from the merge of a merge request, to the 'master' \
-       branch on tezos/tezos." ;
+       branch on tezos/tezos."
+
+(** {3 Release pipelines} *)
+
+let () =
+  (* Matches Octez release tags, e.g. [octez-v1.2.3] or [octez-v1.2.3-rc4]. *)
+  let octez_release_tag_re = "/^octez-v\\d+\\.\\d+(?:\\-rc\\d+)?$/" in
+  (* Matches Octez beta release tags, e.g. [octez-v1.2.3-beta5]. *)
+  let octez_beta_release_tag_re = "/^octez-v\\d+\\.\\d+\\-beta\\d*$/" in
+  (* Matches Etherlink release tags, e.g. [etherlink-v1.2.3] or [etherlink-v1.2.3-rc4]. *)
+  let octez_evm_node_release_tag_re =
+    "/^octez-evm-node-v\\d+\\.\\d+(?:\\-rc\\d+)?$/"
+  in
+  let open Rules in
+  let open Pipeline in
+  (* Matches either Octez release tags or Octez beta release tags,
+     e.g. [octez-v1.2], [octez-v1.2-rc4] or [octez-v1.2-beta5]. *)
+  let has_any_octez_release_tag =
+    If.(
+      has_tag_match octez_release_tag_re
+      || has_tag_match octez_beta_release_tag_re)
+  in
+  let has_non_release_tag =
+    If.(
+      Predefined_vars.ci_commit_tag != null
+      && (not has_any_octez_release_tag)
+      && not (has_tag_match octez_evm_node_release_tag_re))
+  in
+  let release_description =
+    "\n\n\
+     For more information on Octez' release system, see: \
+     https://tezos.gitlab.io/releases/releases.html"
+  in
   (* TODO: rename 'octez_docker_latest_release' ?? *)
   register
     "octez_latest_release"
@@ -232,8 +244,13 @@ let () =
        This pipeline checks that 'non_release_tag' pipelines work as intended, \
        without publishing any release. Developers, or release managers, can \
        create this pipeline by pushing a tag to a fork of 'tezos/tezos', e.g. \
-       to the 'nomadic-labs/tezos' project." ;
-  (* Scheduled pipelines *)
+       to the 'nomadic-labs/tezos' project."
+
+(** {3 Scheduled pipelines} *)
+
+let () =
+  let open Pipeline in
+  let open Rules in
   register
     "schedule_extended_test"
     schedule_extended_tests
@@ -264,6 +281,8 @@ let () =
        This scheduled pipeline exercices the full tezt tests suites, but with \
        Octez nodes configured to use single-process validation."
 
+(** {2 Entry point of the generator binary} *)
+
 let () =
   (* If argument --verbose is set, then log generation info.
      If argument --inline-source, then print generation info in yml files. *)
-- 
GitLab


From efad23cbce9e776c2f680f7c097a16571b3305c6 Mon Sep 17 00:00:00 2001
From: Arvid Jakobsson <arvid.jakobsson@nomadic-labs.com>
Date: Thu, 12 Sep 2024 12:36:23 +0200
Subject: [PATCH 7/9] CIAO: refactor child pipeline registration in
 [debian_repository.ml]

---
 ci/bin/debian_repository.ml | 37 ++++++++++++++++---------------------
 1 file changed, 16 insertions(+), 21 deletions(-)

diff --git a/ci/bin/debian_repository.ml b/ci/bin/debian_repository.ml
index b66edc917cc6..a3a142bd392d 100644
--- a/ci/bin/debian_repository.ml
+++ b/ci/bin/debian_repository.ml
@@ -425,25 +425,20 @@ let jobs pipeline_type =
         job_build_ubuntu_package_current_b,
         job_build_debian_package_current_b )
 
-let register pipeline_type =
-  let pipeline_name, description =
-    match pipeline_type with
-    | Partial ->
-        ( "debian_repository_partial",
-          "A child pipeline of 'before_merging' testing the build of .deb \
-           packages for Debian stable." )
-    | Full ->
-        ( "debian_repository_full",
-          "A child pipeline of 'schedule_extended_test' testing the build of \
-           all .deb packages." )
-    | Release ->
-        ( "debian_repository_release",
-          "A child pipeline testing the builds of and publishes all the .deb \
-           packages." )
-  in
-  let jobs, _, _, _, _ = jobs pipeline_type in
-  Pipeline.register_child pipeline_name ~description ~jobs
-
-let child_pipeline_partial = register Partial
+let child_pipeline_partial =
+  let jobs, _, _, _, _ = jobs Partial in
+  Pipeline.register_child
+    "debian_repository_partial"
+    ~description:
+      "A child pipeline of 'before_merging' (and thus 'merge_train') building \
+       Debian stable .deb packages."
+    ~jobs
 
-let child_pipeline_full = register Full
+let child_pipeline_full =
+  let jobs, _, _, _, _ = jobs Full in
+  Pipeline.register_child
+    "debian_repository_full"
+    ~description:
+      "A child pipeline of 'schedule_extended_test' testing the build of all \
+       .deb packages."
+    ~jobs
-- 
GitLab


From 5468beaddda88ca2fe2be38644e2753f58fd56b9 Mon Sep 17 00:00:00 2001
From: Arvid Jakobsson <arvid.jakobsson@nomadic-labs.com>
Date: Fri, 13 Sep 2024 10:52:08 +0200
Subject: [PATCH 8/9] [ci/makefile]: fix target [help] on non gnu-sed

---
 ci/makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/makefile b/ci/makefile
index 5aa696fcee80..27c0b2f75922 100644
--- a/ci/makefile
+++ b/ci/makefile
@@ -59,4 +59,4 @@ check: all # Used in the CI to verify that [.gitlab-ci.yml] is up to date.
 help: # Display this help.
 	@echo "GitLab CI configuration generator. Available targets:"
 	@echo
-	@grep '^[^[:space:]]\+:.*#' makefile | sed 's/^\([^[:space:]]\+\):.*# \(.*\)/ - \1: \2/'
+	@grep '^[^[:space:]]\+:.*#' makefile | sed 's/:.*# /: /'
-- 
GitLab


From a272e915bf5caea170e95f34410dca1c6039d937 Mon Sep 17 00:00:00 2001
From: Arvid Jakobsson <arvid.jakobsson@nomadic-labs.com>
Date: Fri, 13 Sep 2024 10:52:37 +0200
Subject: [PATCH 9/9] [ci/makefile]: Use [column] to order output of target
 [help]

---
 ci/makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/makefile b/ci/makefile
index 27c0b2f75922..13ae2d028f5e 100644
--- a/ci/makefile
+++ b/ci/makefile
@@ -59,4 +59,4 @@ check: all # Used in the CI to verify that [.gitlab-ci.yml] is up to date.
 help: # Display this help.
 	@echo "GitLab CI configuration generator. Available targets:"
 	@echo
-	@grep '^[^[:space:]]\+:.*#' makefile | sed 's/:.*# /: /'
+	@grep '^[^[:space:]]\+:.*#' makefile | sed 's/:.*# /:/' | column -t -s':'
-- 
GitLab