From b4a2354fb65149e32676b6e7c92c05782711aa6b Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 8 Mar 2024 15:00:43 +0100 Subject: [PATCH 1/3] [scripts/version.sh]: fix instructions post CI-in-OCaml --- scripts/version.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/version.sh b/scripts/version.sh index 6a0fa933db65..e3ef7ee0cb91 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -29,7 +29,8 @@ export alpine_version='3.18' export full_opam_repository_tag=2314da5646931ec7f643bdc9aaa39177971ac857 ## opam_repository is an additional, tezos-specific opam repository. -## This value MUST be the same as `build_deps_image_version` in `.gitlab-ci.yml` +## This value MUST be reflected in the `build_deps_image_version` variable +## of `.gitlab-ci.yml`, which is ensured by running `make -C ci` from the root. export opam_repository_url=https://gitlab.com/tezos/opam-repository export opam_repository_tag="${OPAM_REPOSITORY_TAG:-d8bd1a0e555b2f9a32bce18a4022f6851c700bca}" export opam_repository_git="$opam_repository_url.git" -- GitLab From 87be406ee2613eacd341f7aaa80456e4e31499a5 Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 1 Mar 2024 15:26:48 +0100 Subject: [PATCH 2/3] CI-in-OCaml: explain return value of [Tezos_ci.job_external] --- ci/bin/tezos_ci.mli | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/bin/tezos_ci.mli b/ci/bin/tezos_ci.mli index 3c47c00cbc85..b7c8ce8f6846 100644 --- a/ci/bin/tezos_ci.mli +++ b/ci/bin/tezos_ci.mli @@ -222,7 +222,9 @@ val job : Raises [Failure] if [.gitlab/ci/jobs/DIRECTORY] is not an existing directory. Also [Failure] if destination path has already been - used to write another job. *) + used to write another job. + + The returned job is the same as the input, for ease of chaining. *) val job_external : ?directory:string -> ?filename_suffix:string -> tezos_job -> tezos_job -- GitLab From 535542ba0d7fee6d8255ac0fb87b740010ddcb8d Mon Sep 17 00:00:00 2001 From: Arvid Jakobsson Date: Fri, 8 Mar 2024 12:52:59 +0100 Subject: [PATCH 3/3] CI-in-OCaml: add generation check modelled on manifest --- ci/bin/main.ml | 69 ++++++++++++++++++++++++- ci/bin/tezos_ci.ml | 123 +++++++++++++++++++++++++++++++++++--------- ci/bin/tezos_ci.mli | 17 ++++++ ci/makefile | 7 +++ 4 files changed, 190 insertions(+), 26 deletions(-) diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 9275b6c50ee3..aaab8b1e85c7 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -1197,5 +1197,70 @@ let config () = let () = (* If argument --verbose is set, then log generation info. If argument --inline-source, then print generation info in yml files. *) - let filename = Base.(project_root // ".gitlab-ci.yml") in - To_yaml.to_file ~header:Tezos_ci.header ~filename (config ()) + let filename = ".gitlab-ci.yml" in + Tezos_ci.to_file ~filename (config ()) ; + (* Paths to exclude from generation check. As files are translated + to CI-in-OCaml, they should be removed from this function *) + let exclude = function + | ".gitlab/ci/jobs/build/bin_packages_common.yml" + | ".gitlab/ci/jobs/build/common.yml" + | ".gitlab/ci/jobs/build/oc.build_kernels.yml" + | ".gitlab/ci/jobs/build/oc.build_x86_64-exp-dev-extra.yml" + | ".gitlab/ci/jobs/build/oc.build_x86_64-released.yml" + | ".gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-before_merging.yml" + | ".gitlab/ci/jobs/build/oc.docker:client-libs-dependencies-other.yml" + | ".gitlab/ci/jobs/build/oc.tezt:fetch-records.yml" + | ".gitlab/ci/jobs/build/ocaml-check.yml" + | ".gitlab/ci/jobs/build/select_tezts.yml" + | ".gitlab/ci/jobs/coverage/common.yml" + | ".gitlab/ci/jobs/coverage/oc.unified_coverage-before_merging.yml" + | ".gitlab/ci/jobs/doc/documentation.yml" + | ".gitlab/ci/jobs/doc/documentation:linkcheck.yml" + | ".gitlab/ci/jobs/doc/oc.install_python.yml" + | ".gitlab/ci/jobs/packaging/debian_repository.yml" + | ".gitlab/ci/jobs/packaging/opam:prepare.yml" + | ".gitlab/ci/jobs/packaging/opam_package.yml" + | ".gitlab/ci/jobs/sanity/docker:hadolint-before_merging.yml" + | ".gitlab/ci/jobs/sanity/docker:hadolint-schedule_extended_test.yml" + | ".gitlab/ci/jobs/sanity/docker:hadolint.yml" + | ".gitlab/ci/jobs/sanity/sanity_ci.yml" + | ".gitlab/ci/jobs/shared/images.yml" + | ".gitlab/ci/jobs/shared/templates.yml" + | ".gitlab/ci/jobs/test/commit_titles.yml" + | ".gitlab/ci/jobs/test/common.yml" + | ".gitlab/ci/jobs/test/install_octez.yml" + | ".gitlab/ci/jobs/test/kaitai_checks.yml" + | ".gitlab/ci/jobs/test/kaitai_e2e_checks.yml" + | ".gitlab/ci/jobs/test/misc_opam_checks.yml" + | ".gitlab/ci/jobs/test/oc.check_lift_limits_patch.yml" + | ".gitlab/ci/jobs/test/oc.integration:compiler-rejections.yml" + | ".gitlab/ci/jobs/test/oc.misc_checks-before_merging.yml" + | ".gitlab/ci/jobs/test/oc.misc_checks-schedule_extended_test.yml" + | ".gitlab/ci/jobs/test/oc.script:b58_prefix.yml" + | ".gitlab/ci/jobs/test/oc.script:snapshot_alpha_and_link.yml" + | ".gitlab/ci/jobs/test/oc.script:test-gen-genesis.yml" + | ".gitlab/ci/jobs/test/oc.script:test_octez_release_versions.yml" + | ".gitlab/ci/jobs/test/oc.semgrep.yml" + | ".gitlab/ci/jobs/test/oc.test-liquidity-baking-scripts.yml" + | ".gitlab/ci/jobs/test/oc.unit.yml" + | ".gitlab/ci/jobs/test/test_etherlink_kernel-before_merging.yml" + | ".gitlab/ci/jobs/test/test_etherlink_kernel-schedule_extended_test.yml" + | ".gitlab/ci/jobs/test/test_evm_compatibility.yml" + | ".gitlab/ci/jobs/test/test_kernels.yml" + | ".gitlab/ci/jobs/test/test_risc_v_kernels-before_merging.yml" + | ".gitlab/ci/jobs/test/test_risc_v_kernels-schedule_extended_test.yml" + | ".gitlab/ci/jobs/test/tezt-flaky-before_merging.yml" + | ".gitlab/ci/jobs/test/tezt-flaky-schedule_extended_test.yml" + | ".gitlab/ci/jobs/test/tezt-flaky.yml" + | ".gitlab/ci/jobs/test/tezt-slow-before_merging.yml" + | ".gitlab/ci/jobs/test/tezt-slow-schedule_extended_test.yml" + | ".gitlab/ci/jobs/test/tezt-slow.yml" | ".gitlab/ci/jobs/test/tezt.yml" + | ".gitlab/ci/pipelines/before_merging.yml" + | ".gitlab/ci/pipelines/schedule_extended_test.yml" -> + true + | _ -> false + in + Tezos_ci.check_files + ~remove_extra_files:Cli.config.remove_extra_files + ~exclude + () diff --git a/ci/bin/tezos_ci.ml b/ci/bin/tezos_ci.ml index 7b11ee7ffb82..044cf3761419 100644 --- a/ci/bin/tezos_ci.ml +++ b/ci/bin/tezos_ci.ml @@ -1,13 +1,29 @@ open Gitlab_ci.Util module Cli = struct - type config = {mutable verbose : bool; mutable inline_source_info : bool} + type config = { + mutable verbose : bool; + mutable inline_source_info : bool; + mutable remove_extra_files : bool; + } - let config = {verbose = false; inline_source_info = false} + let config = + {verbose = false; inline_source_info = false; remove_extra_files = false} let verbose fmt = Format.kasprintf (if config.verbose then print_endline else fun _ -> ()) fmt + let has_error = ref false + + let error fmt = + Format.kasprintf + (fun s -> + has_error := true ; + prerr_endline s) + fmt + + let info fmt = Format.kasprintf print_endline fmt + let init () = let speclist = Arg.align @@ -19,6 +35,9 @@ module Cli = struct ( "--inline-source-info", Arg.Unit (fun () -> config.inline_source_info <- true), " Comment each generated job with source information." ); + ( "--remove-extra-files", + Arg.Unit (fun () -> config.remove_extra_files <- true), + " Remove files that are neither generated nor excluded." ); ] in Arg.parse @@ -48,15 +67,27 @@ let tezos_job_to_config_elements (j : tezos_job) = in source_comment @ [Gitlab_ci.Types.Job j.job] +let failwith fmt = Format.kasprintf (fun s -> failwith s) fmt + let header = {|# This file was automatically generated, do not edit. # Edit file ci/bin/main.ml instead. |} -let () = Printexc.register_printer @@ function Failure s -> Some s | _ -> None +let external_files = ref String_set.empty -let failwith fmt = Format.kasprintf (fun s -> failwith s) fmt +let to_file ~filename config = + if String_set.mem filename !external_files then + failwith + "Attempted to write external file %s twice -- perhaps you need to set \ + filename_suffix when using [job_external]?" + filename + else ( + external_files := String_set.add filename !external_files ; + Gitlab_ci.To_yaml.to_file ~header ~filename config) + +let () = Printexc.register_printer @@ function Failure s -> Some s | _ -> None module Stage = struct type t = Stage of string @@ -200,7 +231,7 @@ module Pipeline = struct filename) jobs ; let config = List.concat_map tezos_job_to_config_elements jobs in - Gitlab_ci.To_yaml.to_file ~header ~filename config + to_file ~filename config let workflow_includes () : Gitlab_ci.Types.workflow * Gitlab_ci.Types.include_ list = @@ -387,8 +418,6 @@ let job ?arch ?after_script ?allow_failure ?artifacts ?before_script ?cache in {job; source_position = __POS__} -let external_jobs = ref String_set.empty - let job_external ?directory ?filename_suffix (tezos_job : tezos_job) : tezos_job = let job = tezos_job.job in @@ -412,23 +441,16 @@ let job_external ?directory ?filename_suffix (tezos_job : tezos_job) : tezos_job job.name directory ; let filename = (directory // basename) ^ ".yml" in - if String_set.mem filename !external_jobs then - failwith - "Attempted to write external job %s twice -- perhaps you need to set \ - filename_suffix?" - filename - else ( - external_jobs := String_set.add filename !external_jobs ; - let config = tezos_job_to_config_elements tezos_job in - let source_file, source_line, _, _ = tezos_job.source_position in - Cli.verbose - "%s:%d: generates '%s' in %s" - source_file - source_line - job.name - filename ; - Gitlab_ci.To_yaml.to_file ~header ~filename config ; - tezos_job) + let config = tezos_job_to_config_elements tezos_job in + let source_file, source_line, _, _ = tezos_job.source_position in + Cli.verbose + "%s:%d: generates '%s' in %s" + source_file + source_line + job.name + filename ; + to_file ~filename config ; + tezos_job let add_artifacts ?name ?expose_as ?reports ?expire_in ?when_ paths (tezos_job : tezos_job) = @@ -529,3 +551,56 @@ let append_variables ?(allow_overwrite = false) new_variables old_variables @ List.rev new_variables in {job with variables = Some variables} + +let check_files ~remove_extra_files ?(exclude = fun _ -> false) () = + let all_files = + let root = "." in + let rec loop acc dir = + let dir_contents = Sys.readdir (root // dir) in + let add_item acc filename = + let full_filename = dir // filename in + if Filename.extension filename = ".yml" then + String_set.add full_filename acc + else if filename.[0] = '.' || filename.[0] = '_' then acc + else if + try Sys.is_directory (root // dir // filename) + with Sys_error _ -> false + then loop acc full_filename + else acc + in + Array.fold_left add_item acc dir_contents + in + loop (String_set.singleton ".gitlab-ci.yml") ".gitlab" + in + let all_non_excluded_files = + String_set.filter (fun x -> not (exclude x)) all_files + in + let error_generated_and_excluded = + String_set.filter exclude !external_files + in + String_set.iter + (Cli.error "%s: generated but is excluded") + error_generated_and_excluded ; + let error_not_generated = + String_set.diff all_non_excluded_files !external_files + in + String_set.iter + (fun file -> + if remove_extra_files then ( + Cli.info "%s: exists but was not generated, removing it." file ; + Sys.remove file) + else Cli.error "%s: exists but was not generated" file) + error_not_generated ; + if + not + ((remove_extra_files || String_set.is_empty error_not_generated) + && String_set.is_empty error_generated_and_excluded) + then + Cli.error + "Please modify ci/bin/main.ml to either generate the above file(s)\n\ + or declare them in the 'exclude' function (but not both).\n\ + If this file is a leftover from some previous work on the CI\n\ + system then simply remove with 'make -C ci remove-extra-files' or with:\n\n\ + \ rm %s" + (error_not_generated |> String_set.elements |> String.concat " ") ; + if !Cli.has_error then exit 1 diff --git a/ci/bin/tezos_ci.mli b/ci/bin/tezos_ci.mli index b7c8ce8f6846..e3927fa82fd1 100644 --- a/ci/bin/tezos_ci.mli +++ b/ci/bin/tezos_ci.mli @@ -13,6 +13,21 @@ type tezos_job Warns not to modify the generated files, and refers to the generator. *) val header : string +(** Write a CI configuration to a file *) +val to_file : filename:string -> Gitlab_ci.Types.config -> unit + +(** Checks that should be performed after {!Pipeline.write}. + + Checks that the file [.gitlab-ci.yml] and all the [.yml] files in [.gitlab] + are either generated or excluded. It is an error if a generated file is excluded. + You can use [exclude] to specify which files should be excluded. + [exclude] is given a path relative to the [root] directory + and shall return [true] for excluded paths. + + In case of errors, errors are printed and the process exits with exit code 1. *) +val check_files : + remove_extra_files:bool -> ?exclude:(string -> bool) -> unit -> unit + (** Run-time configuration and command-line processing. *) module Cli : sig (** Type of the command-line configuration for the generator binary. *) @@ -21,6 +36,8 @@ module Cli : sig (** Enable [verbose] output, including the source of generated jobs. *) mutable inline_source_info : bool; (** Enable the emission of source information in generated configuration. *) + mutable remove_extra_files : bool; + (** Remove files that are neither generated nor excluded. *) } (** Populate {!config} from command-line arguments. diff --git a/ci/makefile b/ci/makefile index c9decae1a514..f050b0b7dd52 100644 --- a/ci/makefile +++ b/ci/makefile @@ -10,6 +10,10 @@ verbose: # Regenerate GitLab CI configuration with verbose output. inline-source-info: # Regenerate GitLab CI configuration, inlining source information in generated files. (cd .. && . ./scripts/version.sh && dune exec ci/bin/main.exe -- --inline-source-info) +.PHONY: verbose +remove-extra-files: # Regenerate GitLab CI configuration and remove any non-generated files. + (cd .. && . ./scripts/version.sh && dune exec ci/bin/main.exe -- --remove-extra-files) + .PHONY: docker-do-% docker-do-%: @( cd .. \ @@ -27,6 +31,9 @@ docker-verbose: docker-do-verbose # Build the target 'verbose' using the 'runtim .PHONY: docker-inline-source-info docker-inline-source-info: docker-do-inline-source-info # Build the target 'inline-source-info' using the 'runtime-build-dependencies' Docker image. +.PHONY: docker-remove-extra-files +docker-remove-extra-files: docker-do-remove-extra-files # Build the target 'remove-extra-files' using the 'runtime-build-dependencies' Docker image. + .PHONY: check check: # Used in the CI to verify that [.gitlab-ci.yml] is up to date. @git diff --exit-code HEAD -- ../.gitlab-ci.yml || (echo "Cannot check generated [.gitlab-ci.yml] file, some changes are uncommitted"; exit 1) -- GitLab