diff --git a/.gitlab/ci/jobs/coverage/common.yml b/.gitlab/ci/jobs/coverage/common.yml deleted file mode 100644 index 1ef60e31f25d14baf7350e77607576f616b7b98d..0000000000000000000000000000000000000000 --- a/.gitlab/ci/jobs/coverage/common.yml +++ /dev/null @@ -1,17 +0,0 @@ -.oc.template__coverage_report: - extends: .oc.template__coverage_location - stage: test_coverage - coverage: '/Coverage: ([^%]+%)/' - variables: - SLACK_COVERAGE_CHANNEL: "C02PHBE7W73" - artifacts: - expose_as: 'Coverage report' - reports: - coverage_report: - coverage_format: cobertura - path: _coverage_report/cobertura.xml - paths: - - _coverage_report/ - - $BISECT_FILE - expire_in: 15 days - when: always diff --git a/.gitlab/ci/jobs/coverage/oc.unified_coverage-before_merging.yml b/.gitlab/ci/jobs/coverage/oc.unified_coverage-before_merging.yml index ed7104fe12ee5a0ff090b319ed9b558426fd6534..57652428fe51a38ba29533559f428b011e718b5f 100644 --- a/.gitlab/ci/jobs/coverage/oc.unified_coverage-before_merging.yml +++ b/.gitlab/ci/jobs/coverage/oc.unified_coverage-before_merging.yml @@ -1,101 +1,49 @@ ---- -# This job fetches coverage files by precedent test stage. It creates the html, -# summary and cobertura reports. It also provide a coverage % for the merge request. - -include: .gitlab/ci/jobs/coverage/common.yml +# This file was automatically generated, do not edit. +# Edit file ci/bin/main.ml instead. oc.unified_coverage: - extends: - - .default_settings_template - - .image_template__runtime_e2etest_dependencies - - .oc.template__coverage_report - # We do not run this job when margebot triggers the - # pipeline. Instead, the coverage traces are downloaded in the - # master pipeline and the report is computed there. - - .rules__octez_changes_and_not_margebot - variables: - # This inhibites the Makefile's opam version check, which this - # job's opam-less image cannot pass. - TEZOS_WITHOUT_OPAM: "true" - # This job requires all bisect_ppx artifacts from the stage test, so we override - # the `dependencies: []` in `.default_settings` with a list of jobs. - # Each new job in the stage test needs to be manually added to this list. - # /!\ Warning: this list is read by `scripts/ci/download_coverage/download.ml`. - # The 'dependencies' field must be followed directly by the 'script' field. + image: ${build_deps_image_name}:runtime-e2etest-dependencies--${build_deps_image_version} + stage: test_coverage + tags: + - gcp + rules: + - if: $GITLAB_USER_LOGIN == "nomadic-margebot" + when: never + - changes: + - .gitlab-ci.yml + - .gitlab/**/* + - etherlink/**/* + - michelson_test_scripts/**/* + - src/**/* + - tezt/**/* + - tzt_reference_test_suite/**/* + when: on_success dependencies: - - "tezt 1/60" - - "tezt 2/60" - - "tezt 3/60" - - "tezt 4/60" - - "tezt 5/60" - - "tezt 6/60" - - "tezt 7/60" - - "tezt 8/60" - - "tezt 9/60" - - "tezt 10/60" - - "tezt 11/60" - - "tezt 12/60" - - "tezt 13/60" - - "tezt 14/60" - - "tezt 15/60" - - "tezt 16/60" - - "tezt 17/60" - - "tezt 18/60" - - "tezt 19/60" - - "tezt 20/60" - - "tezt 21/60" - - "tezt 22/60" - - "tezt 23/60" - - "tezt 24/60" - - "tezt 25/60" - - "tezt 26/60" - - "tezt 27/60" - - "tezt 28/60" - - "tezt 29/60" - - "tezt 30/60" - - "tezt 31/60" - - "tezt 32/60" - - "tezt 33/60" - - "tezt 34/60" - - "tezt 35/60" - - "tezt 36/60" - - "tezt 37/60" - - "tezt 38/60" - - "tezt 39/60" - - "tezt 40/60" - - "tezt 41/60" - - "tezt 42/60" - - "tezt 43/60" - - "tezt 44/60" - - "tezt 45/60" - - "tezt 46/60" - - "tezt 47/60" - - "tezt 48/60" - - "tezt 49/60" - - "tezt 50/60" - - "tezt 51/60" - - "tezt 52/60" - - "tezt 53/60" - - "tezt 54/60" - - "tezt 55/60" - - "tezt 56/60" - - "tezt 57/60" - - "tezt 58/60" - - "tezt 59/60" - - "tezt 60/60" - - "tezt-memory-4k 1/4" - - "tezt-memory-4k 2/4" - - "tezt-memory-4k 3/4" - - "tezt-memory-4k 4/4" - - "tezt-memory-3k" - - "tezt-time-sensitive" - - "oc.unit:non-proto-x86_64" - - "oc.unit:proto-x86_64" - - "oc.unit:other-x86_64" - script: - # On the development branches, we compute coverage. - # TODO: https://gitlab.com/tezos/tezos/-/issues/6173 - # We propagate the exit code to temporarily allow corrupted coverage files. - - ./scripts/ci/report_coverage.sh || exit $? + - oc.unit:non-proto-x86_64 + - oc.unit:other-x86_64 + - oc.unit:proto-x86_64 + - tezt-flaky + - tezt + - tezt-memory-4k + - tezt-memory-3k + - tezt-time-sensitive allow_failure: exit_codes: 64 + script: + - ./scripts/ci/report_coverage.sh || exit $? + variables: + TEZOS_WITHOUT_OPAM: "true" + BISECT_FILE: $CI_PROJECT_DIR/_coverage_output/ + SLACK_COVERAGE_CHANNEL: C02PHBE7W73 + artifacts: + expire_in: 15 days + paths: + - _coverage_report/ + - $BISECT_FILE + reports: + coverage_report: + coverage_format: cobertura + path: _coverage_report/cobertura.xml + when: always + expose_as: Coverage report + coverage: '/Coverage: ([^%]+%)/' diff --git a/.gitlab/ci/pipelines/master_branch.yml b/.gitlab/ci/pipelines/master_branch.yml index 1a00e72b5e2a720ac73beadabe8b630cd6e1f6ee..754dbd2a24f41b7534f7f17b53c42acabdb954f6 100644 --- a/.gitlab/ci/pipelines/master_branch.yml +++ b/.gitlab/ci/pipelines/master_branch.yml @@ -215,7 +215,7 @@ oc.unified_coverage: - . ./scripts/version.sh script: - mkdir -p _coverage_report - - dune exec scripts/ci/download_coverage/download.exe -- -a from=last-merged-pipeline + - dune exec scripts/ci/download_coverage/download.exe -- --from last-merged-pipeline --info --log-file _coverage_report/download_coverage.log - ./scripts/ci/report_coverage.sh variables: diff --git a/ci/bin/code_verification.ml b/ci/bin/code_verification.ml index 2b54bed7991bb0329c2d5d55c7aa6c1119119755..fa0c6f2ffb6de246a927255d40b1ba91c99739f7 100644 --- a/ci/bin/code_verification.ml +++ b/ci/bin/code_verification.ml @@ -45,50 +45,6 @@ type manual = | Yes (** Add rule for manual trigger. *) | On_changes of Changeset.t (** Add manual trigger on certain [changes:] *) -(* [make_rules] makes rules for jobs that are: - - automatic in scheduled pipelines; - - conditional in [before_merging] pipelines. - - If a job has non-optional dependencies, then [dependent] must be - set to [true] to ensure that we only run the job in case previous - jobs succeeded (setting [when: on_success]). - - If [label] is set, add rule that selects the job in - [Before_merging] pipelines for merge requests with the given - label. Rules for manual triggers can be configured using - [manual]. - - If [label], [changes] and [manual] are omitted, then rules will - enable the job [On_success] in the [before_merging] - pipeline. This is safe, but prefer specifying a [changes] clause - if possible. *) -let make_rules ?label ?changes ?(manual = No) ?(dependent = false) pipeline_type - = - match pipeline_type with - | Schedule_extended_test -> - (* The scheduled pipeline always runs all jobs unconditionally - -- unless they are dependent on a previous, non-trigger job, in the - pipeline. *) - [job_rule ~when_:(if dependent then On_success else Always) ()] - | Before_merging -> ( - (* MR labels can be used to force tests to run. *) - (match label with - | Some label -> - [job_rule ~if_:Rules.(has_mr_label label) ~when_:On_success ()] - | None -> []) - (* Modifying some files can force tests to run. *) - @ (match changes with - | None -> [] - | Some changes -> - [job_rule ~changes:(Changeset.encode changes) ~when_:On_success ()]) - (* For some tests, it can be relevant to have a manual trigger. *) - @ - match manual with - | No -> [] - | Yes -> [job_rule ~when_:Manual ()] - | On_changes changes -> - [job_rule ~when_:Manual ~changes:(Changeset.encode changes) ()]) - type opam_package_group = Executable | All type opam_package = { @@ -362,8 +318,56 @@ let build_debian_packages_image = (* Encodes the conditional [before_merging] pipeline and its unconditional variant [schedule_extended_test]. *) let jobs pipeline_type = - let make_rules ?label ?changes ?manual ?dependent () = - make_rules ?label ?changes ?manual ?dependent pipeline_type + (* [make_rules] makes rules for jobs that are: + - automatic in scheduled pipelines; + - conditional in [before_merging] pipelines. + + If a job has non-optional dependencies, then [dependent] must be + set to [true] to ensure that we only run the job in case previous + jobs succeeded (setting [when: on_success]). + + If [label] is set, add rule that selects the job in + [Before_merging] pipelines for merge requests with the given + label. Rules for manual triggers can be configured using + [manual]. + + If [label], [changes] and [manual] are omitted, then rules will + enable the job [On_success] in the [before_merging] pipeline. This + is safe, but prefer specifying a [changes] clause if possible. + + If [margebot_disable] is set to true (default false), this job is + disabled when marge-bot triggers the [before_merging] pipeline. *) + let make_rules ?label ?changes ?(manual = No) ?(dependent = false) + ?(margebot_disable = false) () = + match pipeline_type with + | Schedule_extended_test -> + (* The scheduled pipeline always runs all jobs unconditionally + -- unless they are dependent on a previous, non-trigger job, in the + pipeline. *) + [job_rule ~when_:(if dependent then On_success else Always) ()] + | Before_merging -> ( + (* MR labels can be used to force tests to run. *) + (if margebot_disable then + [job_rule ~if_:Rules.triggered_by_marge_bot ~when_:Never ()] + else []) + @ (match label with + | Some label -> + [job_rule ~if_:Rules.(has_mr_label label) ~when_:On_success ()] + | None -> []) + (* Modifying some files can force tests to run. *) + @ (match changes with + | None -> [] + | Some changes -> + [ + job_rule ~changes:(Changeset.encode changes) ~when_:On_success (); + ]) + (* For some tests, it can be relevant to have a manual trigger. *) + @ + match manual with + | No -> [] + | Yes -> [job_rule ~when_:Manual ()] + | On_changes changes -> + [job_rule ~when_:Manual ~changes:(Changeset.encode changes) ()]) in (* Externalization *) let job_external_split ?(before_merging_suffix = "before_merging") @@ -402,6 +406,22 @@ let jobs pipeline_type = in (* Common GitLab CI caches *) let cache_kernels = {key = "kernels"; paths = ["cargo/"]} in + (* Collect coverage trace producing jobs *) + let jobs_with_coverage_output = ref [] in + (* Add variables for bisect_ppx output and store the traces as an + artifact. + + This function should be applied to test jobs that produce coverage. *) + let enable_coverage_output_artifact ?(expire_in = Duration (Days 1)) tezos_job + : tezos_job = + jobs_with_coverage_output := tezos_job :: !jobs_with_coverage_output ; + tezos_job |> enable_coverage_location + |> add_artifacts + ~expire_in + ~name:"coverage-files-$CI_JOB_ID" + ~when_:On_success + ["$BISECT_FILE"] + in (* Stages *) (* All stages should be empty, as explained below, until the full pipeline is generated. *) let trigger_stage, make_dependencies = @@ -1651,6 +1671,53 @@ let jobs pipeline_type = [job_commit_titles] | Schedule_extended_test -> [] in + let coverage = + match pipeline_type with + | Before_merging -> + (* Write the name of each job that produces coverage as input for other scripts. + Only includes the stem of the name: parallel jobs only appear once. + E.g. as [tezt], not [tezt 1/60], [tezt 2/60], etc. *) + Base.write_file + "script-inputs/ci-coverage-producing-jobs" + ~contents: + (String.concat + "\n" + (List.map Tezos_ci.name_of_tezos_job !jobs_with_coverage_output) + ^ "\n") ; + (* This job fetches coverage files by precedent test stage. It creates + the html, summary and cobertura reports. It also provide a coverage % + for the merge request. *) + let job_unified_coverage : tezos_job = + let dependencies = List.rev !jobs_with_coverage_output in + job + ~__POS__ + ~image:Images.runtime_e2etest_dependencies + ~name:"oc.unified_coverage" + ~stage:Stages.test_coverage + ~coverage:"/Coverage: ([^%]+%)/" + ~rules: + (make_rules ~margebot_disable:true ~changes:changeset_octez ()) + ~variables: + [ + (* This inhibits the Makefile's opam version check, which + this job's opam-less image + ([runtime_e2etest_dependencies]) cannot pass. *) + ("TEZOS_WITHOUT_OPAM", "true"); + ] + ~dependencies:(Staged dependencies) + (* On the development branches, we compute coverage. + TODO: https://gitlab.com/tezos/tezos/-/issues/6173 + We propagate the exit code to temporarily allow corrupted coverage files. *) + (script_propagate_exit_code "./scripts/ci/report_coverage.sh") + ~allow_failure:(With_exit_codes [64]) + |> enable_coverage_location |> enable_coverage_report + |> job_external + ~directory:"coverage" + ~filename_suffix:"before_merging" + in + [job_unified_coverage] + | Schedule_extended_test -> [] + in let doc = let jobs_install_python = (* Creates a job that tests installation of the python environment in [image] *) @@ -1891,5 +1958,6 @@ let jobs pipeline_type = (using {!job_external} or {!jobs_external}) and included by hand in the files [.gitlab/ci/pipelines/before_merging.yml] and [.gitlab/ci/pipelines/schedule_extended_test.yml]. *) - ignore (trigger_stage @ sanity @ build @ packaging @ test @ doc @ manual) ; + ignore + (trigger_stage @ sanity @ build @ packaging @ test @ coverage @ doc @ manual) ; [] diff --git a/ci/bin/common.ml b/ci/bin/common.ml index 09c7fd728472561181bfa9b652b4be2d309ba74b..06253e69df20648f7606ccf307fcf3696d0ad7ba 100644 --- a/ci/bin/common.ml +++ b/ci/bin/common.ml @@ -256,20 +256,6 @@ let enable_coverage_location : tezos_job -> tezos_job = Tezos_ci.append_variables [("BISECT_FILE", "$CI_PROJECT_DIR/_coverage_output/")] -(** Add variables for bisect_ppx output and store the traces as an - artifact. - - This function should be applied to test jobs that produce coverage. *) -let enable_coverage_output_artifact ?(expire_in = Duration (Days 1)) : - tezos_job -> tezos_job = - fun job -> - job |> enable_coverage_location - |> Tezos_ci.add_artifacts - ~expire_in - ~name:"coverage-files-$CI_JOB_ID" - ~when_:On_success - ["$BISECT_FILE"] - let enable_coverage_report job : tezos_job = job |> Tezos_ci.add_artifacts diff --git a/ci/bin/main.ml b/ci/bin/main.ml index 495414f3b95c15ed14433b99d0a98bcca37207b1..47c9f16deed3d817e3bfa9e4d4cea60d5d123127 100644 --- a/ci/bin/main.ml +++ b/ci/bin/main.ml @@ -186,7 +186,6 @@ let () = to CI-in-OCaml, they should be removed from this function *) let exclude = function | ".gitlab/ci/jobs/coverage/common.yml" - | ".gitlab/ci/jobs/coverage/oc.unified_coverage-before_merging.yml" | ".gitlab/ci/jobs/shared/images.yml" | ".gitlab/ci/jobs/shared/templates.yml" | ".gitlab/ci/jobs/test/common.yml" | ".gitlab/ci/pipelines/before_merging.yml" diff --git a/ci/bin/master_branch.ml b/ci/bin/master_branch.ml index c364f1282cb99207be800779edd178c430e5e0fd..c54b8ebe932dbbe5ffa5bb00acf687e2778bdbc7 100644 --- a/ci/bin/master_branch.ml +++ b/ci/bin/master_branch.ml @@ -98,8 +98,8 @@ let jobs = [ (* On the project default branch, we fetch coverage from the last merged MR *) "mkdir -p _coverage_report"; - "dune exec scripts/ci/download_coverage/download.exe -- -a \ - from=last-merged-pipeline --info --log-file \ + "dune exec scripts/ci/download_coverage/download.exe -- --from \ + last-merged-pipeline --info --log-file \ _coverage_report/download_coverage.log"; "./scripts/ci/report_coverage.sh"; ] diff --git a/ci/bin/tezos_ci.ml b/ci/bin/tezos_ci.ml index 02ec244b20d86b726446c53cb82fec2771eadf85..f546538ba0056e1cddac0f35828eaa8b55b6d175 100644 --- a/ci/bin/tezos_ci.ml +++ b/ci/bin/tezos_ci.ml @@ -53,6 +53,8 @@ type tezos_job = { source_position : string * int * int * int; } +let name_of_tezos_job tezos_job = tezos_job.job.name + let map_job (tezos_job : tezos_job) (f : Gitlab_ci.Types.job -> Gitlab_ci.Types.job) : tezos_job = {tezos_job with job = f tezos_job.job} @@ -151,45 +153,78 @@ module Pipeline = struct (* Check usage of [needs:] & [depends:] *) Fun.flip List.iter jobs @@ fun {job; _} -> (* Get the [needs:] / [dependencies:] of job *) - let opt_set l = String_set.of_list (Option.value ~default:[] l) in let needs = (* The mandatory set of needs *) - job.needs |> Option.value ~default:[] - |> List.filter_map (fun Gitlab_ci.Types.{job; optional} -> - if not optional then Some job else None) - |> String_set.of_list + match job.needs with + | Some needs -> + Some + (needs + |> List.filter_map (fun Gitlab_ci.Types.{job; optional} -> + if not optional then Some job else None) + |> String_set.of_list) + | None -> None + in + let dependencies = + String_set.of_list (Option.value ~default:[] job.dependencies) in - let dependencies = opt_set job.dependencies in + (* Check that jobs in [needs:]/[dependencies:] jobs are defined *) + (let dep_needs = + String_set.union + dependencies + (Option.value ~default:String_set.empty needs) + in + Fun.flip String_set.iter dep_needs @@ fun need -> + match Hashtbl.find_opt job_by_name need with + | Some _needed_job -> + (* TODO: https://gitlab.com/tezos/tezos/-/issues/7015 + check rule implication *) + () + | None -> + failwith + "[%s] job '%s' has a need on '%s' which is not defined in this \ + pipeline." + pipeline_name + job.name + need) ; (* Check that dependencies are a subset of needs. Note: this is already enforced by the smart constructor {!job} defined below. Is it redundant? Nothing enforces the usage of this smart constructor at this point.*) - String_set.iter - (fun dependency -> - if not (String_set.mem dependency needs) then - failwith - "[%s] the job '%s' has a [dependency:] on '%s' which is not \ - included in it's [need:]" - pipeline_name - job.name - dependency) - dependencies ; - (* Check that needed jobs (which thus includes dependencies) are defined *) - ( Fun.flip String_set.iter needs @@ fun need -> - match Hashtbl.find_opt job_by_name need with - | Some _needed_job -> - (* TODO: https://gitlab.com/tezos/tezos/-/issues/7015 - check rule implication *) - () - | None -> - (* TODO: https://gitlab.com/tezos/tezos/-/issues/7015 - handle optional needs *) - failwith - "[%s] job '%s' has a need on '%s' which is not defined in this \ - pipeline." - pipeline_name - job.name - need ) ; + (match needs with + | Some needs -> + String_set.iter + (fun dependency -> + if not (String_set.mem dependency needs) then + failwith + "[%s] the job '%s' has a [dependency:] on '%s' which is not \ + included in it's [need:]" + pipeline_name + job.name + dependency) + dependencies + | None -> + (* If the job has no needs, then it suffices that the dependency is in + an anterior stage. *) + let stage_index = + let stage_names = Stage.to_string_list () in + fun stage_opt -> + let stage = Option.value ~default:"test" stage_opt in + List.assoc_opt + stage + (List.combine stage_names (range 1 (List.length stage_names))) + in + String_set.iter + (fun dependency -> + (* We use [find] instead of [find_opt] *) + let dependency_job = Hashtbl.find job_by_name dependency in + if stage_index dependency_job.stage >= stage_index job.stage then + failwith + "[%s] the job '%s' has a [dependency:] on '%s' which is not in \ + a anterior stage." + pipeline_name + job.name + dependency) + dependencies) ; (* Check that all [dependencies:] are on jobs that produce artifacts *) ( Fun.flip String_set.iter dependencies @@ fun dependency -> match Hashtbl.find_opt job_by_name dependency with diff --git a/ci/bin/tezos_ci.mli b/ci/bin/tezos_ci.mli index c92d3958eaad4cae3747e01e99f1a42f87e78b29..f5d40afb757d0b22f1e0518a601c670efe546bb4 100644 --- a/ci/bin/tezos_ci.mli +++ b/ci/bin/tezos_ci.mli @@ -8,6 +8,15 @@ (** A GitLab CI job annotated with Octez-specific meta-data. *) type tezos_job +(** The name of a {!tezos_job} as given to [~name] of {!job}. + + This returns the stem of the job name. Parallel jobs produce + multiple job instances ([JOB N/M] for a {!Vector}-parallel job and + [JOB: [foo, bar, ...]] for {!Matrix}-parallel jobs) -- the stem of such + jobs is [JOB]. For non-parallel jobs, the argument given to + [~name] and the stem is equivalent. *) +val name_of_tezos_job : tezos_job -> string + (** A string that should be prepended to all generated files. Warns not to modify the generated files, and refers to the generator. *) diff --git a/script-inputs/ci-coverage-producing-jobs b/script-inputs/ci-coverage-producing-jobs new file mode 100644 index 0000000000000000000000000000000000000000..f1d6e41458378037eab9bb59603e6d6c9ba39c88 --- /dev/null +++ b/script-inputs/ci-coverage-producing-jobs @@ -0,0 +1,8 @@ +tezt-time-sensitive +tezt-memory-3k +tezt-memory-4k +tezt +tezt-flaky +oc.unit:proto-x86_64 +oc.unit:other-x86_64 +oc.unit:non-proto-x86_64 diff --git a/scripts/ci/download_coverage/download.ml b/scripts/ci/download_coverage/download.ml index 2f7a50955114b5364532c9ad86f15d0347905954..30658f6dda0da9641f5e272ea849d8ca1690c23d 100644 --- a/scripts/ci/download_coverage/download.ml +++ b/scripts/ci/download_coverage/download.ml @@ -2,15 +2,17 @@ open Tezt open Base open Tezt_gitlab -let usage () = - prerr_endline - {|Usage: dune exec scripts/ci/download_coverage/download.exe -- [-a from=last-merged-pipeline | -a from=] +let section = + Clap.section + "DOWNLOAD COVERAGE TRACES" + ~description: + {|Download coverage traces from a pipeline. Example: to fetch coverage traces from https://gitlab.com/tezos/tezos/-/pipelines/426773806, run (from the root of the repository): -dune exec scripts/ci/download_coverage/download.exe -- -a from=426773806 +dune exec scripts/ci/download_coverage/download.exe -- --from 426773806 You can use the PROJECT environment variable to specify which GitLab repository to fetch coverage traces from. Default is: tezos/tezos @@ -19,10 +21,9 @@ The script can also be used to fetch coverage traces from the last successful pi latest MR merged to the default branch (configurable through the DEFAULT_BRANCH environment variable) for a given PROJECT: -dune exec scripts/ci/download_coverage/download.exe -- -a from=last-merged-pipeline +dune exec scripts/ci/download_coverage/download.exe -- --from last-merged-pipeline -|} ; - exit 1 +|} let project = Sys.getenv_opt "PROJECT" |> Option.value ~default:"tezos/tezos" @@ -32,34 +33,20 @@ let default_branch = let coverage_traces_directory = Sys.getenv_opt "COVERAGE_OUTPUT" |> Option.value ~default:"_coverage_output" -let coverage_yml_path = - ".gitlab/ci/jobs/coverage/oc.unified_coverage-before_merging.yml" +let coverage_jobs_path = "script-inputs/ci-coverage-producing-jobs" -(** Read {!coverage_yml_path} to find the set of - jobs from which to collect coverage traces *) +(** Read {!coverage_jobs_path} to find the set of jobs from which to + collect coverage traces *) let coverage_jobs = - let coverage_yml = - Base.read_file coverage_yml_path |> String.split_on_char '\n' - in - let _, coverage_yml = - Base.span (fun line -> not (line =~ rex "^\\s+dependencies:$")) coverage_yml - in - let coverage_jobs, _ = - Base.span (fun line -> not (line =~ rex "^\\s+script:$")) coverage_yml - in + (* Note: the contents of this file only includes the stem of each job name. *) let coverage_jobs = - List.map - (function - | line -> ( - match line =~* rex {|- "(.*)"|} with - | Some job_name -> job_name - | None -> - Test.fail "Unexpected line %S in %s" line coverage_yml_path)) - (List.tl coverage_jobs) + Base.read_file coverage_jobs_path + |> String.split_on_char '\n' + |> List.filter (( <> ) "") in Log.debug "From %s, read coverage jobs: [%s]" - coverage_yml_path + coverage_jobs_path (String.concat ", " coverage_jobs) ; coverage_jobs @@ -78,6 +65,7 @@ let fetch_pipeline_coverage_from_jobs pipeline = let get_coverage_trace job = let job_id = JSON.(job |-> "id" |> as_int) in let job_name = JSON.(job |-> "name" |> as_string) in + let job_status = JSON.(job |-> "status" |> as_string) in let artifact_name = Base.replace_string (rex "[\\\\/_ @\\[\\]]+") job_name ~by:"-" ^ ".coverage" @@ -85,7 +73,17 @@ let fetch_pipeline_coverage_from_jobs pipeline = job_names are mangled into coverage trace artifact paths. *) in let artifact_path = coverage_traces_directory // artifact_name in - if List.mem job_name coverage_jobs then + if + (job_status = "success" || job_status = "failed") + && List.exists + (fun coverage_job_stem -> + job_name = coverage_job_stem + (* vector parallel *) + || job_name =~ rex ("^" ^ coverage_job_stem ^ " \\d+/\\d+$") + || (* matrix parallel *) + job_name =~ rex ("^" ^ coverage_job_stem ^ ": \\[.*\\]$")) + coverage_jobs + then Some ( Gitlab.project_job_artifact ~project ~job_id ~artifact_path (), job_name, @@ -99,12 +97,28 @@ let fetch_pipeline_coverage_from_jobs pipeline = type from = Pipeline of int | Last_merged_pipeline -let cli_get_from = - match Cli.get_string_opt "from" with - | Some "last-merged-pipeline" -> Last_merged_pipeline - | Some s -> ( - match int_of_string_opt s with Some i -> Pipeline i | None -> usage ()) - | None -> usage () +let cli_from_type = + let parse = function + | "last-merged-pipeline" -> Some Last_merged_pipeline + | s -> Option.map (fun x -> Pipeline x) (int_of_string_opt s) + in + let show = function + | Pipeline id -> string_of_int id + | Last_merged_pipeline -> "last-merged-pipeline" + in + Clap.typ ~name:"from" ~dummy:Last_merged_pipeline ~parse ~show + +let cli_from = + Clap.default + cli_from_type + ~section + ~long:"from" + ~placeholder:"PIPELINE" + ~description: + "The ID of the pipeline to fetch records from. Also accepts \ + 'last-merged-pipeline', which denotes the last pipeline for the latest \ + merge commit on the default branch." + Last_merged_pipeline let () = (* Register a test to benefit from error handling of Test.run, @@ -112,7 +126,7 @@ let () = ( Test.register ~__FILE__ ~title:"download coverage" ~tags:["update"] @@ fun () -> let* _new_coverage_traces = - match cli_get_from with + match cli_from with | Pipeline pipeline_id -> fetch_pipeline_coverage_from_jobs pipeline_id | Last_merged_pipeline -> let* pipeline_id = diff --git a/scripts/tezt_gitlab/gitlab.ml b/scripts/tezt_gitlab/gitlab.ml index 3281a5db1d110f0fd2a8f011c74762429cb8fb17..cf0fc38f14cd2a150aab79faf0668ad677ab43c7 100644 --- a/scripts/tezt_gitlab/gitlab.ml +++ b/scripts/tezt_gitlab/gitlab.ml @@ -32,13 +32,19 @@ let project_job_artifact ~project ~job_id ~artifact_path = job_id artifact_path) -let curl_params ?(fail_on_http_errors = true) ?output_path ?(location = false) - uri = +let curl_params ?(progress_meter = false) ?(fail_on_http_errors = true) + ?output_path ?(location = false) uri = (if fail_on_http_errors then ["--fail"] else []) @ (match output_path with | Some output_path -> ["--output"; output_path] | None -> []) @ (if location then ["--location"] else []) + @ (if + progress_meter + || Sys.getenv_opt "MY_CURL_VERSION_IS" + |> Option.value ~default:"" = "very_very_old" + then [] + else ["--no-progress-meter"]) @ [Uri.to_string uri] let get ?fail_on_http_errors uri =