diff --git a/.gitlab/ci/tezt.yml b/.gitlab/ci/tezt.yml index 0a6379bb2f2a52b98a2d68f224ff32739c577960..1c18cbbbb1eefd7f0515e8eecb85e1948fffff40 100644 --- a/.gitlab/ci/tezt.yml +++ b/.gitlab/ci/tezt.yml @@ -1,17 +1,42 @@ -# We set a global timeout of 55min (3300s) so that in case of timeout, -# we still get the artifact (CI timeouts cause artifacts to not be uploaded). -tezt:main: +# We use `scripts/run-tezt-tests-ci.sh` to split the Tezt tests into ranges: + +tezt:run-1-33: + extends: .test_template + before_script: + - make + script: + - sh scripts/run-tezt-tests-ci.sh 1 33 + artifacts: + paths: + - tezt.log + expire_in: 1 day + when: on_failure + +tezt:run-34-66: extends: .test_template before_script: - make script: - - dune exec tezt/tests/main.exe -- --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 3300 --time + - sh scripts/run-tezt-tests-ci.sh 34 66 artifacts: paths: - tezt.log expire_in: 1 day when: on_failure +tezt:run-67-end: + extends: .test_template + before_script: + - make + script: + - sh scripts/run-tezt-tests-ci.sh 67 + artifacts: + paths: + - tezt.log + expire_in: 1 day + when: on_failure + + tezt:manual:migration: extends: .test_template when: manual diff --git a/scripts/run-tezt-tests-ci.sh b/scripts/run-tezt-tests-ci.sh new file mode 100755 index 0000000000000000000000000000000000000000..46519fda6e3d4f4bea661c7ea5f1cdd732cfa288 --- /dev/null +++ b/scripts/run-tezt-tests-ci.sh @@ -0,0 +1,60 @@ +#! /bin/sh + +set -e + +usage () { + cat >&2 < [] + +This script allows the CI to run a subset of the Tezt tests (without +hard-to-maintain knowledge of all the tests). + +It builds a temporary script containing a big shell command and then either +displays it when 'dry_run=true' or runs it. + +Examples: + + dry_run=true scripts/run-tezt-tests-ci.sh 98 + +displays a command running the tests in the range [98; last]. + + scripts/run-tezt-tests-ci.sh 5 9 + +runs the tests from the range [5; 9]. +EOF +} + +from_line="$1" +if [ "$from_line" = "" ] ; then + usage + exit 2 +fi +to_line="$2" +if [ "$to_line" = "" ] ; then + to_line=$(dune exec tezt/tests/main.exe -- --list-tsv | wc -l) +fi + +to_run=$(mktemp) +echo "Using $to_run [ $from_line , $to_line ]" >&2 + +cat > "$to_run" <" \' for each test in the range: +dune exec tezt/tests/main.exe -- --list-tsv \ + | sed -n "$from_line,$to_line p" \ + | cut -f 2 \ + | sed "s/\(.*\)/--test \"\1\" \\\\/" >> "$to_run" +# the previous line ends with '\' so we add an empty variable to help the +# shell succeed in parsing the command: +echo '$extra_empty_arg' >> "$to_run" + +if [ "$dry_run" = "true" ] ; then + echo "Selected from $from_line to $to_line:" + cat "$to_run" +else + cat "$to_run" + sh "$to_run" +fi diff --git a/tezt/lib/cli.ml b/tezt/lib/cli.ml index ebfb21cc701ae3e046164a3b0811f8d095677e42..12b492df35e44059cf25dc3fb1a9541ddd296d98 100644 --- a/tezt/lib/cli.ml +++ b/tezt/lib/cli.ml @@ -40,7 +40,7 @@ type options = { tests_to_run : string list; tags_to_run : string list; tags_not_to_run : string list; - list : bool; + list : [`Ascii_art | `Tsv] option; global_timeout : float option; test_timeout : float option; reset_regressions : bool; @@ -62,7 +62,7 @@ let options = let tests_to_run = ref [] in let tags_to_run = ref [] in let tags_not_to_run = ref [] in - let list = ref false in + let list = ref None in let global_timeout = ref None in let test_timeout = ref None in let reset_regressions = ref false in @@ -135,8 +135,14 @@ let options = " If a test fails, continue with the remaining tests instead of \ stopping. Aborting manually with Ctrl+C still stops everything." ); ("-k", Arg.Set keep_going, " Same as --keep-going."); - ("--list", Arg.Set list, " List tests instead of running them."); - ("-l", Arg.Set list, " Same as --list."); + ( "--list", + Arg.Unit (fun () -> list := Some `Ascii_art), + " List tests instead of running them." ); + ("-l", Arg.Unit (fun () -> list := Some `Ascii_art), " Same as --list."); + ( "--list-tsv", + Arg.Unit (fun () -> list := Some `Tsv), + " List tests instead of running them but one-per-line, as \ + tab-separated-values." ); ( "--file", Arg.String (fun file -> files_to_run := file :: !files_to_run), " Only run tests implemented in source file FILE (see \ diff --git a/tezt/lib/cli.mli b/tezt/lib/cli.mli index 6f20fd146c4012d636a09ace05574026e1236bf7..7d99ab3b87efa608a3ddddd81dd4981b8c906f7e 100644 --- a/tezt/lib/cli.mli +++ b/tezt/lib/cli.mli @@ -71,7 +71,7 @@ type options = { tests_to_run : string list; tags_to_run : string list; tags_not_to_run : string list; - list : bool; + list : [`Ascii_art | `Tsv] option; global_timeout : float option; test_timeout : float option; reset_regressions : bool; diff --git a/tezt/lib/log.ml b/tezt/lib/log.ml index 9604418fa469704cf05c30a91fd2a0f100fec33c..3c7068e9a636b21a66b0e44388f1d9eb1e22fa03 100644 --- a/tezt/lib/log.ml +++ b/tezt/lib/log.ml @@ -24,6 +24,36 @@ (* *) (*****************************************************************************) +(* The expected output of a normal run of Tezt is the log of the tests. + So these should be on stdout. + However, for some commands (especially those that are expected to be parsed + by scripts such as --list-tsv), the expected output contains no logs, only data. + For those commands, we thus log to stderr instead. We don't expect much logs + except maybe warnings like "leftover files from a previous run". *) +let channel = + (* The use of a pattern with all fields ensures that we will update this if we + add new commands that should log to stderr. *) + let { Cli.color = _; + log_level = _; + log_file = _; + log_buffer_size = _; + commands = _; + temporary_file_mode = _; + keep_going = _; + files_to_run = _; + tests_to_run = _; + tags_to_run = _; + tags_not_to_run = _; + list; + global_timeout = _; + test_timeout = _; + reset_regressions = _; + loop = _; + time = _ } = + Cli.options + in + match list with None -> stdout | Some (`Ascii_art | `Tsv) -> stderr + (* In theory we could simply escape spaces, backslashes, double quotes and single quotes. But 'some long argument' is arguably more readable than some\ long\ argument. We use this quoting method if the string contains no single quote. *) @@ -207,10 +237,10 @@ let log_string ~(level : Cli.log_level) ?color ?prefix ?prefix_color message = ( if level = Error then Log_buffer.iter @@ fun line -> - log_line_to ~use_colors:Cli.options.color line stdout ) ; + log_line_to ~use_colors:Cli.options.color line channel ) ; Log_buffer.reset () ; - log_line_to ~use_colors:Cli.options.color line stdout ; - flush stdout + log_line_to ~use_colors:Cli.options.color line channel ; + flush channel | ((Quiet | Error | Warn | Report | Info), _) -> Log_buffer.push line in diff --git a/tezt/lib/test.ml b/tezt/lib/test.ml index c222b339690e7520e7a49ec299f3923b76d1e2ea..7301266b33fa76e99c7bbda5f2fce02e03e47f44 100644 --- a/tezt/lib/test.ml +++ b/tezt/lib/test.ml @@ -265,66 +265,74 @@ type test = { (* List of tests added using [add] and that match command-line filters. *) let list : test list ref = ref [] -let list_tests () = - let file_header = "FILE" in - let title_header = "TITLE" in - let tags_header = "TAGS" in - let list = - List.map - (fun {file; title; tags; _} -> (file, title, String.concat ", " tags)) - !list - in - (* Compute the size of each column. *) - let (file_size, title_size, tags_size) = - List.fold_left - (fun (max_file, max_title, max_tags) (file, title, tags) -> - ( max max_file (String.length file), - max max_title (String.length title), - max max_tags (String.length tags) )) - ( String.length file_header, - String.length title_header, - String.length tags_header ) - list - in - (* Prepare the line separator. *) - let line = - "+" - ^ String.make (file_size + 2) '-' - ^ "+" - ^ String.make (title_size + 2) '-' - ^ "+" - ^ String.make (tags_size + 2) '-' - ^ "+\n" - in - (* Print the header row. *) - print_string line ; - let center size header = - let padding = size - String.length header in - let left_padding = padding / 2 in - let right_padding = padding - left_padding in - String.make left_padding ' ' ^ header ^ String.make right_padding ' ' - in - Printf.printf - "| %s | %s | %s |\n" - (center file_size file_header) - (center title_size title_header) - (center tags_size tags_header) ; - print_string line ; - (* Print rows. *) - let pad size text = - let padding = size - String.length text in - text ^ String.make padding ' ' - in - List.iter - (fun (file, title, tags) -> +let list_tests format = + match format with + | `Tsv -> + List.iter + (fun {file; title; tags; _} -> + Printf.printf "%s\t%s\t%s\n%!" file title (String.concat " " tags)) + !list + | `Ascii_art -> + let file_header = "FILE" in + let title_header = "TITLE" in + let tags_header = "TAGS" in + let list = + List.map + (fun {file; title; tags; _} -> + (file, title, String.concat ", " tags)) + !list + in + (* Compute the size of each column. *) + let (file_size, title_size, tags_size) = + List.fold_left + (fun (max_file, max_title, max_tags) (file, title, tags) -> + ( max max_file (String.length file), + max max_title (String.length title), + max max_tags (String.length tags) )) + ( String.length file_header, + String.length title_header, + String.length tags_header ) + list + in + (* Prepare the line separator. *) + let line = + "+" + ^ String.make (file_size + 2) '-' + ^ "+" + ^ String.make (title_size + 2) '-' + ^ "+" + ^ String.make (tags_size + 2) '-' + ^ "+\n" + in + (* Print the header row. *) + print_string line ; + let center size header = + let padding = size - String.length header in + let left_padding = padding / 2 in + let right_padding = padding - left_padding in + String.make left_padding ' ' ^ header ^ String.make right_padding ' ' + in Printf.printf "| %s | %s | %s |\n" - (pad file_size file) - (pad title_size title) - (pad tags_size tags)) - list ; - if list <> [] then print_string line ; - () + (center file_size file_header) + (center title_size title_header) + (center tags_size tags_header) ; + print_string line ; + (* Print rows. *) + let pad size text = + let padding = size - String.length text in + text ^ String.make padding ' ' + in + List.iter + (fun (file, title, tags) -> + Printf.printf + "| %s | %s | %s |\n" + (pad file_size file) + (pad title_size title) + (pad tags_size tags)) + list ; + if list <> [] then print_string line ; + () let display_time_summary () = let sum_time = List.fold_left (fun acc {time; _} -> acc +. time) 0. in @@ -399,22 +407,24 @@ let run () = Cli.options.tests_to_run @ Cli.options.tags_to_run @ List.map (sf "/%s") Cli.options.tags_not_to_run )) ; - if not Cli.options.list then + if Cli.options.list = None then prerr_endline "You can use --list to get the list of tests and their tags." ) ; (* Actually run the tests (or list them). *) - if Cli.options.list then list_tests () - else - let rec run iteration = - let run_and_measure_time test = - let start = Unix.gettimeofday () in - really_run ~iteration test.title test.body ; - let time = Unix.gettimeofday () -. start in - test.time <- test.time +. time + match Cli.options.list with + | Some format -> + list_tests format + | None -> + let rec run iteration = + let run_and_measure_time test = + let start = Unix.gettimeofday () in + really_run ~iteration test.title test.body ; + let time = Unix.gettimeofday () -. start in + test.time <- test.time +. time + in + List.iter run_and_measure_time !list ; + if Cli.options.loop then run (iteration + 1) in - List.iter run_and_measure_time !list ; - if Cli.options.loop then run (iteration + 1) - in - run 1 ; - if !a_test_failed then exit 1 ; - if Cli.options.time then display_time_summary () + run 1 ; + if !a_test_failed then exit 1 ; + if Cli.options.time then display_time_summary ()