From 9602d628e58ea29751fa2e7bef5c3c46d9798f38 Mon Sep 17 00:00:00 2001 From: Romain Bardou Date: Mon, 14 Mar 2022 13:00:31 +0100 Subject: [PATCH 1/2] Tezt: add --test-arg / -a --- tezt/lib/cli.ml | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ tezt/lib/cli.mli | 38 ++++++++++++++++++++++++++++++++++++++ tezt/lib/log.ml | 1 + 3 files changed, 87 insertions(+) diff --git a/tezt/lib/cli.ml b/tezt/lib/cli.ml index ca1e1e637abd..7c0e81f2e909 100644 --- a/tezt/lib/cli.ml +++ b/tezt/lib/cli.ml @@ -63,6 +63,7 @@ type options = { mutable junit : string option; mutable skip : int; mutable only : int option; + mutable test_args : string String_map.t; } let options = @@ -97,6 +98,7 @@ let options = junit = None; skip = 0; only = None; + test_args = String_map.empty; } let () = at_exit @@ fun () -> Option.iter close_out options.log_file @@ -167,6 +169,20 @@ let init ?args () = if value <= 0 then raise (Arg.Bad "--only must be at least one") ; options.only <- Some value in + let add_test_arg value = + let len = String.length value in + let rec find_equal i = + if i >= len then None + else if value.[i] = '=' then Some i + else find_equal (i + 1) + in + let (parameter, value) = + match find_equal 0 with + | None -> (value, "true") + | Some i -> (String.sub value 0 i, String.sub value (i + 1) (len - i - 1)) + in + options.test_args <- String_map.add parameter value options.test_args + in let spec = Arg.align [ @@ -375,6 +391,14 @@ let init ?args () = Arg.Int set_only, " Only run the first COUNT tests. This filter is applied \ after --job and --skip." ); + ( "--test-arg", + Arg.String add_test_arg, + "= Pass a generic argument to tests. Tests can get \ + this argument with Cli.get. --test-arg is a short-hand \ + for: --test-arg =true" ); + ( "-a", + Arg.String add_test_arg, + "= Same as --test-arg." ); ] in let usage = @@ -428,3 +452,27 @@ let init ?args () = | Arg.Help msg -> Printf.printf "%s" msg ; exit 0 + +let get ?default parse parameter = + match String_map.find_opt parameter options.test_args with + | Some value -> ( + match parse value with + | None -> failwith (sf "invalid value for -a %s: %s" parameter value) + | Some value -> value) + | None -> ( + match default with + | None -> + failwith + (sf + "missing test argument %s, please specify it with: -a %s=" + parameter + parameter) + | Some default -> default) + +let get_bool ?default parameter = get ?default bool_of_string_opt parameter + +let get_int ?default parameter = get ?default int_of_string_opt parameter + +let get_float ?default parameter = get ?default float_of_string_opt parameter + +let get_string ?default parameter = get ?default (fun x -> Some x) parameter diff --git a/tezt/lib/cli.mli b/tezt/lib/cli.mli index c6e0e12e5daf..385dafd2b4d6 100644 --- a/tezt/lib/cli.mli +++ b/tezt/lib/cli.mli @@ -26,6 +26,8 @@ (** Command-line interface. *) +open Base + (** Log levels for standard output. The list below is sorted from the most quiet level to the most verbose level. @@ -99,6 +101,7 @@ type options = { mutable junit : string option; mutable skip : int; mutable only : int option; + mutable test_args : string String_map.t; } (** Values for command-line options. *) @@ -119,3 +122,38 @@ val options : options So if you call [init] several times with the same [--log-file] argument, all logs between the calls to [init] are lost in this file. *) val init : ?args:string list -> unit -> unit + +(** Get the value for a parameter specified with [--test-arg]. + + Usage: [get parse parameter] + + If [--test-arg parameter=value] was specified on the command-line, + this calls [parse] on [value]. If [parse] returns [None], this fails. + If [parse] returns [Some x], this returns [x]. + + If no value for [parameter] was specified on the command-line, + this returns [default] if [default] was specified. Else, this fails. + + It is recommended to make it so that specifying parameters with [--test-arg] + is not mandatory for everyday use. This means it is recommended to always + give default values, and that those default values should be suitable + for typical test runs. For parameters that can take a small number of values, + it is usually better to register multiple tests, one for each possible value, + and to use tags to select from the command-line. + + @raise Failure if [parse] returns [None] or if [parameter] was not + specified on the command-line using [--test-arg] and no [default] + value was provided. *) +val get : ?default:'a -> (string -> 'a option) -> string -> 'a + +(** Same as [get bool_of_string_opt]. *) +val get_bool : ?default:bool -> string -> bool + +(** Same as [get int_of_string_opt]. *) +val get_int : ?default:int -> string -> int + +(** Same as [get float_of_string_opt]. *) +val get_float : ?default:float -> string -> float + +(** Same as [get (fun x -> Some x)]. *) +val get_string : ?default:string -> string -> string diff --git a/tezt/lib/log.ml b/tezt/lib/log.ml index c482013841d9..d72620306b46 100644 --- a/tezt/lib/log.ml +++ b/tezt/lib/log.ml @@ -66,6 +66,7 @@ let channel = junit = _; skip = _; only = _; + test_args = _; } = Cli.options in -- GitLab From ee65c398de2e592ca7edbb32c2e2b894fbc8ecd2 Mon Sep 17 00:00:00 2001 From: Mark Karpov Date: Mon, 14 Mar 2022 17:10:09 +0100 Subject: [PATCH 2/2] Tezt: drop Cli.init, TPS: refactor the app As requested by Romain, here is the change that eliminates the problematic Cli.init function from the Tezt framework. Instead of using commands with cmdliner we now use Tezt tags to select the right "command". We pass options by using the newly conceived --test-arg or -a command line option. --- .gitignore | 1 + Makefile | 3 + manifest/main.ml | 1 - src/bin_tps_evaluation/README.md | 38 ++--- .../benchmark_tps_command.ml | 147 +++++------------- .../benchmark_tps_command.mli | 4 +- src/bin_tps_evaluation/dune | 1 - .../estimate_average_block_command.ml | 104 +++++-------- .../estimate_average_block_command.mli | 4 +- src/bin_tps_evaluation/gas_tps_command.ml | 92 +++-------- src/bin_tps_evaluation/gas_tps_command.mli | 4 +- src/bin_tps_evaluation/main_tps_evaluation.ml | 28 +--- src/bin_tps_evaluation/perform-analysis.sh | 2 +- .../tezos-tps-evaluation-benchmark-tps | 3 + ...ezos-tps-evaluation-estimate-average-block | 3 + .../tezos-tps-evaluation-gas-tps | 3 + .../tezos-tps-evaluation.opam | 1 - tezt/lib/cli.ml | 4 +- tezt/lib/cli.mli | 22 +-- tezt/long_tests/main.ml | 1 - tezt/manual_tests/main.ml | 1 - tezt/records/update.ml | 1 - tezt/self_tests/main.ml | 1 - tezt/snoop/main.ml | 1 - tezt/tests/main.ml | 1 - tezt/vesting_contract_test/main.ml | 1 - 26 files changed, 145 insertions(+), 327 deletions(-) create mode 100755 src/bin_tps_evaluation/tezos-tps-evaluation-benchmark-tps create mode 100755 src/bin_tps_evaluation/tezos-tps-evaluation-estimate-average-block create mode 100755 src/bin_tps_evaluation/tezos-tps-evaluation-gas-tps diff --git a/.gitignore b/.gitignore index 47f8166031ac..43ae7341c679 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ __pycache__ /tezos-codec /tezos-snoop /tezos-tps-evaluation +/tezos-tps-evaluation-* /tezos-sc-rollup-node-* /tezos-sc-rollup-client-* diff --git a/Makefile b/Makefile index d33e20e9e384..8cee1b5c8ef4 100644 --- a/Makefile +++ b/Makefile @@ -309,6 +309,9 @@ build-tps-deps: build-tps: lift-protocol-limits-patch build build-tezt @dune build ./src/bin_tps_evaluation @cp -f ./_build/install/default/bin/tezos-tps-evaluation . + @cp -f ./src/bin_tps_evaluation/tezos-tps-evaluation-benchmark-tps . + @cp -f ./src/bin_tps_evaluation/tezos-tps-evaluation-estimate-average-block . + @cp -f ./src/bin_tps_evaluation/tezos-tps-evaluation-gas-tps . .PHONY: docker-image-build docker-image-build: diff --git a/manifest/main.ml b/manifest/main.ml index 06b8580e15db..b320b32f6508 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -3318,7 +3318,6 @@ let _tezos_tps_evaluation = caqti; caqti_driver_postgresql; caqti_lwt; - cmdliner; data_encoding; lwt; ppx_blob; diff --git a/src/bin_tps_evaluation/README.md b/src/bin_tps_evaluation/README.md index 97b22f5a1b92..b39059288b0e 100644 --- a/src/bin_tps_evaluation/README.md +++ b/src/bin_tps_evaluation/README.md @@ -74,7 +74,7 @@ It is possible to get an estimation of the maximal possible TPS by using the protocol parameters. ``` -./tezos-tps-evaluation gas-tps --average-block=src/bin_tps_evaluation/average-block.json +./tezos-tps-evaluation-gas-tps -a average-block=src/bin_tps_evaluation/average-block.json [14:26:25.243] Starting test: tezos_tps_gas [14:26:27.956] Reading description of the average block from src/bin_tps_evaluation/average-block.json [14:26:28.061] Originating smart contracts @@ -87,8 +87,9 @@ protocol parameters. This estimation is obtained by dividing the hard gas limit per block by the average transaction cost. -The `gas-tps` command will also register its result in a database if -`tezt_config.json` exists (see [these instructions][long-tezts-locally]). +The `tezos-tps-evaluation-gas-tps` command will also register its result in +a database if `tezt_config.json` exists (see [these +instructions][long-tezts-locally]). ## Running the TPS benchmark @@ -99,36 +100,23 @@ reached and after that it will run the stress test client command. The TPS parameter that is passed to the stress test command (we call this **target TPS of injection**) depends on the presence of the -`--lift-protocol-limits` flag: +`-a lift-protocol-limits` flag: -* If `--lift-protocol-limits` is passed, an arbitrary big value is passed, +* If `-a lift-protocol-limits` is passed, an arbitrary big value is passed, so that stress test will inject as fast as possible. -* If no `--lift-protocol-limits` is passed, TPS computed from gas (see the +* If no `-a lift-protocol-limits` is passed, TPS computed from gas (see the previous section for details) will be used as the target TPS of injection. By default 10 blocks will be produced, but this can be changed by supplying -the `--blocks-total` command line option. The total number of applied +the `-a blocks-total` command line option. The total number of applied operations will be divided by the total time spent producing the blocks and the resulting value will be presented as the **empirical TPS**. The benchmark is also capable of calculating **de facto TPS of injection** which is useful in judging the results. -It is possible to pass command line arguments to the underlying Tezt -framework. To accomplish that pass arguments after the double dash -delimiter. For example: - -```bash -./tezos-tps-evaluation benchmark-tps -- --help -``` - -will print command line options that the Tezt framework accepts. `-- -v` can -be helpful for obtaining a detailed log. - -Consult `./tezos-tps-evaluation benchmark-tps --help` to see all accepted -command line options. - -The `benchmark-tps` command will also register its result in a database if -`tezt_config.json` exists (see [these instructions][long-tezts-locally]). +The `tezos-tps-evaluation-benchmark-tps` command will also register its +result in a database if `tezt_config.json` exists (see [these +instructions][long-tezts-locally]). ### Making sense of the results @@ -138,7 +126,7 @@ to catch TPS regressions, but not to find where exactly the bottlenecks are. The empirical TPS is significantly affected by the hardware on which the benchmark is run and other factors, such as the amount of logging that is -performed. For example, passing `-- -v` is likely to result in much lower +performed. For example, passing `-v` is likely to result in much lower empirical TPS values. The empirical TPS should normally be very close to the de facto TPS of @@ -173,7 +161,7 @@ trustworthy: ### Example usage ``` -./tezos-tps-evaluation benchmark-tps --average-block=src/bin_tps_evaluation/average_block.json --lift-protocol-limits +./tezos-tps-evaluation-benchmark-tps -a average-block=src/bin_tps_evaluation/average_block.json -a lift-protocol-limits [12:20:57.387] Tezos TPS benchmark [12:20:57.387] Protocol: Alpha [12:20:57.387] Total number of accounts to use: 5 diff --git a/src/bin_tps_evaluation/benchmark_tps_command.ml b/src/bin_tps_evaluation/benchmark_tps_command.ml index 7e827add22d9..7744c2b844e9 100644 --- a/src/bin_tps_evaluation/benchmark_tps_command.ml +++ b/src/bin_tps_evaluation/benchmark_tps_command.ml @@ -243,108 +243,45 @@ let regression_handling defacto_tps_of_injection empirical_tps in save_and_check "empirical_tps" @@ fun () -> empirical_tps -module Term = struct - let accounts_total_arg = - let open Cmdliner in - let doc = "The number of bootstrap accounts to use in the benchmark" in - let docv = "ACCOUNTS_TOTAL" in - Arg.(value & opt int 5 & info ["accounts-total"] ~docv ~doc) - - let blocks_total_arg = - let open Cmdliner in - let doc = "The number of blocks to bake during the benchmark" in - let docv = "BLOCKS_TOTAL" in - Arg.(value & opt int 10 & info ["blocks-total"] ~docv ~doc) - - let average_block_path_arg = - let open Cmdliner in - let doc = "Path to the file with description of the average block" in - let docv = "AVERAGE_BLOCK_PATH" in - Arg.(value & opt (some string) None & info ["average-block"] ~docv ~doc) - - let lift_protocol_limits_arg = - let open Cmdliner in - let doc = - "Remove any protocol settings that may limit the maximum achievable TPS" - in - let docv = "LIFT_PROTOCOL_LIMITS" in - Arg.(value (flag & info ["lift-protocol-limits"] ~docv ~doc)) - - let tps_of_injection_arg = - let open Cmdliner in - let doc = "The injection TPS value that we should use" in - let docv = "TPS_OF_INJECTION" in - Arg.(value & opt (some int) None & info ["tps-of-injection"] ~docv ~doc) - - let tezt_args = - let open Cmdliner in - let doc = - "Extra arguments after -- to be passed directly to Tezt. Contains `-i` \ - by default to display info log level." - in - let docv = "TEZT_ARGS" in - Arg.(value & pos_all string [] & info [] ~docv ~doc) - - let previous_count_arg = - let open Cmdliner in - let doc = - "The number of previously recorded samples that must be compared to the \ - result of this benchmark" - in - let docv = "PREVIOUS_SAMPLE_COUNT" in - Arg.( - value & opt int 10 & info ["regression-previous-sample-count"] ~docv ~doc) - - let term = - let process accounts_total blocks_total average_block_path - lift_protocol_limits provided_tps_of_injection tezt_args previous_count - = - (try Cli.init ~args:("-i" :: tezt_args) () - with Arg.Help help_str -> - Format.eprintf "%s@." help_str ; - exit 0) ; - Long_test.init () ; - let executors = Long_test.[x86_executor1] in - Long_test.register - ~__FILE__ - ~title:"tezos_tps_benchmark" - ~tags:[] - ~timeout:(Long_test.Minutes 60) - ~executors - (fun () -> - let* (defacto_tps_of_injection, empirical_tps) = - run_benchmark - ~lift_protocol_limits - ~provided_tps_of_injection - ~accounts_total - ~blocks_total - ~average_block_path - () - in - regression_handling - defacto_tps_of_injection - empirical_tps - lift_protocol_limits - ~previous_count) ; - Test.run () ; - `Ok () - in - let open Cmdliner.Term in - ret - (const process $ accounts_total_arg $ blocks_total_arg - $ average_block_path_arg $ lift_protocol_limits_arg $ tps_of_injection_arg - $ tezt_args $ previous_count_arg) -end - -module Manpage = struct - let command_description = - "Run the benchmark and print out the results on stdout" - - let description = [`S "DESCRIPTION"; `P command_description] - - let man = description - - let info = Cmdliner.Term.info ~doc:command_description ~man "benchmark-tps" -end - -let cmd = (Term.term, Manpage.info) +let register () = + Long_test.register + ~__FILE__ + ~title:"tezos_tps_benchmark" + ~tags:["tezos_tps_benchmark"] + ~timeout:(Long_test.Minutes 60) + ~executors:Long_test.[x86_executor1] + (fun () -> + let lift_protocol_limits = + Cli.get_bool ~default:false "lift-protocol-limits" + in + let provided_tps_of_injection = + Cli.get + ~default:None + (fun s -> + match int_of_string_opt s with + | None -> None + | Some x -> Some (Some x)) + "provided_tps_of_injection" + in + let accounts_total = Cli.get_int ~default:5 "accounts-total" in + let blocks_total = Cli.get_int ~default:10 "blocks-total" in + let average_block_path = + Cli.get ~default:None (fun s -> Some (Some s)) "average-block" + in + let previous_count = + Cli.get_int ~default:10 "regression-previous-sample-count" + in + let* (defacto_tps_of_injection, empirical_tps) = + run_benchmark + ~lift_protocol_limits + ~provided_tps_of_injection + ~accounts_total + ~blocks_total + ~average_block_path + () + in + regression_handling + defacto_tps_of_injection + empirical_tps + lift_protocol_limits + ~previous_count) diff --git a/src/bin_tps_evaluation/benchmark_tps_command.mli b/src/bin_tps_evaluation/benchmark_tps_command.mli index 6a25822791dc..5769af92b6b0 100644 --- a/src/bin_tps_evaluation/benchmark_tps_command.mli +++ b/src/bin_tps_evaluation/benchmark_tps_command.mli @@ -23,5 +23,5 @@ (* *) (*****************************************************************************) -(** Cmdliner's command definition for the "benchmark TPS" command. *) -val cmd : unit Cmdliner.Term.t * Cmdliner.Term.info +(** Register the TPS benchmark as a Tezt long test. *) +val register : unit -> unit diff --git a/src/bin_tps_evaluation/dune b/src/bin_tps_evaluation/dune index 8593842659ee..f8d5e25bdee1 100644 --- a/src/bin_tps_evaluation/dune +++ b/src/bin_tps_evaluation/dune @@ -11,7 +11,6 @@ caqti caqti-driver-postgresql caqti-lwt - cmdliner data-encoding lwt ppx_blob diff --git a/src/bin_tps_evaluation/estimate_average_block_command.ml b/src/bin_tps_evaluation/estimate_average_block_command.ml index 3bac89c0e4f9..d84659dfb5a3 100644 --- a/src/bin_tps_evaluation/estimate_average_block_command.ml +++ b/src/bin_tps_evaluation/estimate_average_block_command.ml @@ -198,71 +198,39 @@ module Json = struct end (** Execute the query against the database and formats the result. *) -let query_db start_date end_date limit conn_str = - Lwt_main.run - ( Lwt.bind - (Db.get_operation_summary conn_str start_date end_date ()) - (function - | Ok rows -> Lwt.return rows - | Error e -> Stdlib.failwith (Caqti_error.show e)) - >>= fun summary -> - Lwt.bind - (Db.get_top_contracts conn_str start_date end_date limit ()) - (function - | Ok rows -> Lwt.return rows - | Error e -> Stdlib.failwith (Caqti_error.show e)) - >>= fun top_contracts -> - Lwt.return (Json.show_summary summary top_contracts) ) - -module Term = struct - let connection_string_arg = - let open Cmdliner in - let doc = "PostgreSQL connection string" in - let docv = "CONNECTION_STRING" in - Arg.( - value - & opt string "postgresql://postgres:postgres@localhost:5432/postgres" - & info ["c"; "connection-string"] ~docv ~doc) - - let start_date_arg = - let open Cmdliner in - let doc = "The start date to use in the query" in - let docv = "START_DATE" in - let info = Arg.info ["s"; "start-date"] ~docv ~doc in - Arg.required (Arg.opt (Arg.some Arg.string) None info) - - let end_date_arg = - let open Cmdliner in - let doc = "The end date to use in the the query" in - let docv = "END_DATE" in - let info = Arg.info ["e"; "end-date"] ~docv ~doc in - Arg.required (Arg.opt (Arg.some Arg.string) None info) - - let contract_min_percentage_arg = - let open Cmdliner in - let doc = - "The minimum percentage of operations for a contract to be included" - in - let docv = "CONTRACT_MIN_PERCENTAGE" in - Arg.( - value & opt float 0.1 & info ["p"; "contract-min-percentage"] ~docv ~doc) - - let estimate_average_block = - let open Cmdliner.Term in - const query_db $ start_date_arg $ end_date_arg $ contract_min_percentage_arg - $ connection_string_arg -end - -module Manpage = struct - let command_description = - "Use historical data to estimate contents of the average block" - - let description = [`S "DESCRIPTION"; `P command_description] - - let man = description - - let info = - Cmdliner.Term.info ~doc:command_description ~man "estimate-average-block" -end - -let cmd = (Term.estimate_average_block, Manpage.info) +let query_db start_date end_date contract_min_percentage conn_str = + Lwt.bind (Db.get_operation_summary conn_str start_date end_date ()) (function + | Ok rows -> Lwt.return rows + | Error e -> Stdlib.failwith (Caqti_error.show e)) + >>= fun summary -> + Lwt.bind + (Db.get_top_contracts + conn_str + start_date + end_date + contract_min_percentage + ()) + (function + | Ok rows -> Lwt.return rows + | Error e -> Stdlib.failwith (Caqti_error.show e)) + >>= fun top_contracts -> Lwt.return (Json.show_summary summary top_contracts) + +let register () = + Long_test.register + ~__FILE__ + ~title:"tezos_tps_estimate_average_block" + ~tags:["tezos_tps_estimate_average_block"] + ~timeout:(Long_test.Minutes 60) + ~executors:Long_test.[x86_executor1] + (fun () -> + let connection_string = + Cli.get_string + ~default:"postgresql://postgres:postgres@localhost:5432/postgres" + "connection-string" + in + let start_date = Cli.get_string "start-date" in + let end_date = Cli.get_string "end-date" in + let contract_min_percentage = + Cli.get_float ~default:0.1 "contract-min-percentage" + in + query_db start_date end_date contract_min_percentage connection_string) diff --git a/src/bin_tps_evaluation/estimate_average_block_command.mli b/src/bin_tps_evaluation/estimate_average_block_command.mli index 93712c4979fe..4d70741b549d 100644 --- a/src/bin_tps_evaluation/estimate_average_block_command.mli +++ b/src/bin_tps_evaluation/estimate_average_block_command.mli @@ -23,5 +23,5 @@ (* *) (*****************************************************************************) -(** Cmdliner's command definition for "estimate average block" command. *) -val cmd : unit Cmdliner.Term.t * Cmdliner.Term.info +(** Register estimation of average block as a Tezt long test. *) +val register : unit -> unit diff --git a/src/bin_tps_evaluation/gas_tps_command.ml b/src/bin_tps_evaluation/gas_tps_command.ml index 1c6ca02cb7b9..1bf40db88f05 100644 --- a/src/bin_tps_evaluation/gas_tps_command.ml +++ b/src/bin_tps_evaluation/gas_tps_command.ml @@ -60,74 +60,24 @@ let estimate_gas_tps ~average_block_path () = Node.terminate ~kill:true node >>= fun () -> Lwt.return @@ float_of_int gas_tps -module Term = struct - let average_block_path_arg = - let open Cmdliner in - let doc = "Path to the file with description of the average block" in - let docv = "AVERAGE_BLOCK_PATH" in - Arg.(value & opt (some string) None & info ["average-block"] ~docv ~doc) - - let tezt_args = - let open Cmdliner in - let doc = - "Extra arguments after -- to be passed directly to Tezt. Contains `-i` \ - by default to display info log level." - in - let docv = "TEZT_ARGS" in - Arg.(value & pos_all string [] & info [] ~docv ~doc) - - let previous_count_arg = - let open Cmdliner in - let doc = - "The number of previously recorded samples that must be compared to the \ - result of this benchmark" - in - let docv = "PREVIOUS_SAMPLE_COUNT" in - Arg.( - value & opt int 10 & info ["regression-previous-sample-count"] ~docv ~doc) - - let term = - let process average_block_path tezt_args previous_count = - (* We are going to need to call the client stress test command here in - order to get an estimation of gas cost of various transactions that - stress test uses. This functionality is also protocol-dependent, so - we need to start a node, too. Hence we use the tezt network to spin - up the network. *) - (try Cli.init ~args:("-i" :: tezt_args) () - with Arg.Help help_str -> - Format.eprintf "%s@." help_str ; - exit 0) ; - Long_test.init () ; - let executors = Long_test.[x86_executor1] in - Long_test.register - ~__FILE__ - ~title:"tezos_tps_gas" - ~tags:[] - ~timeout:(Long_test.Minutes 1) - ~executors - (fun () -> - Long_test.measure_and_check_regression_lwt - ~previous_count - ~minimum_previous_count:previous_count - ~stddev:false - ~repeat:1 - "tps_evaluation" - @@ estimate_gas_tps ~average_block_path) ; - Test.run () ; - `Ok () - in - let open Cmdliner.Term in - ret (const process $ average_block_path_arg $ tezt_args $ previous_count_arg) -end - -module Manpage = struct - let command_description = "Estimate TPS based on gas" - - let description = [`S "DESCRIPTION"; `P command_description] - - let man = description - - let info = Cmdliner.Term.info ~doc:command_description ~man "gas-tps" -end - -let cmd = (Term.term, Manpage.info) +let register () = + Long_test.register + ~__FILE__ + ~title:"tezos_gas_tps" + ~tags:["tezos_gas_tps"] + ~timeout:(Long_test.Minutes 60) + ~executors:Long_test.[x86_executor1] + (fun () -> + let average_block_path = + Cli.get ~default:None (fun s -> Some (Some s)) "average-block" + in + let previous_count = + Cli.get_int ~default:10 "regression-previous-sample-count" + in + Long_test.measure_and_check_regression_lwt + ~previous_count + ~minimum_previous_count:previous_count + ~stddev:false + ~repeat:1 + "tps_evaluation" + @@ estimate_gas_tps ~average_block_path) diff --git a/src/bin_tps_evaluation/gas_tps_command.mli b/src/bin_tps_evaluation/gas_tps_command.mli index 99a75fba519a..a50e6d45eaaf 100644 --- a/src/bin_tps_evaluation/gas_tps_command.mli +++ b/src/bin_tps_evaluation/gas_tps_command.mli @@ -23,5 +23,5 @@ (* *) (*****************************************************************************) -(** Cmdliner's command definition for the "gas TPS" command. *) -val cmd : unit Cmdliner.Term.t * Cmdliner.Term.info +(** Register gas estimation of TPS as a Tezt long test. *) +val register : unit -> unit diff --git a/src/bin_tps_evaluation/main_tps_evaluation.ml b/src/bin_tps_evaluation/main_tps_evaluation.ml index 75623ef0f474..698f892993c8 100644 --- a/src/bin_tps_evaluation/main_tps_evaluation.ml +++ b/src/bin_tps_evaluation/main_tps_evaluation.ml @@ -23,27 +23,9 @@ (* *) (*****************************************************************************) -let commands = - [ - Benchmark_tps_command.cmd; - Estimate_average_block_command.cmd; - Gas_tps_command.cmd; - ] - -let term = - let open Cmdliner.Term in - ret (const (`Help (`Pager, None))) - -let info = - let version = Tezos_version.Bin_version.version_string in - Cmdliner.Term.info - ~doc:"The Tezos TPS evaluation tool" - ~version - "tezos-tps-evaluation" - let () = - match Cmdliner.Term.eval_choice (term, info) commands with - | `Error _ -> exit 1 - | `Help -> exit 0 - | `Version -> exit 0 - | `Ok () -> exit 0 + Long_test.init () ; + Benchmark_tps_command.register () ; + Estimate_average_block_command.register () ; + Gas_tps_command.register () ; + Test.run () diff --git a/src/bin_tps_evaluation/perform-analysis.sh b/src/bin_tps_evaluation/perform-analysis.sh index 7c0892a3dee8..4ab9cdbc026b 100755 --- a/src/bin_tps_evaluation/perform-analysis.sh +++ b/src/bin_tps_evaluation/perform-analysis.sh @@ -25,7 +25,7 @@ declare -r RESULT_FILE="$START_DATE-to-$END_DATE.json" docker run -d --name tezos_history_analysis -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_USER=tezos postgres:14-alpine bunzip2 -c "$DATABASE_DUMP_FILE" | docker exec -i tezos_history_analysis psql -U tezos -a tezos -../../tezos-tps-evaluation estimate-average-block -c postgresql://tezos:tezos@localhost:5432 --start "$START_DATE" --end "$END_DATE" > "$RESULT_FILE" +../../tezos-tps-evaluation-estimate-average-block -a connection-string=postgresql://tezos:tezos@localhost:5432 -a start-date="$START_DATE" -a end-date="$END_DATE" > "$RESULT_FILE" cp "$RESULT_FILE" "average-block.json" docker stop tezos_history_analysis docker rm tezos_history_analysis diff --git a/src/bin_tps_evaluation/tezos-tps-evaluation-benchmark-tps b/src/bin_tps_evaluation/tezos-tps-evaluation-benchmark-tps new file mode 100755 index 000000000000..bb68fd15e341 --- /dev/null +++ b/src/bin_tps_evaluation/tezos-tps-evaluation-benchmark-tps @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +./tezos-tps-evaluation -i tezos_tps_benchmark "$@" diff --git a/src/bin_tps_evaluation/tezos-tps-evaluation-estimate-average-block b/src/bin_tps_evaluation/tezos-tps-evaluation-estimate-average-block new file mode 100755 index 000000000000..2afa7a00718a --- /dev/null +++ b/src/bin_tps_evaluation/tezos-tps-evaluation-estimate-average-block @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +./tezos-tps-evaluation -i tezos_tps_estimate_average_block "$@" diff --git a/src/bin_tps_evaluation/tezos-tps-evaluation-gas-tps b/src/bin_tps_evaluation/tezos-tps-evaluation-gas-tps new file mode 100755 index 000000000000..9b3be6bd93bf --- /dev/null +++ b/src/bin_tps_evaluation/tezos-tps-evaluation-gas-tps @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +./tezos-tps-evaluation -i tezos_gas_tps "$@" diff --git a/src/bin_tps_evaluation/tezos-tps-evaluation.opam b/src/bin_tps_evaluation/tezos-tps-evaluation.opam index 820f31b7bbe9..0112cd0fef1a 100644 --- a/src/bin_tps_evaluation/tezos-tps-evaluation.opam +++ b/src/bin_tps_evaluation/tezos-tps-evaluation.opam @@ -14,7 +14,6 @@ depends: [ "caqti" "caqti-driver-postgresql" "caqti-lwt" - "cmdliner" "data-encoding" { >= "0.5.1" & < "0.6" } "lwt" { >= "5.4.0" } "tezos-baking-alpha" diff --git a/tezt/lib/cli.ml b/tezt/lib/cli.ml index 7c0e81f2e909..1a58c07c74e4 100644 --- a/tezt/lib/cli.ml +++ b/tezt/lib/cli.ml @@ -453,6 +453,8 @@ let init ?args () = Printf.printf "%s" msg ; exit 0 +let () = init () + let get ?default parse parameter = match String_map.find_opt parameter options.test_args with | Some value -> ( @@ -475,4 +477,4 @@ let get_int ?default parameter = get ?default int_of_string_opt parameter let get_float ?default parameter = get ?default float_of_string_opt parameter -let get_string ?default parameter = get ?default (fun x -> Some x) parameter +let get_string ?default parameter = get ?default Option.some parameter diff --git a/tezt/lib/cli.mli b/tezt/lib/cli.mli index 385dafd2b4d6..9fb13afc741b 100644 --- a/tezt/lib/cli.mli +++ b/tezt/lib/cli.mli @@ -24,7 +24,11 @@ (* *) (*****************************************************************************) -(** Command-line interface. *) +(** Command-line interface. + + When this module is loaded, it parses command line options + unconditionally as a side-effect. +*) open Base @@ -107,22 +111,6 @@ type options = { (** Values for command-line options. *) val options : options -(** Read command-line options to initialize [options]. - - By default arguments are read from [Sys.argv], but you can specify [args] - to override this behavior. Note that [args] must not contain the executable - name ([Sys.argv.(0)]), only actual arguments. - - If you do not call [init], [options] will contain only default values. - - [init] exits the program on failure to parse the arguments (with code 2) or - when either [-help] or [--help] is present (with code 0). - - Warning: if [--log-file] is specified, the file is truncated. - So if you call [init] several times with the same [--log-file] argument, - all logs between the calls to [init] are lost in this file. *) -val init : ?args:string list -> unit -> unit - (** Get the value for a parameter specified with [--test-arg]. Usage: [get parse parameter] diff --git a/tezt/long_tests/main.ml b/tezt/long_tests/main.ml index 8faae6d92753..2e883e874756 100644 --- a/tezt/long_tests/main.ml +++ b/tezt/long_tests/main.ml @@ -50,7 +50,6 @@ let () = let default_executors = Long_test.[x86_executor1] let () = - Cli.init () ; Long_test.init () ; (* Register your tests here. *) (* This test depends on [Tezos_protocol_alpha.*] Tezos libraries *) diff --git a/tezt/manual_tests/main.ml b/tezt/manual_tests/main.ml index 569c4de31f18..f3914108fa41 100644 --- a/tezt/manual_tests/main.ml +++ b/tezt/manual_tests/main.ml @@ -28,7 +28,6 @@ as functions to be called here. *) let () = - Cli.init () ; Migration.register () ; Migration_voting.register () ; (* Test.run () should be the last statement, don't register afterwards! *) diff --git a/tezt/records/update.ml b/tezt/records/update.ml index de0fbe1f3fb9..297587b37fc8 100644 --- a/tezt/records/update.ml +++ b/tezt/records/update.ml @@ -81,7 +81,6 @@ let fetch_pipeline_records () = Lwt_list.iter_p fetch_record records let () = - Cli.init () ; (* Register a test to benefit from error handling of Test.run, as well as [Background.start] etc. *) ( Test.register ~__FILE__ ~title:"update records" ~tags:["update"] @@ fun () -> diff --git a/tezt/self_tests/main.ml b/tezt/self_tests/main.ml index c96caf19098d..7f3695c8e5d1 100644 --- a/tezt/self_tests/main.ml +++ b/tezt/self_tests/main.ml @@ -26,7 +26,6 @@ (* Tests that test Tezt itself. *) let () = - Cli.init () ; Test_check.register () ; Test_daemon.register () ; Test_retry.register () ; diff --git a/tezt/snoop/main.ml b/tezt/snoop/main.ml index 7b64e7b7b72f..5ffab1d004c9 100644 --- a/tezt/snoop/main.ml +++ b/tezt/snoop/main.ml @@ -33,6 +33,5 @@ let run proto = Perform_inference.main ()) let () = - Cli.init () ; Background.start (fun x -> raise x) ; run Protocol.Alpha diff --git a/tezt/tests/main.ml b/tezt/tests/main.ml index 0ba837dda0c4..7fdc690ab429 100644 --- a/tezt/tests/main.ml +++ b/tezt/tests/main.ml @@ -42,7 +42,6 @@ let migrate_to = Protocol.Alpha Each module defines tests which are thematically related, as functions to be called here. *) let () = - Cli.init () ; (* Tests that are relatively protocol-agnostic. We can run them on all protocols, or only one if the CI would be too slow. *) Baker_test.register ~protocols:[Alpha] ; diff --git a/tezt/vesting_contract_test/main.ml b/tezt/vesting_contract_test/main.ml index 9d16475c545c..eaf5773aab4c 100644 --- a/tezt/vesting_contract_test/main.ml +++ b/tezt/vesting_contract_test/main.ml @@ -54,7 +54,6 @@ let tests = let () = (* Register your tests here. *) - Cli.init () ; List.iter (fun (test, title, user_count) -> Test.register -- GitLab