diff --git a/.gitignore b/.gitignore index 47f8166031acc05ac6816d7a3c001d66ad00f484..43ae7341c679c041ac4eb9628fbc9fd5a5f6bf89 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ __pycache__ /tezos-codec /tezos-snoop /tezos-tps-evaluation +/tezos-tps-evaluation-* /tezos-sc-rollup-node-* /tezos-sc-rollup-client-* diff --git a/Makefile b/Makefile index d33e20e9e384782c70a41ca7249dde65503755f3..8cee1b5c8ef4ac1a4ad094b38c700cd61c849893 100644 --- a/Makefile +++ b/Makefile @@ -309,6 +309,9 @@ build-tps-deps: build-tps: lift-protocol-limits-patch build build-tezt @dune build ./src/bin_tps_evaluation @cp -f ./_build/install/default/bin/tezos-tps-evaluation . + @cp -f ./src/bin_tps_evaluation/tezos-tps-evaluation-benchmark-tps . + @cp -f ./src/bin_tps_evaluation/tezos-tps-evaluation-estimate-average-block . + @cp -f ./src/bin_tps_evaluation/tezos-tps-evaluation-gas-tps . .PHONY: docker-image-build docker-image-build: diff --git a/manifest/main.ml b/manifest/main.ml index 06b8580e15db4efd167cf43341d3f75fd825ae47..b320b32f65084a4a93162f79fe071fe599c421c8 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -3318,7 +3318,6 @@ let _tezos_tps_evaluation = caqti; caqti_driver_postgresql; caqti_lwt; - cmdliner; data_encoding; lwt; ppx_blob; diff --git a/src/bin_tps_evaluation/README.md b/src/bin_tps_evaluation/README.md index 97b22f5a1b92cdce65931add17b0698c2e2ce915..b39059288b0e00e70eaffc6d4b697c5557d4def3 100644 --- a/src/bin_tps_evaluation/README.md +++ b/src/bin_tps_evaluation/README.md @@ -74,7 +74,7 @@ It is possible to get an estimation of the maximal possible TPS by using the protocol parameters. ``` -./tezos-tps-evaluation gas-tps --average-block=src/bin_tps_evaluation/average-block.json +./tezos-tps-evaluation-gas-tps -a average-block=src/bin_tps_evaluation/average-block.json [14:26:25.243] Starting test: tezos_tps_gas [14:26:27.956] Reading description of the average block from src/bin_tps_evaluation/average-block.json [14:26:28.061] Originating smart contracts @@ -87,8 +87,9 @@ protocol parameters. This estimation is obtained by dividing the hard gas limit per block by the average transaction cost. -The `gas-tps` command will also register its result in a database if -`tezt_config.json` exists (see [these instructions][long-tezts-locally]). +The `tezos-tps-evaluation-gas-tps` command will also register its result in +a database if `tezt_config.json` exists (see [these +instructions][long-tezts-locally]). ## Running the TPS benchmark @@ -99,36 +100,23 @@ reached and after that it will run the stress test client command. The TPS parameter that is passed to the stress test command (we call this **target TPS of injection**) depends on the presence of the -`--lift-protocol-limits` flag: +`-a lift-protocol-limits` flag: -* If `--lift-protocol-limits` is passed, an arbitrary big value is passed, +* If `-a lift-protocol-limits` is passed, an arbitrary big value is passed, so that stress test will inject as fast as possible. -* If no `--lift-protocol-limits` is passed, TPS computed from gas (see the +* If no `-a lift-protocol-limits` is passed, TPS computed from gas (see the previous section for details) will be used as the target TPS of injection. By default 10 blocks will be produced, but this can be changed by supplying -the `--blocks-total` command line option. The total number of applied +the `-a blocks-total` command line option. The total number of applied operations will be divided by the total time spent producing the blocks and the resulting value will be presented as the **empirical TPS**. The benchmark is also capable of calculating **de facto TPS of injection** which is useful in judging the results. -It is possible to pass command line arguments to the underlying Tezt -framework. To accomplish that pass arguments after the double dash -delimiter. For example: - -```bash -./tezos-tps-evaluation benchmark-tps -- --help -``` - -will print command line options that the Tezt framework accepts. `-- -v` can -be helpful for obtaining a detailed log. - -Consult `./tezos-tps-evaluation benchmark-tps --help` to see all accepted -command line options. - -The `benchmark-tps` command will also register its result in a database if -`tezt_config.json` exists (see [these instructions][long-tezts-locally]). +The `tezos-tps-evaluation-benchmark-tps` command will also register its +result in a database if `tezt_config.json` exists (see [these +instructions][long-tezts-locally]). ### Making sense of the results @@ -138,7 +126,7 @@ to catch TPS regressions, but not to find where exactly the bottlenecks are. The empirical TPS is significantly affected by the hardware on which the benchmark is run and other factors, such as the amount of logging that is -performed. For example, passing `-- -v` is likely to result in much lower +performed. For example, passing `-v` is likely to result in much lower empirical TPS values. The empirical TPS should normally be very close to the de facto TPS of @@ -173,7 +161,7 @@ trustworthy: ### Example usage ``` -./tezos-tps-evaluation benchmark-tps --average-block=src/bin_tps_evaluation/average_block.json --lift-protocol-limits +./tezos-tps-evaluation-benchmark-tps -a average-block=src/bin_tps_evaluation/average_block.json -a lift-protocol-limits [12:20:57.387] Tezos TPS benchmark [12:20:57.387] Protocol: Alpha [12:20:57.387] Total number of accounts to use: 5 diff --git a/src/bin_tps_evaluation/benchmark_tps_command.ml b/src/bin_tps_evaluation/benchmark_tps_command.ml index 7e827add22d974624f961a036bd4269253a3218d..7744c2b844e9ce8ade88366d2e7387584d093988 100644 --- a/src/bin_tps_evaluation/benchmark_tps_command.ml +++ b/src/bin_tps_evaluation/benchmark_tps_command.ml @@ -243,108 +243,45 @@ let regression_handling defacto_tps_of_injection empirical_tps in save_and_check "empirical_tps" @@ fun () -> empirical_tps -module Term = struct - let accounts_total_arg = - let open Cmdliner in - let doc = "The number of bootstrap accounts to use in the benchmark" in - let docv = "ACCOUNTS_TOTAL" in - Arg.(value & opt int 5 & info ["accounts-total"] ~docv ~doc) - - let blocks_total_arg = - let open Cmdliner in - let doc = "The number of blocks to bake during the benchmark" in - let docv = "BLOCKS_TOTAL" in - Arg.(value & opt int 10 & info ["blocks-total"] ~docv ~doc) - - let average_block_path_arg = - let open Cmdliner in - let doc = "Path to the file with description of the average block" in - let docv = "AVERAGE_BLOCK_PATH" in - Arg.(value & opt (some string) None & info ["average-block"] ~docv ~doc) - - let lift_protocol_limits_arg = - let open Cmdliner in - let doc = - "Remove any protocol settings that may limit the maximum achievable TPS" - in - let docv = "LIFT_PROTOCOL_LIMITS" in - Arg.(value (flag & info ["lift-protocol-limits"] ~docv ~doc)) - - let tps_of_injection_arg = - let open Cmdliner in - let doc = "The injection TPS value that we should use" in - let docv = "TPS_OF_INJECTION" in - Arg.(value & opt (some int) None & info ["tps-of-injection"] ~docv ~doc) - - let tezt_args = - let open Cmdliner in - let doc = - "Extra arguments after -- to be passed directly to Tezt. Contains `-i` \ - by default to display info log level." - in - let docv = "TEZT_ARGS" in - Arg.(value & pos_all string [] & info [] ~docv ~doc) - - let previous_count_arg = - let open Cmdliner in - let doc = - "The number of previously recorded samples that must be compared to the \ - result of this benchmark" - in - let docv = "PREVIOUS_SAMPLE_COUNT" in - Arg.( - value & opt int 10 & info ["regression-previous-sample-count"] ~docv ~doc) - - let term = - let process accounts_total blocks_total average_block_path - lift_protocol_limits provided_tps_of_injection tezt_args previous_count - = - (try Cli.init ~args:("-i" :: tezt_args) () - with Arg.Help help_str -> - Format.eprintf "%s@." help_str ; - exit 0) ; - Long_test.init () ; - let executors = Long_test.[x86_executor1] in - Long_test.register - ~__FILE__ - ~title:"tezos_tps_benchmark" - ~tags:[] - ~timeout:(Long_test.Minutes 60) - ~executors - (fun () -> - let* (defacto_tps_of_injection, empirical_tps) = - run_benchmark - ~lift_protocol_limits - ~provided_tps_of_injection - ~accounts_total - ~blocks_total - ~average_block_path - () - in - regression_handling - defacto_tps_of_injection - empirical_tps - lift_protocol_limits - ~previous_count) ; - Test.run () ; - `Ok () - in - let open Cmdliner.Term in - ret - (const process $ accounts_total_arg $ blocks_total_arg - $ average_block_path_arg $ lift_protocol_limits_arg $ tps_of_injection_arg - $ tezt_args $ previous_count_arg) -end - -module Manpage = struct - let command_description = - "Run the benchmark and print out the results on stdout" - - let description = [`S "DESCRIPTION"; `P command_description] - - let man = description - - let info = Cmdliner.Term.info ~doc:command_description ~man "benchmark-tps" -end - -let cmd = (Term.term, Manpage.info) +let register () = + Long_test.register + ~__FILE__ + ~title:"tezos_tps_benchmark" + ~tags:["tezos_tps_benchmark"] + ~timeout:(Long_test.Minutes 60) + ~executors:Long_test.[x86_executor1] + (fun () -> + let lift_protocol_limits = + Cli.get_bool ~default:false "lift-protocol-limits" + in + let provided_tps_of_injection = + Cli.get + ~default:None + (fun s -> + match int_of_string_opt s with + | None -> None + | Some x -> Some (Some x)) + "provided_tps_of_injection" + in + let accounts_total = Cli.get_int ~default:5 "accounts-total" in + let blocks_total = Cli.get_int ~default:10 "blocks-total" in + let average_block_path = + Cli.get ~default:None (fun s -> Some (Some s)) "average-block" + in + let previous_count = + Cli.get_int ~default:10 "regression-previous-sample-count" + in + let* (defacto_tps_of_injection, empirical_tps) = + run_benchmark + ~lift_protocol_limits + ~provided_tps_of_injection + ~accounts_total + ~blocks_total + ~average_block_path + () + in + regression_handling + defacto_tps_of_injection + empirical_tps + lift_protocol_limits + ~previous_count) diff --git a/src/bin_tps_evaluation/benchmark_tps_command.mli b/src/bin_tps_evaluation/benchmark_tps_command.mli index 6a25822791dc16d201a20c422eae96e1c223ddab..5769af92b6b0f36267e3dea8d4a175f98e0a03bb 100644 --- a/src/bin_tps_evaluation/benchmark_tps_command.mli +++ b/src/bin_tps_evaluation/benchmark_tps_command.mli @@ -23,5 +23,5 @@ (* *) (*****************************************************************************) -(** Cmdliner's command definition for the "benchmark TPS" command. *) -val cmd : unit Cmdliner.Term.t * Cmdliner.Term.info +(** Register the TPS benchmark as a Tezt long test. *) +val register : unit -> unit diff --git a/src/bin_tps_evaluation/dune b/src/bin_tps_evaluation/dune index 8593842659eed430985e8ef6bfcda1ca9af6d43f..f8d5e25bdee160a791a7235fa6b6c0c50ae7be8c 100644 --- a/src/bin_tps_evaluation/dune +++ b/src/bin_tps_evaluation/dune @@ -11,7 +11,6 @@ caqti caqti-driver-postgresql caqti-lwt - cmdliner data-encoding lwt ppx_blob diff --git a/src/bin_tps_evaluation/estimate_average_block_command.ml b/src/bin_tps_evaluation/estimate_average_block_command.ml index 3bac89c0e4f97b0300bda0e4aff0a2760b97d1c3..d84659dfb5a361849e7672c07924f61e9da152be 100644 --- a/src/bin_tps_evaluation/estimate_average_block_command.ml +++ b/src/bin_tps_evaluation/estimate_average_block_command.ml @@ -198,71 +198,39 @@ module Json = struct end (** Execute the query against the database and formats the result. *) -let query_db start_date end_date limit conn_str = - Lwt_main.run - ( Lwt.bind - (Db.get_operation_summary conn_str start_date end_date ()) - (function - | Ok rows -> Lwt.return rows - | Error e -> Stdlib.failwith (Caqti_error.show e)) - >>= fun summary -> - Lwt.bind - (Db.get_top_contracts conn_str start_date end_date limit ()) - (function - | Ok rows -> Lwt.return rows - | Error e -> Stdlib.failwith (Caqti_error.show e)) - >>= fun top_contracts -> - Lwt.return (Json.show_summary summary top_contracts) ) - -module Term = struct - let connection_string_arg = - let open Cmdliner in - let doc = "PostgreSQL connection string" in - let docv = "CONNECTION_STRING" in - Arg.( - value - & opt string "postgresql://postgres:postgres@localhost:5432/postgres" - & info ["c"; "connection-string"] ~docv ~doc) - - let start_date_arg = - let open Cmdliner in - let doc = "The start date to use in the query" in - let docv = "START_DATE" in - let info = Arg.info ["s"; "start-date"] ~docv ~doc in - Arg.required (Arg.opt (Arg.some Arg.string) None info) - - let end_date_arg = - let open Cmdliner in - let doc = "The end date to use in the the query" in - let docv = "END_DATE" in - let info = Arg.info ["e"; "end-date"] ~docv ~doc in - Arg.required (Arg.opt (Arg.some Arg.string) None info) - - let contract_min_percentage_arg = - let open Cmdliner in - let doc = - "The minimum percentage of operations for a contract to be included" - in - let docv = "CONTRACT_MIN_PERCENTAGE" in - Arg.( - value & opt float 0.1 & info ["p"; "contract-min-percentage"] ~docv ~doc) - - let estimate_average_block = - let open Cmdliner.Term in - const query_db $ start_date_arg $ end_date_arg $ contract_min_percentage_arg - $ connection_string_arg -end - -module Manpage = struct - let command_description = - "Use historical data to estimate contents of the average block" - - let description = [`S "DESCRIPTION"; `P command_description] - - let man = description - - let info = - Cmdliner.Term.info ~doc:command_description ~man "estimate-average-block" -end - -let cmd = (Term.estimate_average_block, Manpage.info) +let query_db start_date end_date contract_min_percentage conn_str = + Lwt.bind (Db.get_operation_summary conn_str start_date end_date ()) (function + | Ok rows -> Lwt.return rows + | Error e -> Stdlib.failwith (Caqti_error.show e)) + >>= fun summary -> + Lwt.bind + (Db.get_top_contracts + conn_str + start_date + end_date + contract_min_percentage + ()) + (function + | Ok rows -> Lwt.return rows + | Error e -> Stdlib.failwith (Caqti_error.show e)) + >>= fun top_contracts -> Lwt.return (Json.show_summary summary top_contracts) + +let register () = + Long_test.register + ~__FILE__ + ~title:"tezos_tps_estimate_average_block" + ~tags:["tezos_tps_estimate_average_block"] + ~timeout:(Long_test.Minutes 60) + ~executors:Long_test.[x86_executor1] + (fun () -> + let connection_string = + Cli.get_string + ~default:"postgresql://postgres:postgres@localhost:5432/postgres" + "connection-string" + in + let start_date = Cli.get_string "start-date" in + let end_date = Cli.get_string "end-date" in + let contract_min_percentage = + Cli.get_float ~default:0.1 "contract-min-percentage" + in + query_db start_date end_date contract_min_percentage connection_string) diff --git a/src/bin_tps_evaluation/estimate_average_block_command.mli b/src/bin_tps_evaluation/estimate_average_block_command.mli index 93712c4979fe242a902e4ab3cd0ea128ec55aff7..4d70741b549da288b607e803b9c6098d63051e2f 100644 --- a/src/bin_tps_evaluation/estimate_average_block_command.mli +++ b/src/bin_tps_evaluation/estimate_average_block_command.mli @@ -23,5 +23,5 @@ (* *) (*****************************************************************************) -(** Cmdliner's command definition for "estimate average block" command. *) -val cmd : unit Cmdliner.Term.t * Cmdliner.Term.info +(** Register estimation of average block as a Tezt long test. *) +val register : unit -> unit diff --git a/src/bin_tps_evaluation/gas_tps_command.ml b/src/bin_tps_evaluation/gas_tps_command.ml index 1c6ca02cb7b9b1f2521179e723e9cbbb08681fe8..1bf40db88f05c96d4dda8ed9e1cfa72b07ce54ca 100644 --- a/src/bin_tps_evaluation/gas_tps_command.ml +++ b/src/bin_tps_evaluation/gas_tps_command.ml @@ -60,74 +60,24 @@ let estimate_gas_tps ~average_block_path () = Node.terminate ~kill:true node >>= fun () -> Lwt.return @@ float_of_int gas_tps -module Term = struct - let average_block_path_arg = - let open Cmdliner in - let doc = "Path to the file with description of the average block" in - let docv = "AVERAGE_BLOCK_PATH" in - Arg.(value & opt (some string) None & info ["average-block"] ~docv ~doc) - - let tezt_args = - let open Cmdliner in - let doc = - "Extra arguments after -- to be passed directly to Tezt. Contains `-i` \ - by default to display info log level." - in - let docv = "TEZT_ARGS" in - Arg.(value & pos_all string [] & info [] ~docv ~doc) - - let previous_count_arg = - let open Cmdliner in - let doc = - "The number of previously recorded samples that must be compared to the \ - result of this benchmark" - in - let docv = "PREVIOUS_SAMPLE_COUNT" in - Arg.( - value & opt int 10 & info ["regression-previous-sample-count"] ~docv ~doc) - - let term = - let process average_block_path tezt_args previous_count = - (* We are going to need to call the client stress test command here in - order to get an estimation of gas cost of various transactions that - stress test uses. This functionality is also protocol-dependent, so - we need to start a node, too. Hence we use the tezt network to spin - up the network. *) - (try Cli.init ~args:("-i" :: tezt_args) () - with Arg.Help help_str -> - Format.eprintf "%s@." help_str ; - exit 0) ; - Long_test.init () ; - let executors = Long_test.[x86_executor1] in - Long_test.register - ~__FILE__ - ~title:"tezos_tps_gas" - ~tags:[] - ~timeout:(Long_test.Minutes 1) - ~executors - (fun () -> - Long_test.measure_and_check_regression_lwt - ~previous_count - ~minimum_previous_count:previous_count - ~stddev:false - ~repeat:1 - "tps_evaluation" - @@ estimate_gas_tps ~average_block_path) ; - Test.run () ; - `Ok () - in - let open Cmdliner.Term in - ret (const process $ average_block_path_arg $ tezt_args $ previous_count_arg) -end - -module Manpage = struct - let command_description = "Estimate TPS based on gas" - - let description = [`S "DESCRIPTION"; `P command_description] - - let man = description - - let info = Cmdliner.Term.info ~doc:command_description ~man "gas-tps" -end - -let cmd = (Term.term, Manpage.info) +let register () = + Long_test.register + ~__FILE__ + ~title:"tezos_gas_tps" + ~tags:["tezos_gas_tps"] + ~timeout:(Long_test.Minutes 60) + ~executors:Long_test.[x86_executor1] + (fun () -> + let average_block_path = + Cli.get ~default:None (fun s -> Some (Some s)) "average-block" + in + let previous_count = + Cli.get_int ~default:10 "regression-previous-sample-count" + in + Long_test.measure_and_check_regression_lwt + ~previous_count + ~minimum_previous_count:previous_count + ~stddev:false + ~repeat:1 + "tps_evaluation" + @@ estimate_gas_tps ~average_block_path) diff --git a/src/bin_tps_evaluation/gas_tps_command.mli b/src/bin_tps_evaluation/gas_tps_command.mli index 99a75fba519ae75376673ef7136fd94f3d5e4b72..a50e6d45eaafbb326337a1b9935a52877683f2ac 100644 --- a/src/bin_tps_evaluation/gas_tps_command.mli +++ b/src/bin_tps_evaluation/gas_tps_command.mli @@ -23,5 +23,5 @@ (* *) (*****************************************************************************) -(** Cmdliner's command definition for the "gas TPS" command. *) -val cmd : unit Cmdliner.Term.t * Cmdliner.Term.info +(** Register gas estimation of TPS as a Tezt long test. *) +val register : unit -> unit diff --git a/src/bin_tps_evaluation/main_tps_evaluation.ml b/src/bin_tps_evaluation/main_tps_evaluation.ml index 75623ef0f474bed7f4221f0a0957c9b32dd3850f..698f892993c8b5c89b53cde1612b8c69406bf373 100644 --- a/src/bin_tps_evaluation/main_tps_evaluation.ml +++ b/src/bin_tps_evaluation/main_tps_evaluation.ml @@ -23,27 +23,9 @@ (* *) (*****************************************************************************) -let commands = - [ - Benchmark_tps_command.cmd; - Estimate_average_block_command.cmd; - Gas_tps_command.cmd; - ] - -let term = - let open Cmdliner.Term in - ret (const (`Help (`Pager, None))) - -let info = - let version = Tezos_version.Bin_version.version_string in - Cmdliner.Term.info - ~doc:"The Tezos TPS evaluation tool" - ~version - "tezos-tps-evaluation" - let () = - match Cmdliner.Term.eval_choice (term, info) commands with - | `Error _ -> exit 1 - | `Help -> exit 0 - | `Version -> exit 0 - | `Ok () -> exit 0 + Long_test.init () ; + Benchmark_tps_command.register () ; + Estimate_average_block_command.register () ; + Gas_tps_command.register () ; + Test.run () diff --git a/src/bin_tps_evaluation/perform-analysis.sh b/src/bin_tps_evaluation/perform-analysis.sh index 7c0892a3dee8dab259f3ee668be5d3672342463b..4ab9cdbc026bc968c2db04339cffe0136f24c19c 100755 --- a/src/bin_tps_evaluation/perform-analysis.sh +++ b/src/bin_tps_evaluation/perform-analysis.sh @@ -25,7 +25,7 @@ declare -r RESULT_FILE="$START_DATE-to-$END_DATE.json" docker run -d --name tezos_history_analysis -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_USER=tezos postgres:14-alpine bunzip2 -c "$DATABASE_DUMP_FILE" | docker exec -i tezos_history_analysis psql -U tezos -a tezos -../../tezos-tps-evaluation estimate-average-block -c postgresql://tezos:tezos@localhost:5432 --start "$START_DATE" --end "$END_DATE" > "$RESULT_FILE" +../../tezos-tps-evaluation-estimate-average-block -a connection-string=postgresql://tezos:tezos@localhost:5432 -a start-date="$START_DATE" -a end-date="$END_DATE" > "$RESULT_FILE" cp "$RESULT_FILE" "average-block.json" docker stop tezos_history_analysis docker rm tezos_history_analysis diff --git a/src/bin_tps_evaluation/tezos-tps-evaluation-benchmark-tps b/src/bin_tps_evaluation/tezos-tps-evaluation-benchmark-tps new file mode 100755 index 0000000000000000000000000000000000000000..bb68fd15e34113ae0d5f3571edd3c7e4d70007ca --- /dev/null +++ b/src/bin_tps_evaluation/tezos-tps-evaluation-benchmark-tps @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +./tezos-tps-evaluation -i tezos_tps_benchmark "$@" diff --git a/src/bin_tps_evaluation/tezos-tps-evaluation-estimate-average-block b/src/bin_tps_evaluation/tezos-tps-evaluation-estimate-average-block new file mode 100755 index 0000000000000000000000000000000000000000..2afa7a00718aa98ebd1349c59f00f4f227f5fc23 --- /dev/null +++ b/src/bin_tps_evaluation/tezos-tps-evaluation-estimate-average-block @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +./tezos-tps-evaluation -i tezos_tps_estimate_average_block "$@" diff --git a/src/bin_tps_evaluation/tezos-tps-evaluation-gas-tps b/src/bin_tps_evaluation/tezos-tps-evaluation-gas-tps new file mode 100755 index 0000000000000000000000000000000000000000..9b3be6bd93bfea9287bab5bb27ace6465ef76fcd --- /dev/null +++ b/src/bin_tps_evaluation/tezos-tps-evaluation-gas-tps @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +./tezos-tps-evaluation -i tezos_gas_tps "$@" diff --git a/src/bin_tps_evaluation/tezos-tps-evaluation.opam b/src/bin_tps_evaluation/tezos-tps-evaluation.opam index 820f31b7bbe96a9aacdf554dccd2980665b32d1c..0112cd0fef1a2464fa284274b1c9fe61c0b17513 100644 --- a/src/bin_tps_evaluation/tezos-tps-evaluation.opam +++ b/src/bin_tps_evaluation/tezos-tps-evaluation.opam @@ -14,7 +14,6 @@ depends: [ "caqti" "caqti-driver-postgresql" "caqti-lwt" - "cmdliner" "data-encoding" { >= "0.5.1" & < "0.6" } "lwt" { >= "5.4.0" } "tezos-baking-alpha" diff --git a/tezt/lib/cli.ml b/tezt/lib/cli.ml index ca1e1e637abde393921c30a3f6380ef3f2559107..1a58c07c74e4909a548ff207db2a2b26252a25c2 100644 --- a/tezt/lib/cli.ml +++ b/tezt/lib/cli.ml @@ -63,6 +63,7 @@ type options = { mutable junit : string option; mutable skip : int; mutable only : int option; + mutable test_args : string String_map.t; } let options = @@ -97,6 +98,7 @@ let options = junit = None; skip = 0; only = None; + test_args = String_map.empty; } let () = at_exit @@ fun () -> Option.iter close_out options.log_file @@ -167,6 +169,20 @@ let init ?args () = if value <= 0 then raise (Arg.Bad "--only must be at least one") ; options.only <- Some value in + let add_test_arg value = + let len = String.length value in + let rec find_equal i = + if i >= len then None + else if value.[i] = '=' then Some i + else find_equal (i + 1) + in + let (parameter, value) = + match find_equal 0 with + | None -> (value, "true") + | Some i -> (String.sub value 0 i, String.sub value (i + 1) (len - i - 1)) + in + options.test_args <- String_map.add parameter value options.test_args + in let spec = Arg.align [ @@ -375,6 +391,14 @@ let init ?args () = Arg.Int set_only, " Only run the first COUNT tests. This filter is applied \ after --job and --skip." ); + ( "--test-arg", + Arg.String add_test_arg, + "= Pass a generic argument to tests. Tests can get \ + this argument with Cli.get. --test-arg is a short-hand \ + for: --test-arg =true" ); + ( "-a", + Arg.String add_test_arg, + "= Same as --test-arg." ); ] in let usage = @@ -428,3 +452,29 @@ let init ?args () = | Arg.Help msg -> Printf.printf "%s" msg ; exit 0 + +let () = init () + +let get ?default parse parameter = + match String_map.find_opt parameter options.test_args with + | Some value -> ( + match parse value with + | None -> failwith (sf "invalid value for -a %s: %s" parameter value) + | Some value -> value) + | None -> ( + match default with + | None -> + failwith + (sf + "missing test argument %s, please specify it with: -a %s=" + parameter + parameter) + | Some default -> default) + +let get_bool ?default parameter = get ?default bool_of_string_opt parameter + +let get_int ?default parameter = get ?default int_of_string_opt parameter + +let get_float ?default parameter = get ?default float_of_string_opt parameter + +let get_string ?default parameter = get ?default Option.some parameter diff --git a/tezt/lib/cli.mli b/tezt/lib/cli.mli index c6e0e12e5daf4d259e93e978e131d7f472a93762..9fb13afc741be867f2083f2bf63126aad6879449 100644 --- a/tezt/lib/cli.mli +++ b/tezt/lib/cli.mli @@ -24,7 +24,13 @@ (* *) (*****************************************************************************) -(** Command-line interface. *) +(** Command-line interface. + + When this module is loaded, it parses command line options + unconditionally as a side-effect. +*) + +open Base (** Log levels for standard output. @@ -99,23 +105,43 @@ type options = { mutable junit : string option; mutable skip : int; mutable only : int option; + mutable test_args : string String_map.t; } (** Values for command-line options. *) val options : options -(** Read command-line options to initialize [options]. +(** Get the value for a parameter specified with [--test-arg]. + + Usage: [get parse parameter] + + If [--test-arg parameter=value] was specified on the command-line, + this calls [parse] on [value]. If [parse] returns [None], this fails. + If [parse] returns [Some x], this returns [x]. + + If no value for [parameter] was specified on the command-line, + this returns [default] if [default] was specified. Else, this fails. + + It is recommended to make it so that specifying parameters with [--test-arg] + is not mandatory for everyday use. This means it is recommended to always + give default values, and that those default values should be suitable + for typical test runs. For parameters that can take a small number of values, + it is usually better to register multiple tests, one for each possible value, + and to use tags to select from the command-line. + + @raise Failure if [parse] returns [None] or if [parameter] was not + specified on the command-line using [--test-arg] and no [default] + value was provided. *) +val get : ?default:'a -> (string -> 'a option) -> string -> 'a - By default arguments are read from [Sys.argv], but you can specify [args] - to override this behavior. Note that [args] must not contain the executable - name ([Sys.argv.(0)]), only actual arguments. +(** Same as [get bool_of_string_opt]. *) +val get_bool : ?default:bool -> string -> bool - If you do not call [init], [options] will contain only default values. +(** Same as [get int_of_string_opt]. *) +val get_int : ?default:int -> string -> int - [init] exits the program on failure to parse the arguments (with code 2) or - when either [-help] or [--help] is present (with code 0). +(** Same as [get float_of_string_opt]. *) +val get_float : ?default:float -> string -> float - Warning: if [--log-file] is specified, the file is truncated. - So if you call [init] several times with the same [--log-file] argument, - all logs between the calls to [init] are lost in this file. *) -val init : ?args:string list -> unit -> unit +(** Same as [get (fun x -> Some x)]. *) +val get_string : ?default:string -> string -> string diff --git a/tezt/lib/log.ml b/tezt/lib/log.ml index c482013841d91859cddff25f91c6910051389f07..d72620306b467268a77c249baa82a3aea0d3f5dd 100644 --- a/tezt/lib/log.ml +++ b/tezt/lib/log.ml @@ -66,6 +66,7 @@ let channel = junit = _; skip = _; only = _; + test_args = _; } = Cli.options in diff --git a/tezt/long_tests/main.ml b/tezt/long_tests/main.ml index 8faae6d92753b3f94630eb28ba4d66a60a79eb60..2e883e874756f3a485a150124219e7796b5e2043 100644 --- a/tezt/long_tests/main.ml +++ b/tezt/long_tests/main.ml @@ -50,7 +50,6 @@ let () = let default_executors = Long_test.[x86_executor1] let () = - Cli.init () ; Long_test.init () ; (* Register your tests here. *) (* This test depends on [Tezos_protocol_alpha.*] Tezos libraries *) diff --git a/tezt/manual_tests/main.ml b/tezt/manual_tests/main.ml index 569c4de31f18f0000e640c2a518fd726dcca2c3b..f3914108fa41f17c5ac7bd91092d082f31b7698d 100644 --- a/tezt/manual_tests/main.ml +++ b/tezt/manual_tests/main.ml @@ -28,7 +28,6 @@ as functions to be called here. *) let () = - Cli.init () ; Migration.register () ; Migration_voting.register () ; (* Test.run () should be the last statement, don't register afterwards! *) diff --git a/tezt/records/update.ml b/tezt/records/update.ml index de0fbe1f3fb9754661d07cda40570754407175db..297587b37fc8716f595ca0dcfd66f6eb6861c3d3 100644 --- a/tezt/records/update.ml +++ b/tezt/records/update.ml @@ -81,7 +81,6 @@ let fetch_pipeline_records () = Lwt_list.iter_p fetch_record records let () = - Cli.init () ; (* Register a test to benefit from error handling of Test.run, as well as [Background.start] etc. *) ( Test.register ~__FILE__ ~title:"update records" ~tags:["update"] @@ fun () -> diff --git a/tezt/self_tests/main.ml b/tezt/self_tests/main.ml index c96caf19098dbb7a00b9fdd69384f29aa65d21bf..7f3695c8e5d1aebe1ccd4cdf30ab66a69cfcf92d 100644 --- a/tezt/self_tests/main.ml +++ b/tezt/self_tests/main.ml @@ -26,7 +26,6 @@ (* Tests that test Tezt itself. *) let () = - Cli.init () ; Test_check.register () ; Test_daemon.register () ; Test_retry.register () ; diff --git a/tezt/snoop/main.ml b/tezt/snoop/main.ml index 7b64e7b7b72f4e29c46c520ac34ca085949cf795..5ffab1d004c9786ebaeaca92589e99c62abcf2bc 100644 --- a/tezt/snoop/main.ml +++ b/tezt/snoop/main.ml @@ -33,6 +33,5 @@ let run proto = Perform_inference.main ()) let () = - Cli.init () ; Background.start (fun x -> raise x) ; run Protocol.Alpha diff --git a/tezt/tests/main.ml b/tezt/tests/main.ml index 0ba837dda0c46299cfcec9189c3716d803318f73..7fdc690ab4296a323359601d80d8d4b732be363c 100644 --- a/tezt/tests/main.ml +++ b/tezt/tests/main.ml @@ -42,7 +42,6 @@ let migrate_to = Protocol.Alpha Each module defines tests which are thematically related, as functions to be called here. *) let () = - Cli.init () ; (* Tests that are relatively protocol-agnostic. We can run them on all protocols, or only one if the CI would be too slow. *) Baker_test.register ~protocols:[Alpha] ; diff --git a/tezt/vesting_contract_test/main.ml b/tezt/vesting_contract_test/main.ml index 9d16475c545cc40ef7e593000a3f98fce0078027..eaf5773aab4c6daa5ad5de82b3d6e672f9b3b9b6 100644 --- a/tezt/vesting_contract_test/main.ml +++ b/tezt/vesting_contract_test/main.ml @@ -54,7 +54,6 @@ let tests = let () = (* Register your tests here. *) - Cli.init () ; List.iter (fun (test, title, user_count) -> Test.register