From ef27c29ca520edfd9814c1552059857c4f0cbb37 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Fri, 6 Sep 2024 12:49:56 +0300 Subject: [PATCH] Profiler: introduce shell profilers Co-authored-by: vbot --- src/bin_node/node_replay_command.ml | 16 +++++ src/bin_node/node_run_command.ml | 16 +++++ .../environment_profiler.ml | 10 +++ .../tezos_protocol_environment.ml | 1 + src/lib_shell/chain_validator.ml | 19 ++++++ src/lib_shell/profiler_directory.ml | 11 ++++ src/lib_shell_services/shell_profiling.ml | 64 +++++++++++++++++++ 7 files changed, 137 insertions(+) create mode 100644 src/lib_protocol_environment/environment_profiler.ml create mode 100644 src/lib_shell/profiler_directory.ml create mode 100644 src/lib_shell_services/shell_profiling.ml diff --git a/src/bin_node/node_replay_command.ml b/src/bin_node/node_replay_command.ml index 1c8b9e66c0d9..459413942c7c 100644 --- a/src/bin_node/node_replay_command.ml +++ b/src/bin_node/node_replay_command.ml @@ -492,6 +492,22 @@ let run ?verbosity ~singleprocess ~strict ~operation_metadata_size_limit let*! () = Tezos_base_unix.Internal_event_unix.init ~config:internal_events () in + (match Option.map String.lowercase_ascii @@ Sys.getenv_opt "PROFILING" with + | Some (("true" | "on" | "yes" | "terse" | "detailed" | "verbose") as mode) -> + let max_lod = + match mode with + | "detailed" -> Profiler.Detailed + | "verbose" -> Profiler.Verbose + | _ -> Profiler.Terse + in + let instance = + Profiler.instance + Tezos_base_unix.Simple_profiler.auto_write_to_txt_file + Filename.Infix.(config.data_dir // "/node_profiling.txt", max_lod) + in + Tezos_base.Profiler.(plug main) instance ; + Tezos_protocol_environment.Environment_profiler.plug instance + | _ -> ()) ; Updater.init (Data_version.protocol_dir config.data_dir) ; Lwt_exit.( wrap_and_exit diff --git a/src/bin_node/node_run_command.ml b/src/bin_node/node_run_command.ml index 059a131a55b6..3656fe8471ec 100644 --- a/src/bin_node/node_run_command.ml +++ b/src/bin_node/node_run_command.ml @@ -723,6 +723,22 @@ let run ?verbosity ?sandbox ?target ?(cli_warnings = []) let*! () = Tezos_base_unix.Internal_event_unix.init ~config:internal_events () in + let () = + match Option.map String.lowercase_ascii @@ Sys.getenv_opt "PROFILING" with + | Some (("true" | "on" | "yes" | "terse" | "detailed" | "verbose") as mode) + -> + let max_lod = + match mode with + | "detailed" -> Profiler.Detailed + | "verbose" -> Profiler.Verbose + | _ -> Profiler.Terse + in + let profiler_maker = + Tezos_shell.Profiler_directory.profiler_maker config.data_dir max_lod + in + Shell_profiling.activate_all ~profiler_maker + | _ -> () + in let*! () = Lwt_list.iter_s (fun evt -> Internal_event.Simple.emit evt ()) cli_warnings in diff --git a/src/lib_protocol_environment/environment_profiler.ml b/src/lib_protocol_environment/environment_profiler.ml new file mode 100644 index 000000000000..24535d4d3f10 --- /dev/null +++ b/src/lib_protocol_environment/environment_profiler.ml @@ -0,0 +1,10 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* SPDX-FileCopyrightText: 2024 Nomadic Labs *) +(* *) +(*****************************************************************************) + +let profiler = Profiler.unplugged () + +include (val Profiler.wrap profiler) diff --git a/src/lib_protocol_environment/tezos_protocol_environment.ml b/src/lib_protocol_environment/tezos_protocol_environment.ml index 1610e8087909..47099cac9ad7 100644 --- a/src/lib_protocol_environment/tezos_protocol_environment.ml +++ b/src/lib_protocol_environment/tezos_protocol_environment.ml @@ -26,6 +26,7 @@ module Context = Environment_context.Context module Register = Environment_context.Register +module Environment_profiler = Environment_profiler let err_implementation_mismatch = Environment_context.err_implementation_mismatch diff --git a/src/lib_shell/chain_validator.ml b/src/lib_shell/chain_validator.ml index 2493cecbccbc..688cf80bfd9a 100644 --- a/src/lib_shell/chain_validator.ml +++ b/src/lib_shell/chain_validator.ml @@ -464,6 +464,24 @@ let may_synchronise_context synchronisation_state chain_store = Context_ops.sync context_index else Lwt.return_unit +let reset_profilers block = + let profilers = + Shell_profiling. + [ + p2p_reader_profiler; + requester_profiler; + chain_validator_profiler; + rpc_server_profiler; + ] + in + List.iter + (fun profiler -> + (try Tezos_base.Profiler.stop profiler with _ -> ()) ; + Tezos_base.Profiler.record + profiler + (Block_hash.to_b58check (Store.Block.hash block))) + profilers + let on_validation_request w peer start_testchain active_chains spawn_child block = let open Lwt_result_syntax in @@ -484,6 +502,7 @@ let on_validation_request w peer start_testchain active_chains spawn_child block if not accepted_head then return Ignored_head else let* previous = Store.Chain.set_head chain_store block in + reset_profilers block ; let () = if is_bootstrapped nv then Distributed_db.Advertise.current_head nv.chain_db block diff --git a/src/lib_shell/profiler_directory.ml b/src/lib_shell/profiler_directory.ml new file mode 100644 index 000000000000..c97c39bceb67 --- /dev/null +++ b/src/lib_shell/profiler_directory.ml @@ -0,0 +1,11 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* SPDX-FileCopyrightText: 2024 Nomadic Labs *) +(* *) +(*****************************************************************************) + +let profiler_maker data_dir ~name max_lod = + Tezos_base.Profiler.instance + Tezos_base_unix.Simple_profiler.auto_write_to_txt_file + Filename.Infix.((data_dir // name) ^ "_profiling.txt", max_lod) diff --git a/src/lib_shell_services/shell_profiling.ml b/src/lib_shell_services/shell_profiling.ml new file mode 100644 index 000000000000..143fe71ca95f --- /dev/null +++ b/src/lib_shell_services/shell_profiling.ml @@ -0,0 +1,64 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* SPDX-FileCopyrightText: 2024 Nomadic Labs *) +(* *) +(*****************************************************************************) + +open Tezos_base.Profiler + +let mempool_profiler = unplugged () + +let store_profiler = unplugged () + +let chain_validator_profiler = unplugged () + +let block_validator_profiler = unplugged () + +let rpc_server_profiler = unplugged () + +let create_reset_block_section profiler = + let last_block = ref None in + fun b -> + match !last_block with + | None -> + record profiler (Block_hash.to_b58check b) ; + last_block := Some b + | Some b' when Block_hash.equal b' b -> () + | Some _ -> + stop profiler ; + record profiler (Block_hash.to_b58check b) ; + last_block := Some b + +let merge_profiler = unplugged () + +let p2p_reader_profiler = unplugged () + +let requester_profiler = unplugged () + +let all_profilers = + [ + ("mempool", mempool_profiler); + ("store", store_profiler); + ("chain_validator", chain_validator_profiler); + ("block_validator", block_validator_profiler); + ("merge", merge_profiler); + ("p2p_reader", p2p_reader_profiler); + ("requester", requester_profiler); + ] + +let activate_all ~profiler_maker = + List.iter (fun (name, p) -> plug p (profiler_maker ~name)) all_profilers + +let deactivate_all () = + List.iter (fun (_name, p) -> close_and_unplug_all p) all_profilers + +let activate ~profiler_maker name = + List.assoc ~equal:( = ) name all_profilers |> function + | None -> Format.ksprintf invalid_arg "unknown '%s' profiler" name + | Some p -> plug p (profiler_maker ~name) + +let deactivate name = + List.assoc ~equal:( = ) name all_profilers |> function + | None -> Format.ksprintf invalid_arg "unknown '%s' profiler" name + | Some p -> close_and_unplug_all p -- GitLab