From 4b117761420be5b0e2d57e343bc72ca26a8215f1 Mon Sep 17 00:00:00 2001 From: mattiasdrp Date: Tue, 17 Jun 2025 16:28:22 +0200 Subject: [PATCH 1/2] Profiler: Allow multiple backends to be plugged when profiling --- src/bin_node/node_replay_command.ml | 24 ++++++++------ src/bin_node/node_run_command.ml | 24 ++++++++------ src/lib_agnostic_baker/daemon.ml | 11 ++++--- src/lib_dal_node/daemon.ml | 11 ++++--- .../backends/complex/prometheus_profiler.mli | 2 +- src/lib_profiler/unix/profiler_directory.ml | 11 ++++--- src/lib_profiler/unix/profiler_instance.ml | 32 ++++++++++++------- src/lib_profiler/unix/profiler_instance.mli | 12 +++---- src/lib_profiler/unix/profiler_services.ml | 14 ++++---- src/lib_profiler/unix/profiler_services.mli | 2 +- .../lib_delegate/client_daemon.ml | 13 +++++--- .../lib_delegate/client_daemon.ml | 13 +++++--- src/proto_alpha/lib_delegate/client_daemon.ml | 13 +++++--- tezt/manual_tests/eio_benchmarks/test.ml | 17 ++++++---- tezt/tests/cloud/tezos.ml | 2 +- 15 files changed, 121 insertions(+), 80 deletions(-) diff --git a/src/bin_node/node_replay_command.ml b/src/bin_node/node_replay_command.ml index 7790987e81f5..22c54bfd9779 100644 --- a/src/bin_node/node_replay_command.ml +++ b/src/bin_node/node_replay_command.ml @@ -606,16 +606,20 @@ let replay ~internal_events ~singleprocess ~strict ~repeat ~stats_output Store.close_store store) let[@warning "-32"] may_start_profiler data_dir = - match Tezos_profiler_unix.Profiler_instance.selected_backend () with - | Some {instance_maker; _} -> ( - let profiler_maker = instance_maker ~directory:data_dir in - Shell_profiling.activate_all ~profiler_maker ; - match profiler_maker ~name:"context" with - | Some instance -> - Tezos_protocol_environment.Environment_profiler.Context_ops_profiler - .plug - instance - | None -> ()) + match Tezos_profiler_unix.Profiler_instance.selected_backends () with + | Some backends -> + List.iter + (fun Tezos_profiler_unix.Profiler_instance.{instance_maker; _} -> + let profiler_maker = instance_maker ~directory:data_dir in + Shell_profiling.activate_all ~profiler_maker ; + match profiler_maker ~name:"context" with + | Some instance -> + Tezos_protocol_environment.Environment_profiler + .Context_ops_profiler + .plug + instance + | None -> ()) + backends | None -> () let run ?verbosity ~singleprocess ~strict ~repeat ~stats_output diff --git a/src/bin_node/node_run_command.ml b/src/bin_node/node_run_command.ml index 8733bd418925..0708ad40a0e2 100644 --- a/src/bin_node/node_run_command.ml +++ b/src/bin_node/node_run_command.ml @@ -721,16 +721,20 @@ let init_rpc (config : Config_file.t) (node : Node.t) internal_events = return (local_rpc_server :: [rpc_server]) let[@warning "-32"] may_start_profiler data_dir = - match Tezos_profiler_unix.Profiler_instance.selected_backend () with - | Some {instance_maker; _} -> ( - let profiler_maker = instance_maker ~directory:data_dir in - Shell_profiling.activate_all ~profiler_maker ; - match profiler_maker ~name:"context" with - | Some instance -> - Tezos_protocol_environment.Environment_profiler.Context_ops_profiler - .plug - instance - | None -> ()) + match Tezos_profiler_unix.Profiler_instance.selected_backends () with + | Some backends -> + List.iter + (fun Tezos_profiler_unix.Profiler_instance.{instance_maker; _} -> + let profiler_maker = instance_maker ~directory:data_dir in + Shell_profiling.activate_all ~profiler_maker ; + match profiler_maker ~name:"context" with + | Some instance -> + Tezos_protocol_environment.Environment_profiler + .Context_ops_profiler + .plug + instance + | None -> ()) + backends | None -> () let run ?verbosity ?sandbox ?target ?(cli_warnings = []) diff --git a/src/lib_agnostic_baker/daemon.ml b/src/lib_agnostic_baker/daemon.ml index a0987f889b35..665873610373 100644 --- a/src/lib_agnostic_baker/daemon.ml +++ b/src/lib_agnostic_baker/daemon.ml @@ -412,10 +412,13 @@ module Make_daemon (Agent : AGENT) : to determine when to switch to a new protocol baker process. *) let[@warning "-32"] may_start_profiler baking_dir = - match Tezos_profiler_unix.Profiler_instance.selected_backend () with - | Some {instance_maker; _} -> - let profiler_maker = instance_maker ~directory:baking_dir in - Agnostic_baker_profiler.init profiler_maker + match Tezos_profiler_unix.Profiler_instance.selected_backends () with + | Some backends -> + List.iter + (fun Tezos_profiler_unix.Profiler_instance.{instance_maker; _} -> + let profiler_maker = instance_maker ~directory:baking_dir in + Agnostic_baker_profiler.init profiler_maker) + backends | None -> () let run ~keep_alive ~command cctxt = diff --git a/src/lib_dal_node/daemon.ml b/src/lib_dal_node/daemon.ml index fd1368d5413f..f4b9d7f74f00 100644 --- a/src/lib_dal_node/daemon.ml +++ b/src/lib_dal_node/daemon.ml @@ -32,10 +32,13 @@ module Profiler = struct end let[@warning "-32"] may_start_profiler data_dir = - match Tezos_profiler_unix.Profiler_instance.selected_backend () with - | Some {instance_maker; _} -> - let profiler_maker = instance_maker ~directory:data_dir in - Dal_profiler.init profiler_maker + match Tezos_profiler_unix.Profiler_instance.selected_backends () with + | Some backends -> + List.iter + (fun Tezos_profiler_unix.Profiler_instance.{instance_maker; _} -> + let profiler_maker = instance_maker ~directory:data_dir in + Dal_profiler.init profiler_maker) + backends | None -> () let init_cryptobox config proto_parameters profile = diff --git a/src/lib_profiler/backends/complex/prometheus_profiler.mli b/src/lib_profiler/backends/complex/prometheus_profiler.mli index 0c6b3f64196b..a8afcaf587a8 100644 --- a/src/lib_profiler/backends/complex/prometheus_profiler.mli +++ b/src/lib_profiler/backends/complex/prometheus_profiler.mli @@ -7,7 +7,7 @@ (** Driver registering its report in a promotheus collector whenever a toplevel section ends. - Enable it using [PROFILING_BACKEND=prometheus] environment variable. + Enable it using [PROFILING_BACKENDS=prometheus] environment variable. This backend relies on the presence of a "prometheus" attribute in the metadata field of profiler calls. If this attribute is set to diff --git a/src/lib_profiler/unix/profiler_directory.ml b/src/lib_profiler/unix/profiler_directory.ml index a576a4afbfb9..570de453c689 100644 --- a/src/lib_profiler/unix/profiler_directory.ml +++ b/src/lib_profiler/unix/profiler_directory.ml @@ -10,9 +10,12 @@ let build_rpc_directory () = let dir = Tezos_rpc.Directory.empty in let dir = Tezos_rpc.Directory.register0 dir Registered.S.registered (fun () () -> - let registered_backend = - match Profiler_instance.selected_backend () with - | Some backend_info -> [backend_info.Profiler_instance.view] + let registered_backends = + match Profiler_instance.selected_backends () with + | Some backends -> + List.map + (fun backend_info -> backend_info.Profiler_instance.view) + backends | None -> [] in let backends = @@ -21,6 +24,6 @@ let build_rpc_directory () = |> List.map (fun (env_var, infos) -> (env_var, infos.Profiler_instance.view)) in - Lwt.return_ok Registered.{registered_backend; backends}) + Lwt.return_ok Registered.{registered_backends; backends}) in dir diff --git a/src/lib_profiler/unix/profiler_instance.ml b/src/lib_profiler/unix/profiler_instance.ml index 7d4b10ba4afb..b4cf9aca74e3 100644 --- a/src/lib_profiler/unix/profiler_instance.ml +++ b/src/lib_profiler/unix/profiler_instance.ml @@ -102,11 +102,12 @@ let wrap_backend_verbosity instance_maker ~directory ~name = type wrapped_instance_maker = directory:string -> name:string -> Profiler.instance option -let selected_backend () = +let selected_backends () = let fail s = Fmt.failwith "@[%s.@,\ - You can set a backend with PROFILING_BACKEND=backend.@,\ + You can set backends with PROFILING_BACKENDS=.@,\ @[Available backends are:@,\ %a@." s @@ -117,7 +118,7 @@ let selected_backend () = []) in match - Sys.getenv_opt "PROFILING_BACKEND" |> Option.map String.lowercase_ascii + Sys.getenv_opt "PROFILING_BACKENDS" |> Option.map String.lowercase_ascii with | None -> ( match Sys.getenv_opt "PROFILING" with @@ -126,19 +127,28 @@ let selected_backend () = Format.sprintf "No backend selected but profilers were enabled in PROFILING." |> fail) - | Some b -> ( - match BackendMap.find b !registered_backends with - | Some {instance_maker; view} -> - Some {instance_maker = wrap_backend_verbosity instance_maker; view} - | None -> - Format.sprintf "No backend registered for value \"%s\"" b |> fail) + | Some backends -> + String.split_no_empty ';' backends + |> List.fold_left + (fun acc backend -> + match + BackendMap.find (String.trim backend) !registered_backends + with + | Some {instance_maker; view} -> + {instance_maker = wrap_backend_verbosity instance_maker; view} + :: acc + | None -> + Format.sprintf "No backend registered for value \"%s\"" backend + |> fail) + [] + |> Option.some let () = if (true [@profiler.overwrite false]) - && Sys.getenv_opt "PROFILING_BACKEND" <> None + && Sys.getenv_opt "PROFILING_BACKENDS" <> None then Fmt.failwith - "The profiling has been enabled with PROFILING_BACKEND='...' but the \ + "The profiling has been enabled with PROFILING_BACKENDS='...' but the \ program hasn't been compiled with TEZOS_PPX_PROFILER='...'" else () diff --git a/src/lib_profiler/unix/profiler_instance.mli b/src/lib_profiler/unix/profiler_instance.mli index 6fd92f7db3be..660cf6d5b610 100644 --- a/src/lib_profiler/unix/profiler_instance.mli +++ b/src/lib_profiler/unix/profiler_instance.mli @@ -32,12 +32,12 @@ val registered_backends : instance_maker backend_infos BackendMap.t ref let () = register_backend ["text"; "txt"] (profiler ~suffix:".txt") write_to_txt_file ]} - The identifiers are matched against the PROFILING_BACKEND environment + The identifiers are matched against the PROFILING_BACKENDS environment variable in order to be selected when profiling. This module registers two default profilers defined in {!Simple_profiler}. - One for [PROFILING_BACKEND=json] which outputs raw data to json files, and - one for [PROFILING_BACKEND=txt] and [PROFILING_BACKEND=text] that outputs + One for [PROFILING_BACKENDS=json] which outputs raw data to json files, and + one for [PROFILING_BACKENDS=txt] and [PROFILING_BACKENDS=text] that outputs plain text formatted reports. Note that these profilers add [_profiling.txt] or [_profiling.json] @@ -48,6 +48,6 @@ val register_backend : type wrapped_instance_maker = directory:string -> name:string -> Profiler.instance option -(** [selected_backend ()] returns the backend selected using the environment - variable [PROFILING_BACKEND]. *) -val selected_backend : unit -> wrapped_instance_maker backend_infos option +(** [selected_backends ()] returns the selected backends using the environment + variable [PROFILING_BACKENDS]. *) +val selected_backends : unit -> wrapped_instance_maker backend_infos list option diff --git a/src/lib_profiler/unix/profiler_services.ml b/src/lib_profiler/unix/profiler_services.ml index dd328ec7c090..092fb05ef468 100644 --- a/src/lib_profiler/unix/profiler_services.ml +++ b/src/lib_profiler/unix/profiler_services.ml @@ -9,7 +9,7 @@ open Tezos_rpc.Context module type REGISTERED = sig type t = { - registered_backend : Profiler.view list; + registered_backends : Profiler.view list; backends : (string * Profiler.view) list; } @@ -25,7 +25,7 @@ end module MakeRegistered () : REGISTERED = struct type t = { - registered_backend : Profiler.view list; + registered_backends : Profiler.view list; backends : (string * Profiler.view) list; } @@ -39,12 +39,14 @@ module MakeRegistered () : REGISTERED = struct let encoding = let open Data_encoding in - def "profiler registered backend" ~description:"Registered backend." + def "profiler registered backends" ~description:"Registered backends." @@ conv - (fun {registered_backend; backends} -> (registered_backend, backends)) - (fun (registered_backend, backends) -> {registered_backend; backends}) + (fun {registered_backends; backends} -> + (registered_backends, backends)) + (fun (registered_backends, backends) -> + {registered_backends; backends}) (obj2 - (req "registered_backend" (list Profiler_kind.kind_encoding)) + (req "registered_backends" (list Profiler_kind.kind_encoding)) (req "backends" backends_encoding)) module S = struct diff --git a/src/lib_profiler/unix/profiler_services.mli b/src/lib_profiler/unix/profiler_services.mli index 17bb20c2e7cc..666fde03a63d 100644 --- a/src/lib_profiler/unix/profiler_services.mli +++ b/src/lib_profiler/unix/profiler_services.mli @@ -9,7 +9,7 @@ open Tezos_rpc.Context module type REGISTERED = sig type t = { - registered_backend : Profiler.view list; + registered_backends : Profiler.view list; backends : (string * Profiler.view) list; } diff --git a/src/proto_022_PsRiotum/lib_delegate/client_daemon.ml b/src/proto_022_PsRiotum/lib_delegate/client_daemon.ml index 1ae3f90cb5bd..c30105c11f9d 100644 --- a/src/proto_022_PsRiotum/lib_delegate/client_daemon.ml +++ b/src/proto_022_PsRiotum/lib_delegate/client_daemon.ml @@ -57,11 +57,14 @@ let await_protocol_start (cctxt : #Protocol_client_context.full) ~chain = Node_rpc.await_protocol_activation cctxt ~chain () let[@warning "-32"] may_start_profiler baking_dir = - match Tezos_profiler_unix.Profiler_instance.selected_backend () with - | Some {instance_maker; _} -> - let profiler_maker = instance_maker ~directory:baking_dir in - Baking_profiler.activate_all ~profiler_maker ; - RPC_profiler.init profiler_maker + match Tezos_profiler_unix.Profiler_instance.selected_backends () with + | Some backends -> + List.iter + (fun Tezos_profiler_unix.Profiler_instance.{instance_maker; _} -> + let profiler_maker = instance_maker ~directory:baking_dir in + Baking_profiler.activate_all ~profiler_maker ; + RPC_profiler.init profiler_maker) + backends | None -> () module Baker = struct diff --git a/src/proto_023_PtSEouLo/lib_delegate/client_daemon.ml b/src/proto_023_PtSEouLo/lib_delegate/client_daemon.ml index b09939346765..1624e9be149c 100644 --- a/src/proto_023_PtSEouLo/lib_delegate/client_daemon.ml +++ b/src/proto_023_PtSEouLo/lib_delegate/client_daemon.ml @@ -57,11 +57,14 @@ let await_protocol_start (cctxt : #Protocol_client_context.full) ~chain = Node_rpc.await_protocol_activation cctxt ~chain () let[@warning "-32"] may_start_profiler baking_dir = - match Tezos_profiler_unix.Profiler_instance.selected_backend () with - | Some {instance_maker; _} -> - let profiler_maker = instance_maker ~directory:baking_dir in - Baking_profiler.activate_all ~profiler_maker ; - RPC_profiler.init profiler_maker + match Tezos_profiler_unix.Profiler_instance.selected_backends () with + | Some backends -> + List.iter + (fun Tezos_profiler_unix.Profiler_instance.{instance_maker; _} -> + let profiler_maker = instance_maker ~directory:baking_dir in + Baking_profiler.activate_all ~profiler_maker ; + RPC_profiler.init profiler_maker) + backends | None -> () module Baker = struct diff --git a/src/proto_alpha/lib_delegate/client_daemon.ml b/src/proto_alpha/lib_delegate/client_daemon.ml index b09939346765..1624e9be149c 100644 --- a/src/proto_alpha/lib_delegate/client_daemon.ml +++ b/src/proto_alpha/lib_delegate/client_daemon.ml @@ -57,11 +57,14 @@ let await_protocol_start (cctxt : #Protocol_client_context.full) ~chain = Node_rpc.await_protocol_activation cctxt ~chain () let[@warning "-32"] may_start_profiler baking_dir = - match Tezos_profiler_unix.Profiler_instance.selected_backend () with - | Some {instance_maker; _} -> - let profiler_maker = instance_maker ~directory:baking_dir in - Baking_profiler.activate_all ~profiler_maker ; - RPC_profiler.init profiler_maker + match Tezos_profiler_unix.Profiler_instance.selected_backends () with + | Some backends -> + List.iter + (fun Tezos_profiler_unix.Profiler_instance.{instance_maker; _} -> + let profiler_maker = instance_maker ~directory:baking_dir in + Baking_profiler.activate_all ~profiler_maker ; + RPC_profiler.init profiler_maker) + backends | None -> () module Baker = struct diff --git a/tezt/manual_tests/eio_benchmarks/test.ml b/tezt/manual_tests/eio_benchmarks/test.ml index 854ec1227659..29de26cf8f3f 100644 --- a/tezt/manual_tests/eio_benchmarks/test.ml +++ b/tezt/manual_tests/eio_benchmarks/test.ml @@ -6,7 +6,7 @@ (*****************************************************************************) (* You can run the benchmark using: - TEZOS_PPX_PROFILER=t dune build tezt/manual_tests/eio_benchmarks && PROFILING="main->debug" PROFILING_BACKEND=txt _build/default/tezt/manual_tests/eio_benchmarks/test.exe 32 + TEZOS_PPX_PROFILER=t dune build tezt/manual_tests/eio_benchmarks && PROFILING="main->debug" PROFILING_BACKENDS=txt _build/default/tezt/manual_tests/eio_benchmarks/test.exe 32 *) open Tezos_base @@ -21,12 +21,15 @@ let () = exit 1) let () = - match Tezos_profiler_unix.Profiler_instance.selected_backend () with - | Some {instance_maker; _} -> ( - let profiler_maker = instance_maker ~directory:"/tmp" ~name:"main" in - match profiler_maker with - | Some instance -> Tezos_profiler.Profiler.(plug main) instance - | None -> ()) + match Tezos_profiler_unix.Profiler_instance.selected_backends () with + | Some backends -> + List.iter + (fun Tezos_profiler_unix.Profiler_instance.{instance_maker; _} -> + let profiler_maker = instance_maker ~directory:"/tmp" ~name:"main" in + match profiler_maker with + | Some instance -> Tezos_profiler.Profiler.(plug main) instance + | None -> ()) + backends | None -> () module Profiler = diff --git a/tezt/tests/cloud/tezos.ml b/tezt/tests/cloud/tezos.ml index e7c0e3fe4eb6..e95cb940ead9 100644 --- a/tezt/tests/cloud/tezos.ml +++ b/tezt/tests/cloud/tezos.ml @@ -39,7 +39,7 @@ module Env = struct let ppx_profiler_env enable env = env |> may_add enable "PROFILING" "Debug" - |> may_add enable "PROFILING_BACKEND" "txt" + |> may_add enable "PROFILING_BACKENDS" "txt" let initialize_env ~memtrace ~memtrace_output_filename ~disable_shard_validation ~otel_endpoint ~service_name ~ignore_pkhs -- GitLab From f38361471c4162314e994fb8970d4ea3c6d3cb7f Mon Sep 17 00:00:00 2001 From: mattiasdrp Date: Tue, 17 Jun 2025 16:55:13 +0200 Subject: [PATCH 2/2] Profiler doc: Improve and fix the profiler documentation --- docs/developer/profiler_module.rst | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/developer/profiler_module.rst b/docs/developer/profiler_module.rst index 2a907c42e98d..cfaaa00698be 100644 --- a/docs/developer/profiler_module.rst +++ b/docs/developer/profiler_module.rst @@ -102,12 +102,12 @@ encouraged to use their preferred option. To choose the verbosity of the profiler at runtime, the ``PROFILING`` environment variable is used. It follows the same pattern as the ``TEZOS_LOG`` -environment variable (see :doc:`../user/logging`). +environment variable (see :doc:`../user/logging`) except that it doesn't execute +profilers which verbosity is not set. Starting a node with ``PROFILING='shell_profiling->Notice;mempool_profiling->Debug'`` will set the -maximum verbosity for these two profilers and execute the rest of the profilers -with no maximal verbosity. +maximum verbosity for these two profilers and not execute the rest of the profilers. We can now easily create an instance for a ``Driver``: @@ -160,7 +160,17 @@ code. Since ``Read_profiler`` is already plugged to ``read_instance``, calling ``Read_profiler`` functions will work as expected. -We can now start monitoring our code. We can start with a simple change: +We can now start monitoring our code. + +First, we need to plug our profilers to backends to gather results. Plugging a +profiler to one or more backends is done with ``PROFILER_BACKENDS=``. For example, you could run a node with + +.. code-block:: + + PROFILING="*->Debug" PROFILER_BACKENDS="txt; json" octez-node run + +We can start with a simple change: .. code-block:: OCaml -- GitLab