From adb4250726fcdbd1364ef07250e2252f0c64bec6 Mon Sep 17 00:00:00 2001 From: Pierrick Couderc Date: Mon, 23 Jun 2025 10:03:57 +0200 Subject: [PATCH 1/3] Tezt/Cloud: etherlink DAL node uses configuration and profiler --- tezt/tests/cloud/dal.ml | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tezt/tests/cloud/dal.ml b/tezt/tests/cloud/dal.ml index b9cade79a029..89231916a531 100644 --- a/tezt/tests/cloud/dal.ml +++ b/tezt/tests/cloud/dal.ml @@ -2873,8 +2873,16 @@ let init_observer cloud configuration ~bootstrap teztale ~topic i agent = in Lwt.return {node; dal_node; topic} -let init_etherlink_dal_node ~bootstrap ~next_agent ~dal_slots ~network ~snapshot - ~otel ~memtrace ~rpc_external ~cloud = +let init_etherlink_dal_node + { + external_rpc; + network; + snapshot; + ppx_profiling; + ppx_profiler_backends; + memtrace; + _; + } ~bootstrap ~dal_slots ~next_agent ~otel ~cloud = match dal_slots with | [] -> toplog "Etherlink will run without DAL support" ; @@ -2891,7 +2899,7 @@ let init_etherlink_dal_node ~bootstrap ~next_agent ~dal_slots ~network ~snapshot ~name ~arguments: [Peer bootstrap.node_p2p_endpoint; History_mode (Rolling (Some 79))] - ~rpc_external + ~rpc_external:external_rpc network ~snapshot cloud @@ -2905,7 +2913,7 @@ let init_etherlink_dal_node ~bootstrap ~next_agent ~dal_slots ~network ~snapshot ~peers:(Option.to_list bootstrap.dal_node_p2p_endpoint) dal_node in - let* () = Dal_node.Agent.run ?otel dal_node in + let* () = Dal_node.Agent.run ?otel ~ppx_profiling dal_node in some dal_node | _ :: _ :: _ -> (* On several slot indices, we launch one observer DAL node per @@ -2927,7 +2935,7 @@ let init_etherlink_dal_node ~bootstrap ~next_agent ~dal_slots ~network ~snapshot ~name ~arguments: [Peer bootstrap.node_p2p_endpoint; History_mode (Rolling (Some 79))] - ~rpc_external + ~rpc_external:external_rpc network ~snapshot cloud @@ -2956,7 +2964,7 @@ let init_etherlink_dal_node ~bootstrap ~next_agent ~dal_slots ~network ~snapshot Peer bootstrap.node_p2p_endpoint; History_mode (Rolling (Some 79)); ] - ~rpc_external + ~rpc_external:external_rpc network ~snapshot cloud @@ -3047,14 +3055,11 @@ let init_etherlink_operator_setup cloud configuration etherlink_configuration let otel = Cloud.open_telemetry_endpoint cloud in let* dal_node = init_etherlink_dal_node + configuration ~bootstrap ~next_agent ~dal_slots:etherlink_configuration.etherlink_dal_slots - ~network:configuration.network - ~snapshot:configuration.snapshot - ~rpc_external:configuration.external_rpc ~otel - ~memtrace:configuration.memtrace ~cloud in let operators = -- GitLab From 5ff958fae85a2967a6f8e6e81e4e1c046a902b35 Mon Sep 17 00:00:00 2001 From: Pierrick Couderc Date: Thu, 19 Jun 2025 15:25:37 +0200 Subject: [PATCH 2/3] Tezt/Cloud: add Cli parameter to select profiler backends to use --- tezt/tests/cloud/dal.ml | 5 ++++- tezt/tests/cloud/scenarios_cli.ml | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tezt/tests/cloud/dal.ml b/tezt/tests/cloud/dal.ml index 89231916a531..1a06ddeba1e8 100644 --- a/tezt/tests/cloud/dal.ml +++ b/tezt/tests/cloud/dal.ml @@ -520,6 +520,7 @@ type configuration = { disable_shard_validation : bool; ignore_pkhs : string list; ppx_profiling : bool; + ppx_profiling_backends : string list; } type bootstrap = { @@ -2879,7 +2880,7 @@ let init_etherlink_dal_node network; snapshot; ppx_profiling; - ppx_profiler_backends; + ppx_profiling_backends; memtrace; _; } ~bootstrap ~dal_slots ~next_agent ~otel ~cloud = @@ -3912,6 +3913,7 @@ let register (module Cli : Scenarios_cli.Dal) = let external_rpc = Cli.node_external_rpc_server in let disable_shard_validation = Cli.disable_shard_validation in let ppx_profiling = Cli.ppx_profiling in + let ppx_profiling_backends = Cli.ppx_profiling_backends in let t = { with_dal; @@ -3940,6 +3942,7 @@ let register (module Cli : Scenarios_cli.Dal) = disable_shard_validation; ignore_pkhs; ppx_profiling; + ppx_profiling_backends; } in (t, etherlink) diff --git a/tezt/tests/cloud/scenarios_cli.ml b/tezt/tests/cloud/scenarios_cli.ml index b6cfae72d501..965d72c68bcc 100644 --- a/tezt/tests/cloud/scenarios_cli.ml +++ b/tezt/tests/cloud/scenarios_cli.ml @@ -91,6 +91,8 @@ module type Dal = sig val ignore_pkhs : string list val ppx_profiling : bool + + val ppx_profiling_backends : string list end module Dal () : Dal = struct @@ -480,6 +482,16 @@ module Dal () : Dal = struct "Enable PPX profiling on all components. The level of verbosity is by \ default `Debug` and the format of the output is `txt`. " false + + let ppx_profiling_backends = + Clap.list_string + ~section + ~long:"ppx-profiling-backends" + ~description: + "Select the backends used by the profiler, bypassing the defaults \ + selection: always `txt` and `json`, and also `prometheus` if \ + `--prometheus` and `opentelemetry` if `--opentelemetry`." + () end module type Layer1 = sig -- GitLab From 292265bea74935e0b3a1e10dc9b3a0f86d541566 Mon Sep 17 00:00:00 2001 From: Pierrick Couderc Date: Thu, 19 Jun 2025 15:42:23 +0200 Subject: [PATCH 3/3] Tezt/Cloud: allow selection of profiler backends --- tezt/tests/cloud/dal.ml | 28 +++++++++++++++++-- tezt/tests/cloud/tezos.ml | 58 +++++++++++++++++++++++++++++---------- 2 files changed, 69 insertions(+), 17 deletions(-) diff --git a/tezt/tests/cloud/dal.ml b/tezt/tests/cloud/dal.ml index 1a06ddeba1e8..fbd198485c08 100644 --- a/tezt/tests/cloud/dal.ml +++ b/tezt/tests/cloud/dal.ml @@ -2234,6 +2234,7 @@ let init_public_network cloud (configuration : configuration) ~event_level:`Notice ~disable_shard_validation ~ppx_profiling:configuration.ppx_profiling + ~ppx_profiling_backends:configuration.ppx_profiling_backends dal_node in Lwt.return_some dal_node @@ -2542,6 +2543,7 @@ let init_sandbox_and_activate_protocol cloud (configuration : configuration) ~event_level:`Notice ~disable_shard_validation:configuration.disable_shard_validation ~ppx_profiling:configuration.ppx_profiling + ~ppx_profiling_backends:configuration.ppx_profiling_backends dal_bootstrap_node in let* () = @@ -2635,6 +2637,7 @@ let init_baker ?stake cloud (configuration : configuration) ~bootstrap teztale ~event_level:`Notice ~disable_shard_validation:configuration.disable_shard_validation ~ppx_profiling:configuration.ppx_profiling + ~ppx_profiling_backends:configuration.ppx_profiling_backends dal_node in Lwt.return_some dal_node @@ -2786,6 +2789,7 @@ let init_producer cloud configuration ~bootstrap teztale account i slot_index ~disable_shard_validation:configuration.disable_shard_validation ?ignore_pkhs ~ppx_profiling:configuration.ppx_profiling + ~ppx_profiling_backends:configuration.ppx_profiling_backends dal_node in let () = toplog "Init producer %s: DAL node is ready" name in @@ -2859,6 +2863,7 @@ let init_observer cloud configuration ~bootstrap teztale ~topic i agent = ~event_level:`Notice ~disable_shard_validation:configuration.disable_shard_validation ~ppx_profiling:configuration.ppx_profiling + ~ppx_profiling_backends:configuration.ppx_profiling_backends dal_node in let* () = @@ -2914,7 +2919,9 @@ let init_etherlink_dal_node ~peers:(Option.to_list bootstrap.dal_node_p2p_endpoint) dal_node in - let* () = Dal_node.Agent.run ?otel ~ppx_profiling dal_node in + let* () = + Dal_node.Agent.run ?otel ~ppx_profiling ~ppx_profiling_backends dal_node + in some dal_node | _ :: _ :: _ -> (* On several slot indices, we launch one observer DAL node per @@ -2949,7 +2956,14 @@ let init_etherlink_dal_node ~peers:(Option.to_list bootstrap.dal_node_p2p_endpoint) default_dal_node in - let* () = Dal_node.Agent.run ?otel ~memtrace default_dal_node in + let* () = + Dal_node.Agent.run + ?otel + ~memtrace + ~ppx_profiling + ~ppx_profiling_backends + default_dal_node + in let default_endpoint = Dal_node.rpc_endpoint default_dal_node in let* dal_slots_and_nodes = @@ -2979,7 +2993,14 @@ let init_etherlink_dal_node ~peers:(Option.to_list bootstrap.dal_node_p2p_endpoint) dal_node in - let* () = Dal_node.Agent.run ?otel ~memtrace dal_node in + let* () = + Dal_node.Agent.run + ?otel + ~memtrace + ~ppx_profiling + ~ppx_profiling_backends + dal_node + in return (slot_index, Dal_node.rpc_endpoint dal_node)) in let* reverse_proxy_dal_node = @@ -3641,6 +3662,7 @@ let on_new_level t level ~metadata = ?otel:t.otel ~memtrace:t.configuration.memtrace ~ppx_profiling:t.configuration.ppx_profiling + ~ppx_profiling_backends:t.configuration.ppx_profiling_backends dal_node) in Lwt.return {t with disconnection_state = Some disconnection_state} diff --git a/tezt/tests/cloud/tezos.ml b/tezt/tests/cloud/tezos.ml index 4302a7d01bb9..bde980731df5 100644 --- a/tezt/tests/cloud/tezos.ml +++ b/tezt/tests/cloud/tezos.ml @@ -38,38 +38,67 @@ module Env = struct let default_profiling_verbosity = "Debug" - let default_profiling_backends = ["txt"; "json"] + let txt_profiler_backend = "txt" - let ppx_profiler_env ?prometheus ?otel enable env = + let json_profiler_backed = "json" + + let prometheus_profiler_backend = "prometheus" + + let opentelemetry_profiler_backend = "opentelemetry" + + let default_profiling_backends = [txt_profiler_backend; json_profiler_backed] + + let ppx_profiling_backends ?prometheus ?otel backends = let cons_if b v l = if b then v :: l else l in + let is_selected backend = backends = [] || List.mem backend backends in + let profiling_backends = + [] + |> cons_if + (Option.is_some otel && is_selected opentelemetry_profiler_backend) + opentelemetry_profiler_backend + |> cons_if + (Option.is_some prometheus && is_selected prometheus_profiler_backend) + prometheus_profiler_backend + |> cons_if (is_selected txt_profiler_backend) txt_profiler_backend + |> cons_if (is_selected json_profiler_backed) json_profiler_backed + in + "\"" ^ String.concat ";" profiling_backends ^ "\"" + + let ppx_profiler_env ?prometheus ?otel enable selected_backends env = let profiling_backends = - default_profiling_backends - |> cons_if (Option.is_some otel) "opentelemetry" - |> cons_if (Option.is_some prometheus) "prometheus" + ppx_profiling_backends ?prometheus ?otel selected_backends in env |> may_add enable "PROFILING" default_profiling_verbosity - |> may_add - enable - "PROFILING_BACKENDS" - (String.concat ";" profiling_backends) + |> may_add enable "PROFILING_BACKENDS" profiling_backends let initialize_env ~memtrace ~memtrace_output_filename ~disable_shard_validation ~prometheus ~otel_endpoint ~service_name - ~ignore_pkhs ~ppx_profiling = + ~ignore_pkhs ~ppx_profiling ~ppx_profiling_backends = empty |> memtrace_env memtrace memtrace_output_filename |> otel_env otel_endpoint service_name |> disable_shard_validation_env disable_shard_validation |> ignore_topics_env ignore_pkhs - |> ppx_profiler_env ?prometheus ?otel:otel_endpoint ppx_profiling + |> ppx_profiler_env + ?prometheus + ?otel:otel_endpoint + ppx_profiling + ppx_profiling_backends end let may_add_profiling_to_env ~ppx_profiling = function | None -> - if ppx_profiling then Some (Env.ppx_profiler_env ppx_profiling Env.empty) + if ppx_profiling then + Some + (Env.ppx_profiler_env + ppx_profiling + Env.default_profiling_backends + Env.empty) else None - | Some env -> Some (Env.ppx_profiler_env ppx_profiling env) + | Some env -> + Some + (Env.ppx_profiler_env ppx_profiling Env.default_profiling_backends env) let create_dir ?runner dir = let* () = Process.spawn ?runner "rm" ["-rf"; dir] |> Process.check in @@ -298,7 +327,7 @@ module Dal_node = struct let run ?prometheus ?otel ?(memtrace = false) ?event_level ?(disable_shard_validation = false) ?ignore_pkhs - ?(ppx_profiling = false) dal_node = + ?(ppx_profiling = false) ?(ppx_profiling_backends = []) dal_node = let service_name = name dal_node in let memtrace_output_filename = Format.asprintf "%s/%s-trace.ctf" Path.tmp_dir service_name @@ -313,6 +342,7 @@ module Dal_node = struct ~service_name ~ignore_pkhs ~ppx_profiling + ~ppx_profiling_backends in let* () = run ~env ?event_level dal_node in (* Update the state in the service manager *) -- GitLab