From ec562094341fa4e21970c2d10850420782d4f2e8 Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Tue, 18 Feb 2025 15:49:50 +0100 Subject: [PATCH 1/6] DAL/Node: add an option --verbose to control details of some frequent events --- src/bin_dal_node/cli.ml | 13 +++++++++++-- src/bin_dal_node/cli.mli | 4 +++- src/bin_dal_node/configuration_file.ml | 21 +++++++++++++++++---- src/bin_dal_node/configuration_file.mli | 3 +++ src/bin_dal_node/main.ml | 2 ++ 5 files changed, 36 insertions(+), 7 deletions(-) diff --git a/src/bin_dal_node/cli.ml b/src/bin_dal_node/cli.ml index 34e0a917f9a2..d6cc8d17391c 100644 --- a/src/bin_dal_node/cli.ml +++ b/src/bin_dal_node/cli.ml @@ -343,6 +343,13 @@ module Term = struct & opt (some bool) None & info ~docs ~doc ~docv:"true | false" ["fetch-trusted-setup"]) + let verbose = + let open Cmdliner in + let doc = + "Controls the verbosity of some emitted events. Default value is false." + in + Arg.(value & flag & info ~docs ~doc ["verbose"]) + (* Experimental features. *) let sqlite3_backend = @@ -364,7 +371,7 @@ module Term = struct $ public_addr $ endpoint $ metrics_addr $ attester_profile $ producer_profile $ observer_profile $ bootstrap_profile $ peers $ history_mode $ service_name $ service_namespace $ sqlite3_backend - $ fetch_trusted_setup)) + $ fetch_trusted_setup $ verbose)) end type t = Run | Config_init | Config_update | Debug_print_store_schemas @@ -522,13 +529,14 @@ type options = { service_namespace : string option; experimental_features : experimental_features; fetch_trusted_setup : bool option; + verbose : bool; } let make ~run = let run subcommand data_dir rpc_addr expected_pow listen_addr public_addr endpoint metrics_addr attesters producers observers bootstrap_flag peers history_mode service_name service_namespace sqlite3_backend - fetch_trusted_setup = + fetch_trusted_setup verbose = let run profile = run subcommand @@ -547,6 +555,7 @@ let make ~run = service_namespace; experimental_features = {sqlite3_backend}; fetch_trusted_setup; + verbose; } in let profile = Operator_profile.make ~attesters ~producers ?observers () in diff --git a/src/bin_dal_node/cli.mli b/src/bin_dal_node/cli.mli index 1d9df010f2a0..6ca134096f5e 100644 --- a/src/bin_dal_node/cli.mli +++ b/src/bin_dal_node/cli.mli @@ -57,7 +57,9 @@ type options = { experimental_features : experimental_features; (** Experimental features. *) fetch_trusted_setup : bool option; (** Should the trusted setup be installed if required and invalid? - In case of [None] at init it is considered as yes.*) + In case of [None] at init it is considered as yes.*) + verbose : bool; + (** Emit events related to connections. Default value is false. *) } (** Subcommands that can be used by the DAL node. In the future this type diff --git a/src/bin_dal_node/configuration_file.ml b/src/bin_dal_node/configuration_file.ml index deb937903363..732a6bd3d342 100644 --- a/src/bin_dal_node/configuration_file.ml +++ b/src/bin_dal_node/configuration_file.ml @@ -81,6 +81,7 @@ type t = { service_namespace : string option; experimental_features : experimental_features; fetch_trusted_setup : bool; + verbose : bool; } let default_data_dir = Filename.concat (Sys.getenv "HOME") ".tezos-dal-node" @@ -143,6 +144,7 @@ let default = service_namespace = None; experimental_features = default_experimental_features; fetch_trusted_setup = default_fetch_trusted_setup; + verbose = false; } let neighbor_encoding : neighbor Data_encoding.t = @@ -198,6 +200,7 @@ let encoding : t Data_encoding.t = service_namespace; experimental_features; fetch_trusted_setup; + verbose; } -> ( ( data_dir, rpc_addr, @@ -215,7 +218,8 @@ let encoding : t Data_encoding.t = service_name, service_namespace, experimental_features, - fetch_trusted_setup ) )) + fetch_trusted_setup, + verbose ) )) (fun ( ( data_dir, rpc_addr, listen_addr, @@ -232,7 +236,8 @@ let encoding : t Data_encoding.t = service_name, service_namespace, experimental_features, - fetch_trusted_setup ) ) -> + fetch_trusted_setup, + verbose ) ) -> { data_dir; rpc_addr; @@ -251,6 +256,7 @@ let encoding : t Data_encoding.t = service_namespace; experimental_features; fetch_trusted_setup; + verbose; }) (merge_objs (obj10 @@ -304,7 +310,7 @@ let encoding : t Data_encoding.t = ~description:"The point for the DAL node metrics server" (Encoding.option P2p_point.Id.encoding) None)) - (obj7 + (obj8 (dft "history_mode" ~description:"The history mode for the DAL node" @@ -335,7 +341,13 @@ let encoding : t Data_encoding.t = "fetch_trusted_setup" ~description:"Install trusted setup" bool - true))) + true) + (dft + "verbose" + ~description: + "Whether to emit details about frequent logging events" + bool + default.verbose))) module V0 = struct type t = { @@ -490,6 +502,7 @@ let from_v0 v0 = service_namespace = None; experimental_features = default_experimental_features; fetch_trusted_setup = true; + verbose = false; } type error += DAL_node_unable_to_write_configuration_file of string diff --git a/src/bin_dal_node/configuration_file.mli b/src/bin_dal_node/configuration_file.mli index 927998aa311f..0de121a9d75b 100644 --- a/src/bin_dal_node/configuration_file.mli +++ b/src/bin_dal_node/configuration_file.mli @@ -68,6 +68,9 @@ type t = { experimental_features : experimental_features; (** Experimental features. *) fetch_trusted_setup : bool; (** Should the trusted setup be downloaded if not found or has invalid hash. *) + verbose : bool; + (** Whether to emit detailed events for frequently received control + messages from remote peers. *) } (** [default] is the default configuration. *) diff --git a/src/bin_dal_node/main.ml b/src/bin_dal_node/main.ml index 3e71eb6307ea..874703f52d28 100644 --- a/src/bin_dal_node/main.ml +++ b/src/bin_dal_node/main.ml @@ -44,6 +44,7 @@ let merge service_namespace; experimental_features; fetch_trusted_setup; + verbose; } configuration = let profile = match profile with @@ -81,6 +82,7 @@ let merge merge_experimental_features experimental_features configuration.experimental_features; + verbose = configuration.verbose || verbose; } let wrap_with_error main_promise = -- GitLab From 1c5902d8673b501e1bec9f138586da6290405758 Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Tue, 18 Feb 2025 16:04:21 +0100 Subject: [PATCH 2/6] DAL/Node: Gs_logging.event takes an argument verbose (not used yet) This prepares for the ability to reduce the verbosity of some GS messages received via P2P --- src/bin_dal_node/daemon.ml | 2 +- src/lib_dal_node/gossipsub/gossipsub.mli | 2 +- src/lib_dal_node/gossipsub/gs_logging.ml | 2 +- src/lib_dal_node/gossipsub/gs_logging.mli | 6 ++++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/bin_dal_node/daemon.ml b/src/bin_dal_node/daemon.ml index 974828fba201..3d1764bc7e53 100644 --- a/src/bin_dal_node/daemon.ml +++ b/src/bin_dal_node/daemon.ml @@ -1321,7 +1321,7 @@ let run ~data_dir ~configuration_override = Gossipsub.Worker.( make ~bootstrap_points:get_bootstrap_points - ~events_logging:Logging.event + ~events_logging:(Logging.event ~verbose:config.verbose) ~self rng limits diff --git a/src/lib_dal_node/gossipsub/gossipsub.mli b/src/lib_dal_node/gossipsub/gossipsub.mli index aff50c6c985e..750ae905ee5d 100644 --- a/src/lib_dal_node/gossipsub/gossipsub.mli +++ b/src/lib_dal_node/gossipsub/gossipsub.mli @@ -56,7 +56,7 @@ module Worker : sig and type Point.t = Types.Point.t module Logging : sig - val event : event -> unit Monad.t + val event : verbose:bool -> event -> unit Monad.t end (** A hook to set or update messages and messages IDs validation diff --git a/src/lib_dal_node/gossipsub/gs_logging.ml b/src/lib_dal_node/gossipsub/gs_logging.ml index b93d584eb8c9..43cf1619401d 100644 --- a/src/lib_dal_node/gossipsub/gs_logging.ml +++ b/src/lib_dal_node/gossipsub/gs_logging.ml @@ -195,7 +195,7 @@ module Events = struct ("message_ids", list Types.Message_id.encoding) end -let event = +let event ~verbose:_ = let open Events in function | Check_unknown_messages -> emit check_unknown_messages () diff --git a/src/lib_dal_node/gossipsub/gs_logging.mli b/src/lib_dal_node/gossipsub/gs_logging.mli index 1b8e61a76b0f..0bd320b30c0e 100644 --- a/src/lib_dal_node/gossipsub/gs_logging.mli +++ b/src/lib_dal_node/gossipsub/gs_logging.mli @@ -24,5 +24,7 @@ (* *) (*****************************************************************************) -(** [event e] logs the event [e] of the Gossipsub worker *) -val event : Gs_interface.Worker_instance.event -> unit Lwt.t +(** [event ~verbose e] logs the event [e] of the Gossipsub worker. When + [verbose] is unset, only a few messages of GS control messages are + logged. *) +val event : verbose:bool -> Gs_interface.Worker_instance.event -> unit Lwt.t -- GitLab From 32585e7aa7e3f52661c2a7d9d47a4162f2382786 Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Tue, 18 Feb 2025 16:07:56 +0100 Subject: [PATCH 3/6] DAL/Node: no Graft/Prune events when verbose is unset --- src/lib_dal_node/gossipsub/gs_logging.ml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/lib_dal_node/gossipsub/gs_logging.ml b/src/lib_dal_node/gossipsub/gs_logging.ml index 43cf1619401d..5674cfef0005 100644 --- a/src/lib_dal_node/gossipsub/gs_logging.ml +++ b/src/lib_dal_node/gossipsub/gs_logging.ml @@ -195,7 +195,7 @@ module Events = struct ("message_ids", list Types.Message_id.encoding) end -let event ~verbose:_ = +let event ~verbose = let open Events in function | Check_unknown_messages -> emit check_unknown_messages () @@ -217,15 +217,19 @@ let event ~verbose:_ = emit message_with_header (from_peer.peer_id, topic, message_id) | Subscribe {topic} -> emit subscribe (from_peer.peer_id, topic) | Unsubscribe {topic} -> emit unsubscribe (from_peer.peer_id, topic) - | Graft {topic} -> emit graft (from_peer.peer_id, topic) + | Graft {topic} -> + if not verbose then Lwt.return_unit + else emit graft (from_peer.peer_id, topic) | Prune {topic; px; backoff} -> - emit - prune - ( from_peer.peer_id, - topic, - backoff, - List.of_seq px - |> List.map (fun Types.Peer.{peer_id; _} -> peer_id) ) + if not verbose then Lwt.return_unit + else + emit + prune + ( from_peer.peer_id, + topic, + backoff, + List.of_seq px + |> List.map (fun Types.Peer.{peer_id; _} -> peer_id) ) | IHave {topic; message_ids} -> emit ihave (from_peer.peer_id, topic, message_ids) | IWant {message_ids} -> emit iwant (from_peer.peer_id, message_ids))) -- GitLab From 29c14e9e195065e5f7fa480ee6c6dbd7afcd6358 Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Tue, 18 Feb 2025 17:22:52 +0100 Subject: [PATCH 4/6] DAL/Node: prepare ability to reduce # of app messages notification events --- src/bin_dal_node/daemon.ml | 8 +++++--- src/lib_dal_node/gossipsub/gossipsub.mli | 1 + src/lib_dal_node/gossipsub/gs_transport_connection.ml | 6 +++--- src/lib_dal_node/gossipsub/gs_transport_connection.mli | 7 +++++-- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/bin_dal_node/daemon.ml b/src/bin_dal_node/daemon.ml index 3d1764bc7e53..f98178d717c0 100644 --- a/src/bin_dal_node/daemon.ml +++ b/src/bin_dal_node/daemon.ml @@ -807,7 +807,7 @@ let update_timing_shard_received node_ctxt shards_timing_table slot_id timing let connect_gossipsub_with_p2p proto_parameters gs_worker transport_layer - node_store node_ctxt amplificator = + node_store node_ctxt amplificator ~verbose = let open Gossipsub in let timing_table_size = 2 * proto_parameters.Types.attestation_lag @@ -865,7 +865,8 @@ let connect_gossipsub_with_p2p proto_parameters gs_worker transport_layer Transport_layer_hooks.activate gs_worker transport_layer - ~app_messages_callback:(shards_handler node_store)) + ~app_messages_callback:(shards_handler node_store) + ~verbose) (fun exn -> "[dal_node] error in Daemon.connect_gossipsub_with_p2p: " ^ Printexc.to_string exn @@ -1509,7 +1510,8 @@ let run ~data_dir ~configuration_override = transport_layer shards_store ctxt - amplificator ; + amplificator + ~verbose:config.verbose ; let*! () = Gossipsub.Transport_layer.activate ~additional_points:points transport_layer in diff --git a/src/lib_dal_node/gossipsub/gossipsub.mli b/src/lib_dal_node/gossipsub/gossipsub.mli index 750ae905ee5d..e57488c61650 100644 --- a/src/lib_dal_node/gossipsub/gossipsub.mli +++ b/src/lib_dal_node/gossipsub/gossipsub.mli @@ -196,6 +196,7 @@ module Transport_layer_hooks : sig Transport_layer.t -> app_messages_callback: (Types.Message.t -> Types.Message_id.t -> unit tzresult Lwt.t) -> + verbose:bool -> unit Lwt.t end diff --git a/src/lib_dal_node/gossipsub/gs_transport_connection.ml b/src/lib_dal_node/gossipsub/gs_transport_connection.ml index 0846dfcf0af0..edbbf9a0d3ec 100644 --- a/src/lib_dal_node/gossipsub/gs_transport_connection.ml +++ b/src/lib_dal_node/gossipsub/gs_transport_connection.ml @@ -287,7 +287,7 @@ let transport_layer_inputs_handler gs_worker p2p_layer = (** This loop pops messages from application output stream and calls the given [app_messages_callback] on them. *) -let app_messages_handler gs_worker ~app_messages_callback = +let app_messages_handler gs_worker ~app_messages_callback ~verbose:_ = let open Lwt_syntax in let rec loop app_output_stream = let* Worker.{message; message_id; topic = _} = @@ -303,7 +303,7 @@ let app_messages_handler gs_worker ~app_messages_callback = in Worker.app_output_stream gs_worker |> loop -let activate gs_worker p2p_layer ~app_messages_callback = +let activate gs_worker p2p_layer ~app_messages_callback ~verbose = (* Register a handler to notify new P2P connections to GS. *) let () = new_connections_handler gs_worker p2p_layer @@ -315,5 +315,5 @@ let activate gs_worker p2p_layer ~app_messages_callback = [ gs_worker_p2p_output_handler gs_worker p2p_layer; transport_layer_inputs_handler gs_worker p2p_layer; - app_messages_handler gs_worker ~app_messages_callback; + app_messages_handler gs_worker ~app_messages_callback ~verbose; ] diff --git a/src/lib_dal_node/gossipsub/gs_transport_connection.mli b/src/lib_dal_node/gossipsub/gs_transport_connection.mli index e9f7c2bc8dea..e6d0eaf1eecb 100644 --- a/src/lib_dal_node/gossipsub/gs_transport_connection.mli +++ b/src/lib_dal_node/gossipsub/gs_transport_connection.mli @@ -24,7 +24,7 @@ (* *) (*****************************************************************************) -(** [activate gs_worker transport_layer ~app_messages_handler] connects the +(** [activate gs_worker transport_layer ~app_messages_handler ~verbose] connects the given [gs_worker] and [transport_layer]. (Dis)connections and messages of the transport layer are forwarded to the GS worker. P2P output messages and (dis)connection requests are forwarded from the GS worker to the transport @@ -32,7 +32,9 @@ The [app_messages_handler] is invoked when some application messages are put by the Gossipsub worker in the application output stream. -*) + + The [verbose] flag controls the amount of events produced for some frequent + GS messages like the notification of messages to the application layer. *) val activate : Gs_interface.Worker_instance.t -> ( Gs_interface.Worker_instance.p2p_message, @@ -41,4 +43,5 @@ val activate : P2p.t -> app_messages_callback: (Types.Message.t -> Types.Message_id.t -> unit tzresult Lwt.t) -> + verbose:bool -> unit Lwt.t -- GitLab From bfbfe4442be0cdc5e1e5c808e1cfe3c0549e4dc4 Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Tue, 18 Feb 2025 17:50:05 +0100 Subject: [PATCH 5/6] DAL/Node: no app msg notifs when verbose in unset --- src/lib_dal_node/gossipsub/gs_transport_connection.ml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib_dal_node/gossipsub/gs_transport_connection.ml b/src/lib_dal_node/gossipsub/gs_transport_connection.ml index edbbf9a0d3ec..8ba136b07cd8 100644 --- a/src/lib_dal_node/gossipsub/gs_transport_connection.ml +++ b/src/lib_dal_node/gossipsub/gs_transport_connection.ml @@ -287,7 +287,7 @@ let transport_layer_inputs_handler gs_worker p2p_layer = (** This loop pops messages from application output stream and calls the given [app_messages_callback] on them. *) -let app_messages_handler gs_worker ~app_messages_callback ~verbose:_ = +let app_messages_handler gs_worker ~app_messages_callback ~verbose = let open Lwt_syntax in let rec loop app_output_stream = let* Worker.{message; message_id; topic = _} = @@ -296,7 +296,9 @@ let app_messages_handler gs_worker ~app_messages_callback ~verbose:_ = let* res = app_messages_callback message message_id in let* () = match res with - | Ok () -> Events.(emit message_notified_to_app message_id) + | Ok () -> + if not verbose then return_unit + else Events.(emit message_notified_to_app message_id) | Error err -> Events.(emit app_message_callback_failed (message_id, err)) in loop app_output_stream -- GitLab From 7e884da42e2aa396e99252ac5cad9ed4016c2392 Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Wed, 19 Feb 2025 08:04:44 +0100 Subject: [PATCH 6/6] DAL/Tezt: for tezt test, enable --verbose as we track detailed events --- tezt/lib_tezos/dal_node.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tezt/lib_tezos/dal_node.ml b/tezt/lib_tezos/dal_node.ml index 4553bb62ca72..3f401126473c 100644 --- a/tezt/lib_tezos/dal_node.ml +++ b/tezt/lib_tezos/dal_node.ml @@ -388,7 +388,7 @@ let run ?env ?event_level node = ?env ?event_level node - ["run"; "--data-dir"; node.persistent_state.data_dir] + ["run"; "--verbose"; "--data-dir"; node.persistent_state.data_dir] let run ?(wait_ready = true) ?env ?event_level node = let* () = run ?env ?event_level node in -- GitLab