From 3f2c3fbc327ac16beaca334029879cf98c5699b9 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Fri, 13 Jun 2025 11:17:24 +0100 Subject: [PATCH 1/3] Dal_node: Add profiler --- manifest/product_octez.ml | 4 ++++ src/lib_dal_node/dal_profiler.ml | 18 ++++++++++++++++++ src/lib_dal_node/dal_profiler.mli | 17 +++++++++++++++++ src/lib_dal_node/dune | 8 ++++++-- 4 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 src/lib_dal_node/dal_profiler.ml create mode 100644 src/lib_dal_node/dal_profiler.mli diff --git a/manifest/product_octez.ml b/manifest/product_octez.ml index aa88ae2659cb..886c3e991d1d 100644 --- a/manifest/product_octez.ml +++ b/manifest/product_octez.ml @@ -5276,6 +5276,7 @@ let octez_crawler = ] let octez_dal_node_lib = + let (PPX {preprocess; preprocessor_deps}) = ppx_profiler in public_lib "tezos-dal-node-lib" ~path:"src/lib_dal_node" @@ -5314,7 +5315,10 @@ let octez_dal_node_lib = prometheus_app; prometheus; octez_crawler |> open_; + octez_profiler |> open_; ] + ~preprocess + ~preprocessor_deps let _octez_dal_node_lib_test = tezt diff --git a/src/lib_dal_node/dal_profiler.ml b/src/lib_dal_node/dal_profiler.ml new file mode 100644 index 000000000000..e965b1992228 --- /dev/null +++ b/src/lib_dal_node/dal_profiler.ml @@ -0,0 +1,18 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2025 Trilitech *) +(* *) +(*****************************************************************************) + +open Profiler + +let dal_profiler = unplugged () + +let init profiler_maker = + match profiler_maker ~name:"dal" with + | Some instance -> plug dal_profiler instance + | None -> () + +let create_reset_block_section = + Profiler.section_maker Block_hash.equal Block_hash.to_b58check diff --git a/src/lib_dal_node/dal_profiler.mli b/src/lib_dal_node/dal_profiler.mli new file mode 100644 index 000000000000..223e26d8bc85 --- /dev/null +++ b/src/lib_dal_node/dal_profiler.mli @@ -0,0 +1,17 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2025 Trilitech *) +(* *) +(*****************************************************************************) + +open Profiler + +(** Unplugged DAL node profiler. *) +val dal_profiler : profiler + +(** Plug the DAL node profiler given its name and Profiler instance option. *) +val init : (name:string -> instance option) -> unit + +(** Creates a function to reset the block section *) +val create_reset_block_section : profiler -> Block_hash.t * metadata -> unit diff --git a/src/lib_dal_node/dune b/src/lib_dal_node/dune index 77b6f34211ba..db258314af75 100644 --- a/src/lib_dal_node/dune +++ b/src/lib_dal_node/dune @@ -36,7 +36,10 @@ octez-libs.opentelemetry-client-cohttp-lwt octez-libs.prometheus-app octez-libs.prometheus - octez-crawler) + octez-crawler + octez-libs.octez-profiler) + (preprocess (pps octez-libs.ppx_profiler)) + (preprocessor_deps (env_var TEZOS_PPX_PROFILER)) (flags (:standard) -open Tezos_base.TzPervasives @@ -57,4 +60,5 @@ -open Tezos_crypto -open Tezos_base_p2p_identity_file -open Tezos_shell_services - -open Octez_crawler)) + -open Octez_crawler + -open Tezos_profiler)) -- GitLab From 744c0d9585d1f8b5752fb19c0fb9e909808c9d9a Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Fri, 13 Jun 2025 11:53:23 +0100 Subject: [PATCH 2/3] Dal_node: Plug profiler --- manifest/product_octez.ml | 1 + src/bin_dal_node/dune | 4 +++- src/bin_dal_node/main.ml | 3 +++ src/lib_dal_node/daemon.ml | 17 +++++++++++++++++ 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/manifest/product_octez.ml b/manifest/product_octez.ml index 886c3e991d1d..f1353054c80b 100644 --- a/manifest/product_octez.ml +++ b/manifest/product_octez.ml @@ -9055,6 +9055,7 @@ let _octez_dal_node = octez_stdlib_unix |> open_; octez_dal_node_lib |> open_; memtrace; + octez_profiler_backends |> open_; ] @ protocol_deps) ~conflicts:[Conflicts.checkseum] diff --git a/src/bin_dal_node/dune b/src/bin_dal_node/dune index 2edf8735ab53..c66019c9c75b 100644 --- a/src/bin_dal_node/dune +++ b/src/bin_dal_node/dune @@ -13,6 +13,7 @@ octez-libs.stdlib-unix tezos-dal-node-lib memtrace + octez-libs.octez-profiler.backends (select void_for_linking-octez-protocol-021-PsQuebec-libs-dal from (octez-protocol-021-PsQuebec-libs.dal -> void_for_linking-octez-protocol-021-PsQuebec-libs-dal.empty) (-> void_for_linking-octez-protocol-021-PsQuebec-libs-dal.empty)) @@ -29,7 +30,8 @@ (:standard) -open Tezos_base.TzPervasives -open Tezos_stdlib_unix - -open Tezos_dal_node_lib)) + -open Tezos_dal_node_lib + -open Tezos_profiler_backends)) (rule (action diff --git a/src/bin_dal_node/main.ml b/src/bin_dal_node/main.ml index bfe7c6d144fa..f8d9864e0cc7 100644 --- a/src/bin_dal_node/main.ml +++ b/src/bin_dal_node/main.ml @@ -22,6 +22,9 @@ (* DEALINGS IN THE SOFTWARE. *) (* *) (*****************************************************************************) +(** Force linking for having Simple_profiler to init few backends by itself *) +let () = ignore Simple_profiler.headless + let _ = (* Memtrace can be activated via the environment variable MEMTRACE whose value is the file collecting the trace. The trace can then diff --git a/src/lib_dal_node/daemon.ml b/src/lib_dal_node/daemon.ml index c19c32e657df..ef7b2acb6c54 100644 --- a/src/lib_dal_node/daemon.ml +++ b/src/lib_dal_node/daemon.ml @@ -2,6 +2,7 @@ (* *) (* Open Source License *) (* Copyright (c) 2022 Nomadic Labs, *) +(* Copyright (c) 2025 Trilitech *) (* *) (* Permission is hereby granted, free of charge, to any person obtaining a *) (* copy of this software and associated documentation files (the "Software"),*) @@ -23,6 +24,20 @@ (* *) (*****************************************************************************) +module Profiler = struct + include (val Profiler.wrap Dal_profiler.dal_profiler) + + let[@warning "-32"] reset_block_section = + Dal_profiler.create_reset_block_section Dal_profiler.dal_profiler +end + +let[@warning "-32"] may_start_profiler data_dir = + match Tezos_profiler_unix.Profiler_instance.selected_backend () with + | Some {instance_maker; _} -> + let profiler_maker = instance_maker ~directory:data_dir in + Dal_profiler.init profiler_maker + | None -> () + let init_cryptobox config proto_parameters profile = let open Lwt_result_syntax in let prover_srs = Profile_manager.is_prover_profile profile in @@ -65,6 +80,7 @@ let on_new_finalized_head ctxt cctxt crawler = match next_final_head with | None -> Lwt.fail_with "L1 crawler lib shut down" | Some (finalized_block_hash, finalized_shell_header) -> + () [@profiler.reset_block_section finalized_block_hash] ; let* () = Block_handler.new_finalized_head ctxt @@ -247,6 +263,7 @@ let run ?(disable_logging = false) ?(disable_shard_validation = false) in Tezos_base_unix.Internal_event_unix.init ~config:internal_events () in + () [@profiler.overwrite may_start_profiler data_dir] ; let*! () = Event.emit_starting_node () in let* ({ rpc_addr; -- GitLab From eded91022e477141f8160629a3aac0da6f28a04c Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Mon, 16 Jun 2025 11:06:43 +0100 Subject: [PATCH 3/3] Dal_node: Initial profiling --- src/lib_dal_node/block_handler.ml | 132 ++++++++++++++++++------------ src/lib_dal_node/daemon.ml | 17 ++-- 2 files changed, 89 insertions(+), 60 deletions(-) diff --git a/src/lib_dal_node/block_handler.ml b/src/lib_dal_node/block_handler.ml index d42a495f45db..e623a88f7d00 100644 --- a/src/lib_dal_node/block_handler.ml +++ b/src/lib_dal_node/block_handler.ml @@ -23,6 +23,8 @@ (* *) (*****************************************************************************) +module Profiler = (val Profiler.wrap Dal_profiler.dal_profiler) + (* This function removes from the store the given slot and its shards. In case of error, this function emits a warning instead of failing. *) @@ -364,10 +366,11 @@ let process_block_data ctxt cctxt store proto_parameters block_level (module Plugin : Dal_plugin.T) = let open Lwt_result_syntax in let* block_info = - Plugin.block_info - cctxt - ~block:(`Level block_level) - ~operations_metadata:`Never + (Plugin.block_info + cctxt + ~block:(`Level block_level) + ~operations_metadata:`Never + [@profiler.record_s {verbosity = Notice} "block_info"]) in let* () = if Node_context.supports_refutations ctxt then @@ -377,21 +380,29 @@ let process_block_data ctxt cctxt store proto_parameters block_level proto_parameters ~attested_level:block_level (module Plugin : Dal_plugin.T) + [@profiler.record_s {verbosity = Notice} "store_skip_list_cells"] else return_unit in - let* slot_headers = Plugin.get_published_slot_headers ~block_level cctxt in + let* slot_headers = + (Plugin.get_published_slot_headers + ~block_level + cctxt [@profiler.record_s {verbosity = Notice} "slot_headers"]) + in let* () = - Slot_manager.store_slot_headers - ~number_of_slots:proto_parameters.Types.number_of_slots - ~block_level - slot_headers - store + (Slot_manager.store_slot_headers + ~number_of_slots:proto_parameters.Types.number_of_slots + ~block_level + slot_headers + store [@profiler.record_s {verbosity = Notice} "store_slot_headers"]) in let* () = (* If a slot header was posted to the L1 and we have the corresponding data, post it to gossipsub. Note that this is done independently of the profile. *) - let level_committee = Node_context.fetch_committee ctxt in + let level_committee = + (Node_context.fetch_committee + ctxt [@profiler.record_f {verbosity = Notice} "fetch_committee"]) + in let slot_size = proto_parameters.cryptobox_parameters.slot_size in let gs_worker = Node_context.get_gs_worker ctxt in List.iter_es @@ -399,48 +410,62 @@ let process_block_data ctxt cctxt store proto_parameters block_level let slot_id : Types.slot_id = {slot_level = published_level; slot_index} in - Slot_manager.publish_slot_data - ctxt - ~level_committee - ~slot_size - gs_worker - proto_parameters - commitment - slot_id) + (Slot_manager.publish_slot_data + ctxt + ~level_committee + ~slot_size + gs_worker + proto_parameters + commitment + slot_id + [@profiler.aggregate_s {verbosity = Notice} "publish_slot_data"])) slot_headers in - let*? dal_attestation = Plugin.dal_attestation block_info in + let*? dal_attestation = + (Plugin.dal_attestation + block_info [@profiler.record_f {verbosity = Notice} "dal_attestation"]) + in let* () = - Slot_manager.update_selected_slot_headers_statuses - ~block_level - ~attestation_lag:proto_parameters.attestation_lag - ~number_of_slots:proto_parameters.number_of_slots - (Plugin.is_attested dal_attestation) - store + (Slot_manager.update_selected_slot_headers_statuses + ~block_level + ~attestation_lag:proto_parameters.attestation_lag + ~number_of_slots:proto_parameters.number_of_slots + (Plugin.is_attested dal_attestation) + store + [@profiler.record_s + {verbosity = Notice} "update_selected_slot_headers_statuses"]) in let*! () = - remove_unattested_slots_and_shards - proto_parameters - ctxt - ~published_level: - Int32.(sub block_level (of_int proto_parameters.attestation_lag)) - (Plugin.is_attested dal_attestation) + (remove_unattested_slots_and_shards + proto_parameters + ctxt + ~published_level: + Int32.(sub block_level (of_int proto_parameters.attestation_lag)) + (Plugin.is_attested dal_attestation) + [@profiler.record_s + {verbosity = Notice} "remove_unattested_slots_and_shards"]) + in + let* attestations = + (Plugin.get_attestations + ~block_level + cctxt [@profiler.record_s {verbosity = Notice} "get_attestations"]) in - let* attestations = Plugin.get_attestations ~block_level cctxt in let* () = - check_attesters_attested - ctxt - proto_parameters - ~block_level - attestations - Plugin.is_attested + (check_attesters_attested + ctxt + proto_parameters + ~block_level + attestations + Plugin.is_attested + [@profiler.record_s {verbosity = Notice} "check_attesters_attested"]) in - Accuser.inject_entrapment_evidences - (module Plugin) - attestations - ctxt - cctxt - ~attested_level:block_level + (Accuser.inject_entrapment_evidences + (module Plugin) + attestations + ctxt + cctxt + ~attested_level:block_level + [@profiler.record_s {verbosity = Notice} "inject_entrapment_evidences"]) let process_block ctxt cctxt l1_crawler proto_parameters finalized_shell_header finalized_block_hash = @@ -463,6 +488,7 @@ let process_block ctxt cctxt l1_crawler proto_parameters finalized_shell_header proto_parameters block_level (module Plugin) + [@profiler.record_s {verbosity = Notice} "process_block_data"] else return_unit in let*? block_round = Plugin.get_round finalized_shell_header.fitness in @@ -491,13 +517,14 @@ let rec try_process_block ~retries ctxt cctxt l1_crawler proto_parameters finalized_shell_header finalized_block_hash = let open Lwt_syntax in let* res = - process_block - ctxt - cctxt - l1_crawler - proto_parameters - finalized_shell_header - finalized_block_hash + (process_block + ctxt + cctxt + l1_crawler + proto_parameters + finalized_shell_header + finalized_block_hash + [@profiler.record_s {verbosity = Notice} "process_block"]) in match res with | Error e when Layer_1.is_connection_error e && retries > 0 -> @@ -573,6 +600,7 @@ let new_finalized_head ctxt cctxt l1_crawler cryptobox finalized_block_hash proto_parameters finalized_shell_header finalized_block_hash + [@profiler.record_s {verbosity = Notice} "try_process_block"] in let end_time = Unix.gettimeofday () in Dal_metrics.per_level_processing_time (end_time -. launch_time) ; diff --git a/src/lib_dal_node/daemon.ml b/src/lib_dal_node/daemon.ml index ef7b2acb6c54..fd1368d5413f 100644 --- a/src/lib_dal_node/daemon.ml +++ b/src/lib_dal_node/daemon.ml @@ -82,14 +82,15 @@ let on_new_finalized_head ctxt cctxt crawler = | Some (finalized_block_hash, finalized_shell_header) -> () [@profiler.reset_block_section finalized_block_hash] ; let* () = - Block_handler.new_finalized_head - ctxt - cctxt - crawler - cryptobox - finalized_block_hash - finalized_shell_header - ~launch_time + (Block_handler.new_finalized_head + ctxt + cctxt + crawler + cryptobox + finalized_block_hash + finalized_shell_header + ~launch_time + [@profiler.record_s {verbosity = Notice} "new_finalized_head"]) in loop () in -- GitLab