From 1c7d7b7d10e6188abdb69047b16e5ed4de6010fb Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Mon, 3 Feb 2025 15:07:05 +0100 Subject: [PATCH 1/3] DAL/Node: optimize should_store_skip_list_cells --- src/bin_dal_node/daemon.ml | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/bin_dal_node/daemon.ml b/src/bin_dal_node/daemon.ml index fea1a5dd3212..aa167859a29b 100644 --- a/src/bin_dal_node/daemon.ml +++ b/src/bin_dal_node/daemon.ml @@ -998,21 +998,26 @@ let clean_up_store ctxt cctxt ~last_processed_level ~first_seen_level head_level the plugin. *) Int32.(sub level (of_int period)) in - let should_store_skip_list_cells ~head_level ~level = - let profile_ctxt = Node_context.get_profile_ctxt ctxt in - let period = - get_storage_period - profile_ctxt - proto_parameters - head_level - first_seen_level - + skip_list_offset proto_parameters - in - supports_refutations - && level >= first_level_for_skip_list_storage period head_level + let should_store_skip_list_cells ~head_level = + if not supports_refutations then fun ~level:_ -> false + else + let profile_ctxt = Node_context.get_profile_ctxt ctxt in + let period = + get_storage_period + profile_ctxt + proto_parameters + head_level + first_seen_level + + skip_list_offset proto_parameters + in + let first_level = first_level_for_skip_list_storage period head_level in + fun ~level -> level >= first_level in let rec do_clean_up last_processed_level head_level = let last_level = target_level head_level in + let should_store_skip_list_cells = + should_store_skip_list_cells ~head_level + in let rec clean_up_at_level level = if level > last_level then return_unit else @@ -1020,7 +1025,7 @@ let clean_up_store ctxt cctxt ~last_processed_level ~first_seen_level head_level Handler.remove_old_level_stored_data proto_parameters ctxt level in let* () = - if should_store_skip_list_cells ~head_level ~level then + if should_store_skip_list_cells ~level then store_skip_list_cells ~level else return_unit in -- GitLab From 451d2ff2b3122a153e246fa451c02257deaece98 Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Mon, 3 Feb 2025 16:59:13 +0100 Subject: [PATCH 2/3] DAL/Node: emit catch-up events --- src/bin_dal_node/daemon.ml | 20 ++++++++++++++++++-- src/bin_dal_node/event.ml | 28 ++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/src/bin_dal_node/daemon.ml b/src/bin_dal_node/daemon.ml index aa167859a29b..82056cdb975f 100644 --- a/src/bin_dal_node/daemon.ml +++ b/src/bin_dal_node/daemon.ml @@ -1032,6 +1032,10 @@ let clean_up_store ctxt cctxt ~last_processed_level ~first_seen_level head_level let* () = Store.Last_processed_level.save store.last_processed_level level in + let*! () = + if Int32.to_int level mod 1000 = 0 then Event.(emit catching_up level) + else Lwt.return_unit + in clean_up_at_level (Int32.succ level) in (* Clean up from [last_processed_level] to [last_level]. *) @@ -1043,9 +1047,21 @@ let clean_up_store ctxt cctxt ~last_processed_level ~first_seen_level head_level in let new_head_level = header.Block_header.level in if new_head_level > head_level then do_clean_up last_level new_head_level - else return_unit + else + let*! () = Event.(emit end_catchup ()) in + return_unit in - do_clean_up last_processed_level head_level + let*! () = + let levels_to_clean_up = + Int32.(succ @@ sub head_level last_processed_level) + in + if levels_to_clean_up > 0l then + Event.( + emit start_catchup (last_processed_level, head_level, levels_to_clean_up)) + else Lwt.return_unit + in + let* () = do_clean_up last_processed_level head_level in + return_unit (* FIXME: https://gitlab.com/tezos/tezos/-/issues/3605 Improve general architecture, handle L1 disconnection etc diff --git a/src/bin_dal_node/event.ml b/src/bin_dal_node/event.ml index fdced3482ff9..34c618f3939d 100644 --- a/src/bin_dal_node/event.ml +++ b/src/bin_dal_node/event.ml @@ -710,3 +710,31 @@ let warn_attester_did_not_attest_slot = ("slot_index", Data_encoding.int31) ("attested_level", Data_encoding.int32) ~pp1:Signature.Public_key_hash.pp_short + +let start_catchup = + declare_3 + ~section + ~name:"start_catchup" + ~msg: + "catching up to level {end_level}, from last processed level \ + {start_level} (that is, {levels_to_clean_up} levels to process)" + ~level:Notice + ("start_level", Data_encoding.int32) + ("end_level", Data_encoding.int32) + ("levels_to_clean_up", Data_encoding.int32) + +let catching_up = + declare_1 + ~section + ~name:"catching_up" + ~msg:"caught up the store up to level {current_level}" + ~level:Notice + ("current_level", Data_encoding.int32) + +let end_catchup = + declare_0 + ~section + ~name:"end_catchup" + ~msg:"done catching up" + ~level:Notice + () -- GitLab From 03b374c7d87ac11daa7b80178d23ed297d69b95c Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Mon, 17 Feb 2025 20:55:51 +0100 Subject: [PATCH 3/3] DAL/Node: rename clean_up_store to mention catching up --- src/bin_dal_node/daemon.ml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bin_dal_node/daemon.ml b/src/bin_dal_node/daemon.ml index 82056cdb975f..6e1dccc4b195 100644 --- a/src/bin_dal_node/daemon.ml +++ b/src/bin_dal_node/daemon.ml @@ -969,8 +969,8 @@ let get_proto_plugins cctxt profile_ctxt ~last_processed_level ~first_seen_level and we don't detect it if this code starts running just before the migration level, and the head changes meanwhile to be above the migration level. *) -let clean_up_store ctxt cctxt ~last_processed_level ~first_seen_level head_level - proto_parameters = +let clean_up_store_and_catch_up ctxt cctxt ~last_processed_level + ~first_seen_level head_level proto_parameters = let open Lwt_result_syntax in let store_skip_list_cells ~level = let*? (module Plugin) = @@ -1332,7 +1332,7 @@ let run ~data_dir ~configuration_override = match last_processed_level with | None -> (* there's nothing to clean up *) return_unit | Some last_processed_level -> - clean_up_store + clean_up_store_and_catch_up ctxt cctxt ~last_processed_level -- GitLab