From e6194b4b3bd456cbba080e6abb6b7221c3ba132a Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Wed, 5 Nov 2025 18:35:16 +0100 Subject: [PATCH 1/8] DAL/Node: add warning for status not found --- src/lib_dal_node/RPC_server.ml | 6 +++++- src/lib_dal_node/event.ml | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/lib_dal_node/RPC_server.ml b/src/lib_dal_node/RPC_server.ml index d43ce3309b6d..5616cec50963 100644 --- a/src/lib_dal_node/RPC_server.ml +++ b/src/lib_dal_node/RPC_server.ml @@ -323,7 +323,11 @@ module Profile_handlers = struct in match res with | Error `Not_found -> - (* that is, it was not published *) + let* () = + Event.emit_slot_header_status_not_found + ~published_level + ~slot_index + in Lwt.return_none | Error (`Other error) -> let* () = diff --git a/src/lib_dal_node/event.ml b/src/lib_dal_node/event.ml index 047fd86551f9..f9de50eefc0c 100644 --- a/src/lib_dal_node/event.ml +++ b/src/lib_dal_node/event.ml @@ -396,6 +396,18 @@ open struct ("published_level", Data_encoding.int32) ("slot_index", Data_encoding.int31) + let slot_header_status_not_found = + declare_2 + ~section + ~prefix_name_with_section:true + ~name:"slot_header_status_not_found" + ~msg: + "Slot header status not found for level {published_level}, slot index \ + {slot_index}." + ~level:Warning + ("published_level", Data_encoding.int32) + ("slot_index", Data_encoding.int31) + let slot_header_status_storage_error = declare_3 ~section @@ -1401,6 +1413,9 @@ let emit_removed_slot_shards ~published_level ~slot_index = let emit_removed_slot ~published_level ~slot_index = emit removed_slot (published_level, slot_index) +let emit_slot_header_status_not_found ~published_level ~slot_index = + emit slot_header_status_not_found (published_level, slot_index) + let emit_slot_header_status_storage_error ~published_level ~slot_index ~error = emit slot_header_status_storage_error (published_level, slot_index, error) -- GitLab From ae9c0f527ab99cc1af61e5192366f548dee5dcc2 Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Wed, 5 Nov 2025 18:36:26 +0100 Subject: [PATCH 2/8] DAL/Node: remove easy TODO in warn_missing_shards --- src/lib_dal_node/RPC_server.ml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/lib_dal_node/RPC_server.ml b/src/lib_dal_node/RPC_server.ml index 5616cec50963..970de89a67d5 100644 --- a/src/lib_dal_node/RPC_server.ml +++ b/src/lib_dal_node/RPC_server.ml @@ -361,11 +361,13 @@ module Profile_handlers = struct let* () = if List.is_empty ok then Lwt.return_unit else - let ok = - List.filter_map (function `Ok v -> Some v | `Not_ok _ -> None) ok + let slots_indices = + List.filter_map + (function + | `Ok (slot_index, _num_stored) -> Some slot_index + | `Not_ok _ -> None) + ok in - (* TODO: improve (do not go twice through the list) *) - let slots_indices, _ = List.split ok in Event.emit_get_attestable_slots_ok_notice ~attester ~published_level -- GitLab From 68c4014a8700ade5ada1d74975d7b425302c49bb Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Wed, 5 Nov 2025 18:32:42 +0100 Subject: [PATCH 3/8] DAL/Tests: wait for the DAL node after each bake_for in skip-list test --- tezt/tests/dal.ml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tezt/tests/dal.ml b/tezt/tests/dal.ml index 6e986c54c042..9ec5e7357706 100644 --- a/tezt/tests/dal.ml +++ b/tezt/tests/dal.ml @@ -5845,9 +5845,6 @@ module Skip_list_rpcs = struct last_confirmed_published_level last_attested_level max_level ; - let wait_for_dal_node = - wait_for_layer1_final_block dal_node last_attested_level - in let* second_level_new_proto = Client.level client in assert (second_level_new_proto = migration_level) ; @@ -5869,7 +5866,14 @@ module Skip_list_rpcs = struct let* current_level = Node.get_level node in let count = max_level - current_level in Log.info "Current level is %d. Bake %d more blocks." current_level count ; - bake_for ~count client + let wait_for_level = ref (current_level - 1) in + repeat count (fun () -> + let wait_for_dal_node = + wait_for_layer1_final_block dal_node !wait_for_level + in + incr wait_for_level ; + let* () = bake_for client in + wait_for_dal_node) in let module SeenIndexes = Set.Make (struct @@ -6002,7 +6006,6 @@ module Skip_list_rpcs = struct let* () = check_cell cell ~check_level:(Some level) in check_history (level + 1) in - let* () = wait_for_dal_node in Log.info "Check skip-list using commitments_history RPCs" ; let* () = check_history 1 in -- GitLab From 381415f15048055e2627b074e91140c6f03e76ed Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Wed, 5 Nov 2025 18:37:22 +0100 Subject: [PATCH 4/8] Baker/Seoul: wait 0.75s instead of 0.5s for the DAL node --- src/proto_023_PtSeouLo/lib_delegate/baking_actions.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/proto_023_PtSeouLo/lib_delegate/baking_actions.ml b/src/proto_023_PtSeouLo/lib_delegate/baking_actions.ml index cdd0dba8f493..1cce957d02f6 100644 --- a/src/proto_023_PtSeouLo/lib_delegate/baking_actions.ml +++ b/src/proto_023_PtSeouLo/lib_delegate/baking_actions.ml @@ -575,7 +575,7 @@ let may_get_dal_content state consensus_vote = wait for a bit for the DAL node to provide an answer. *) Lwt.pick [ - (let*! () = Lwt_unix.sleep 0.5 in + (let*! () = Lwt_unix.sleep 0.75 in Lwt.return `RPC_timeout); (let*! tz_res = promise in Lwt.return (`RPC_result tz_res)); -- GitLab From 9e0f8caac50aa6b5c501d73be46e89040eaf5c3a Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Wed, 5 Nov 2025 18:38:34 +0100 Subject: [PATCH 5/8] Baker/Tallinn: wait 0.75s instead of 0.5s for the DAL node --- src/proto_024_PsD5wVTJ/lib_delegate/baking_actions.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/proto_024_PsD5wVTJ/lib_delegate/baking_actions.ml b/src/proto_024_PsD5wVTJ/lib_delegate/baking_actions.ml index 6f2cfc167fd0..7f68156e82f6 100644 --- a/src/proto_024_PsD5wVTJ/lib_delegate/baking_actions.ml +++ b/src/proto_024_PsD5wVTJ/lib_delegate/baking_actions.ml @@ -567,7 +567,7 @@ let may_get_dal_content state consensus_vote = wait for a bit for the DAL node to provide an answer. *) Lwt.pick [ - (let*! () = Lwt_unix.sleep 0.5 in + (let*! () = Lwt_unix.sleep 0.75 in Lwt.return `RPC_timeout); (let*! tz_res = promise in Lwt.return (`RPC_result tz_res)); -- GitLab From 1b9e9050f85b2738d1e4907d0bb3a1454a0feef4 Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Wed, 5 Nov 2025 18:39:21 +0100 Subject: [PATCH 6/8] Baker/Seoul: add level info to event about DAL timeout --- src/proto_023_PtSeouLo/lib_delegate/baking_actions.ml | 2 +- src/proto_023_PtSeouLo/lib_delegate/baking_events.ml | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/proto_023_PtSeouLo/lib_delegate/baking_actions.ml b/src/proto_023_PtSeouLo/lib_delegate/baking_actions.ml index 1cce957d02f6..010086955f62 100644 --- a/src/proto_023_PtSeouLo/lib_delegate/baking_actions.ml +++ b/src/proto_023_PtSeouLo/lib_delegate/baking_actions.ml @@ -515,7 +515,7 @@ let process_dal_rpc_result state delegate level round = function | `RPC_timeout -> let*! () = - Events.(emit failed_to_get_dal_attestations_in_time delegate) + Events.(emit failed_to_get_dal_attestations_in_time (delegate, level)) in return_none | `RPC_result (Error errs) -> diff --git a/src/proto_023_PtSeouLo/lib_delegate/baking_events.ml b/src/proto_023_PtSeouLo/lib_delegate/baking_events.ml index ddeab8ec7b6a..68e2cd46a7b0 100644 --- a/src/proto_023_PtSeouLo/lib_delegate/baking_events.ml +++ b/src/proto_023_PtSeouLo/lib_delegate/baking_events.ml @@ -846,12 +846,15 @@ module Actions = struct ("trace", Error_monad.trace_encoding) let failed_to_get_dal_attestations_in_time = - declare_1 + declare_2 ~section ~name:"failed_to_get_attestations_in_time" ~level:Error - ~msg:"unable to get DAL attestation for {delegate} in time" + ~msg: + "unable to get DAL attestation for {delegate} in time for attestation \ + level {level}" ("delegate", Delegate_id.encoding) + ("level", Data_encoding.int32) let failed_to_inject_consensus_vote = declare_2 -- GitLab From 89aa221d9be8501a6cab0f2b1944258a579d7301 Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Wed, 5 Nov 2025 18:39:43 +0100 Subject: [PATCH 7/8] Baker/Tallinn: add level info to event about DAL timeout --- src/proto_024_PsD5wVTJ/lib_delegate/baking_actions.ml | 2 +- src/proto_024_PsD5wVTJ/lib_delegate/baking_events.ml | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/proto_024_PsD5wVTJ/lib_delegate/baking_actions.ml b/src/proto_024_PsD5wVTJ/lib_delegate/baking_actions.ml index 7f68156e82f6..0ee00d77bcd1 100644 --- a/src/proto_024_PsD5wVTJ/lib_delegate/baking_actions.ml +++ b/src/proto_024_PsD5wVTJ/lib_delegate/baking_actions.ml @@ -507,7 +507,7 @@ let process_dal_rpc_result state delegate level round = function | `RPC_timeout -> let*! () = - Events.(emit failed_to_get_dal_attestations_in_time delegate) + Events.(emit failed_to_get_dal_attestations_in_time (delegate, level)) in return_none | `RPC_result (Error errs) -> diff --git a/src/proto_024_PsD5wVTJ/lib_delegate/baking_events.ml b/src/proto_024_PsD5wVTJ/lib_delegate/baking_events.ml index 031f3d873fc5..f45222f3ae86 100644 --- a/src/proto_024_PsD5wVTJ/lib_delegate/baking_events.ml +++ b/src/proto_024_PsD5wVTJ/lib_delegate/baking_events.ml @@ -846,12 +846,15 @@ module Actions = struct ("trace", Error_monad.trace_encoding) let failed_to_get_dal_attestations_in_time = - declare_1 + declare_2 ~section ~name:"failed_to_get_attestations_in_time" ~level:Error - ~msg:"unable to get DAL attestation for {delegate} in time" + ~msg: + "unable to get DAL attestation for {delegate} in time for attestation \ + level {level}" ("delegate", Delegate_id.encoding) + ("level", Data_encoding.int32) let failed_to_inject_consensus_vote = declare_2 -- GitLab From b31df9ec63d14ad85c45550a28d0be9dbc17ce72 Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Fri, 7 Nov 2025 20:18:49 +0100 Subject: [PATCH 8/8] Baker/Alpha: rephrase message of event about DAL timeout --- src/proto_alpha/lib_delegate/baking_events.ml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/proto_alpha/lib_delegate/baking_events.ml b/src/proto_alpha/lib_delegate/baking_events.ml index a151e8e42e5e..f45222f3ae86 100644 --- a/src/proto_alpha/lib_delegate/baking_events.ml +++ b/src/proto_alpha/lib_delegate/baking_events.ml @@ -851,7 +851,8 @@ module Actions = struct ~name:"failed_to_get_attestations_in_time" ~level:Error ~msg: - "unable to get DAL attestation for {delegate} in time for level {level}" + "unable to get DAL attestation for {delegate} in time for attestation \ + level {level}" ("delegate", Delegate_id.encoding) ("level", Data_encoding.int32) -- GitLab