From 8976b0fd06b8678d60b03910260f9595b29c2f8b Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Wed, 8 Oct 2025 10:22:04 +0200 Subject: [PATCH 01/15] DAL/Plugin: cells_of_level returns the attestation_lag as well --- src/lib_dal_node/block_handler.ml | 2 +- src/lib_dal_node/dal_plugin.ml | 4 +++- src/lib_dal_node/dal_plugin.mli | 4 +++- src/lib_dal_node/slot_manager.ml | 5 +++-- .../lib_dal/dal_plugin_registration.ml | 10 +++++----- .../lib_dal/dal_plugin_registration.ml | 10 +++++----- .../lib_dal/dal_plugin_registration.ml | 10 +++++----- src/proto_alpha/lib_dal/dal_plugin_registration.ml | 8 ++++---- 8 files changed, 29 insertions(+), 24 deletions(-) diff --git a/src/lib_dal_node/block_handler.ml b/src/lib_dal_node/block_handler.ml index 415ae47224b2..2f8721ce2414 100644 --- a/src/lib_dal_node/block_handler.ml +++ b/src/lib_dal_node/block_handler.ml @@ -183,7 +183,7 @@ let store_skip_list_cells ctxt cctxt dal_constants ~attested_level in let cells_of_level = List.map - (fun (hash, cell, slot_index) -> + (fun (hash, cell, slot_index, _cell_attestation_lag) -> ( Dal_proto_types.Skip_list_hash.of_proto Plugin.Skip_list.hash_encoding hash, diff --git a/src/lib_dal_node/dal_plugin.ml b/src/lib_dal_node/dal_plugin.ml index 81100c925a1e..d379adfaa9e1 100644 --- a/src/lib_dal_node/dal_plugin.ml +++ b/src/lib_dal_node/dal_plugin.ml @@ -27,6 +27,8 @@ type operation_application_result = Succeeded | Failed type slot_index = int +type attestation_lag = int + type slot_header = { published_level : int32; slot_index : slot_index; @@ -125,7 +127,7 @@ module type T = sig dal_constants:Tezos_dal_node_services.Types.proto_parameters -> pred_publication_level_dal_constants: Tezos_dal_node_services.Types.proto_parameters tzresult Lwt.t Lazy.t -> - (hash * cell * slot_index) list tzresult Lwt.t + (hash * cell * slot_index * attestation_lag) list tzresult Lwt.t val slot_header_of_cell : cell -> slot_header option diff --git a/src/lib_dal_node/dal_plugin.mli b/src/lib_dal_node/dal_plugin.mli index 403338f8f2eb..d09a92f3b7a9 100644 --- a/src/lib_dal_node/dal_plugin.mli +++ b/src/lib_dal_node/dal_plugin.mli @@ -33,6 +33,8 @@ type operation_application_result = type slot_index = int +type attestation_lag = int + (** Information extracted from DAL slots headers operations included in L1 blocks. Each slot header is made of an L1 level for which it is published, the slot's index and commitment. *) @@ -176,7 +178,7 @@ module type T = sig dal_constants:Tezos_dal_node_services.Types.proto_parameters -> pred_publication_level_dal_constants: Tezos_dal_node_services.Types.proto_parameters tzresult Lwt.t Lazy.t -> - (hash * cell * slot_index) list tzresult Lwt.t + (hash * cell * slot_index * attestation_lag) list tzresult Lwt.t (** Extracts and returns the slot header of the given cell if it was published to L1. *) diff --git a/src/lib_dal_node/slot_manager.ml b/src/lib_dal_node/slot_manager.ml index d9c382ff319c..7ac2e653b453 100644 --- a/src/lib_dal_node/slot_manager.ml +++ b/src/lib_dal_node/slot_manager.ml @@ -405,10 +405,11 @@ let try_get_slot_header_from_L1_skip_list (module Plugin : Dal_plugin.T) ctxt in match List.find_all - (fun (_hash, _cell, cell_slot_index) -> cell_slot_index = slot_index) + (fun (_hash, _cell, cell_slot_index, _cell_attestation_lag) -> + cell_slot_index = slot_index) cells_of_level with - | [(_cell_hash, cell, _slot_index)] -> + | [(_cell_hash, cell, _slot_index, _cell_attestation_lag)] -> Plugin.Skip_list.slot_header_of_cell cell |> return | _ -> (* This should not happen (unless the slot index is not valid). In fact, diff --git a/src/proto_021_PsQuebec/lib_dal/dal_plugin_registration.ml b/src/proto_021_PsQuebec/lib_dal/dal_plugin_registration.ml index 0ab4095079cc..7ba64e712cc0 100644 --- a/src/proto_021_PsQuebec/lib_dal/dal_plugin_registration.ml +++ b/src/proto_021_PsQuebec/lib_dal/dal_plugin_registration.ml @@ -293,11 +293,11 @@ module Plugin = struct ~block:(`Level attested_level) ~operations_metadata:`Never in + let attestation_lag = + dal_constants.Tezos_dal_node_services.Types.attestation_lag + in let published_level = - Int32.sub - attested_level - (Int32.of_int - dal_constants.Tezos_dal_node_services.Types.attestation_lag) + Int32.sub attested_level (Int32.of_int attestation_lag) in (* 1. There are no cells for [published_level = 0]. *) if published_level <= 0l then return [] @@ -434,7 +434,7 @@ module Plugin = struct (* This should never happen: all hashes from [ordered_hashes_by_insertion] must exist in [last_cells_map]. *) assert false - | Some cell -> (hash, cell, slot_index)) + | Some cell -> (hash, cell, slot_index, attestation_lag)) ordered_hashes_by_insertion in return last_cells_ordered_by_insertion diff --git a/src/proto_022_PsRiotum/lib_dal/dal_plugin_registration.ml b/src/proto_022_PsRiotum/lib_dal/dal_plugin_registration.ml index f7f32a11ce97..76fa0002ee46 100644 --- a/src/proto_022_PsRiotum/lib_dal/dal_plugin_registration.ml +++ b/src/proto_022_PsRiotum/lib_dal/dal_plugin_registration.ml @@ -311,11 +311,11 @@ module Plugin = struct ~pred_publication_level_dal_constants:_ = let open Lwt_result_syntax in let cpctxt = new Protocol_client_context.wrap_rpc_context ctxt in + let attestation_lag = + dal_constants.Tezos_dal_node_services.Types.attestation_lag + in let published_level = - Int32.sub - attested_level - (Int32.of_int - dal_constants.Tezos_dal_node_services.Types.attestation_lag) + Int32.sub attested_level (Int32.of_int attestation_lag) in (* 1. There are no cells for [published_level = 0]. *) if published_level <= 0l then return [] @@ -335,7 +335,7 @@ module Plugin = struct Slots_history.(content cell |> content_id).index |> Slot_index.to_int) in - (hash, cell, slot_index)) + (hash, cell, slot_index, attestation_lag)) cells let slot_header_of_cell cell = diff --git a/src/proto_023_PtSeouLo/lib_dal/dal_plugin_registration.ml b/src/proto_023_PtSeouLo/lib_dal/dal_plugin_registration.ml index e826e831e5d9..efdc202b55d3 100644 --- a/src/proto_023_PtSeouLo/lib_dal/dal_plugin_registration.ml +++ b/src/proto_023_PtSeouLo/lib_dal/dal_plugin_registration.ml @@ -352,11 +352,11 @@ module Plugin = struct ~pred_publication_level_dal_constants:_ = let open Lwt_result_syntax in let cpctxt = new Protocol_client_context.wrap_rpc_context ctxt in + let attestation_lag = + dal_constants.Tezos_dal_node_services.Types.attestation_lag + in let published_level = - Int32.sub - attested_level - (Int32.of_int - dal_constants.Tezos_dal_node_services.Types.attestation_lag) + Int32.sub attested_level (Int32.of_int attestation_lag) in (* 1. There are no cells for [published_level = 0]. *) if published_level <= 0l then return [] @@ -376,7 +376,7 @@ module Plugin = struct Slots_history.(content cell |> content_id).index |> Slot_index.to_int) in - (hash, cell, slot_index)) + (hash, cell, slot_index, attestation_lag)) cells let slot_header_of_cell cell = diff --git a/src/proto_alpha/lib_dal/dal_plugin_registration.ml b/src/proto_alpha/lib_dal/dal_plugin_registration.ml index a9dc8002074f..2b79f6f965a1 100644 --- a/src/proto_alpha/lib_dal/dal_plugin_registration.ml +++ b/src/proto_alpha/lib_dal/dal_plugin_registration.ml @@ -411,10 +411,10 @@ module Plugin = struct let cell_id = H.(content cell |> content_id) in let slot_id = cell_id.H.header_id in let slot_index = Dal.Slot_index.to_int slot_id.index in + let attestation_lag = + H.attestation_lag_value cell_id.attestation_lag + in let* () = - let attestation_lag = - H.attestation_lag_value cell_id.attestation_lag - in let expected_attested_level = Raw_level.( add slot_id.published_level attestation_lag |> to_int32) @@ -424,7 +424,7 @@ module Plugin = struct (Attested_level_mismatch {attested_level; published_level; attestation_lag}) in - return (hash, cell, slot_index)) + return (hash, cell, slot_index, attestation_lag)) cells let slot_header_of_cell cell = -- GitLab From ac18f5ce69c369f64d87ea021602fc195dad8823 Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Wed, 8 Oct 2025 10:25:14 +0200 Subject: [PATCH 02/15] DAL/Plugin: fixup! some extra node needed for the check at migration block --- src/proto_alpha/lib_dal/dal_plugin_registration.ml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/proto_alpha/lib_dal/dal_plugin_registration.ml b/src/proto_alpha/lib_dal/dal_plugin_registration.ml index 2b79f6f965a1..5fbbc22dd778 100644 --- a/src/proto_alpha/lib_dal/dal_plugin_registration.ml +++ b/src/proto_alpha/lib_dal/dal_plugin_registration.ml @@ -420,6 +420,7 @@ module Plugin = struct add slot_id.published_level attestation_lag |> to_int32) in fail_unless + (* TODO: Needs to be adapted for migration block *) (Int32.equal attested_level expected_attested_level) (Attested_level_mismatch {attested_level; published_level; attestation_lag}) -- GitLab From a822a1a9c204d46a5ed6892d9518d51b67ca965c Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Mon, 6 Oct 2025 15:44:51 +0200 Subject: [PATCH 03/15] Protos/Tests: do not execute tests at top-level --- .../lib_protocol/test/unit/test_staking_operations.ml | 8 ++++---- .../lib_protocol/test/unit/test_staking_operations.ml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/proto_023_PtSeouLo/lib_protocol/test/unit/test_staking_operations.ml b/src/proto_023_PtSeouLo/lib_protocol/test/unit/test_staking_operations.ml index 7a8b38e61bb5..afa67b700afd 100644 --- a/src/proto_023_PtSeouLo/lib_protocol/test/unit/test_staking_operations.ml +++ b/src/proto_023_PtSeouLo/lib_protocol/test/unit/test_staking_operations.ml @@ -423,7 +423,7 @@ let () = (* Finn POC: creates 2 stakers whose unstake requests are finalized from a different account (finn). Note that we delay unstake requests so that they are issued in different cycles. *) -let finn_finalize_batch = +let finn_finalize_batch () = let open Lwt_result_syntax in let amount = Tez_helpers.of_int 10_000 in let* b, delegate = Context.init_with_constants1 constants in @@ -570,14 +570,14 @@ let finn_finalize_batch = let () = register_test ~title:"Finn: Finn finalizes staker1 and staker2 in a single batch." - @@ fun () -> finn_finalize_batch + @@ fun () -> finn_finalize_batch () (* Finn POC: Following Raphael's remark we test what happens if the finalize operation is submitted twice in a block. We create 3 stakers whose unstake requests are finalized from a diffrerent account (finn). Note that we delay unstake requests so that they are issued in different cycles. *) -let finn_finalize_interferance = +let finn_finalize_interferance () = let open Lwt_result_syntax in let amount = Tez_helpers.of_int 10_000 in let* b, delegate = Context.init_with_constants1 constants in @@ -772,4 +772,4 @@ let () = ~title: "Finn: Finn batch finalizes correctly even if one of the stakers submits \ a finalize operation in parallel." - @@ fun () -> finn_finalize_interferance + @@ fun () -> finn_finalize_interferance () diff --git a/src/proto_alpha/lib_protocol/test/unit/test_staking_operations.ml b/src/proto_alpha/lib_protocol/test/unit/test_staking_operations.ml index 8e5f0de438a9..59c9f4c57b74 100644 --- a/src/proto_alpha/lib_protocol/test/unit/test_staking_operations.ml +++ b/src/proto_alpha/lib_protocol/test/unit/test_staking_operations.ml @@ -423,7 +423,7 @@ let () = (* Finn POC: creates 2 stakers whose unstake requests are finalized from a different account (finn). Note that we delay unstake requests so that they are issued in different cycles. *) -let finn_finalize_batch = +let finn_finalize_batch () = let open Lwt_result_syntax in let amount = Tez_helpers.of_int 10_000 in let* b, delegate = Context.init_with_constants1 constants in @@ -570,14 +570,14 @@ let finn_finalize_batch = let () = register_test ~title:"Finn: Finn finalizes staker1 and staker2 in a single batch." - @@ fun () -> finn_finalize_batch + @@ fun () -> finn_finalize_batch () (* Finn POC: Following Raphael's remark we test what happens if the finalize operation is submitted twice in a block. We create 3 stakers whose unstake requests are finalized from a diffrerent account (finn). Note that we delay unstake requests so that they are issued in different cycles. *) -let finn_finalize_interferance = +let finn_finalize_interferance () = let open Lwt_result_syntax in let amount = Tez_helpers.of_int 10_000 in let* b, delegate = Context.init_with_constants1 constants in @@ -772,4 +772,4 @@ let () = ~title: "Finn: Finn batch finalizes correctly even if one of the stakers submits \ a finalize operation in parallel." - @@ fun () -> finn_finalize_interferance + @@ fun () -> finn_finalize_interferance () -- GitLab From a3a6ca4709ee9b9fe45e60c01cc82df8ddfe3d4f Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Tue, 30 Sep 2025 12:36:12 +0200 Subject: [PATCH 04/15] Proto/DAL: update [attestation_lag] to 5 --- src/proto_alpha/lib_parameters/default_parameters.ml | 2 +- src/proto_alpha/lib_protocol/raw_context.ml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/proto_alpha/lib_parameters/default_parameters.ml b/src/proto_alpha/lib_parameters/default_parameters.ml index a99a5879776c..8882314e1109 100644 --- a/src/proto_alpha/lib_parameters/default_parameters.ml +++ b/src/proto_alpha/lib_parameters/default_parameters.ml @@ -158,7 +158,7 @@ let default_dal = feature_enable = true; incentives_enable = true; number_of_slots = 32; - attestation_lag = 8; + attestation_lag = 5; attestation_threshold = 66; cryptobox_parameters = default_cryptobox_parameters; minimal_participation_ratio = Q.(64 // 100); diff --git a/src/proto_alpha/lib_protocol/raw_context.ml b/src/proto_alpha/lib_protocol/raw_context.ml index a83689e43882..8beb71227266 100644 --- a/src/proto_alpha/lib_protocol/raw_context.ml +++ b/src/proto_alpha/lib_protocol/raw_context.ml @@ -1469,7 +1469,7 @@ let prepare_first_block ~level ~timestamp chain_id ctxt = feature_enable; incentives_enable; number_of_slots; - attestation_lag; + attestation_lag = _; attestation_threshold; cryptobox_parameters; minimal_participation_ratio; @@ -1483,7 +1483,7 @@ let prepare_first_block ~level ~timestamp chain_id ctxt = Constants_parametric_repr.feature_enable; incentives_enable; number_of_slots; - attestation_lag; + attestation_lag = 5; attestation_threshold; cryptobox_parameters; minimal_participation_ratio; -- GitLab From 22771df1994b145b8dbd86d596f1f25007ab4e61 Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Thu, 2 Oct 2025 15:02:34 +0200 Subject: [PATCH 05/15] Tezt/lib_tezos: Dal.Parameters.from_client takes an optional argument ?block --- tezt/lib_tezos/dal_common.ml | 4 ++-- tezt/lib_tezos/dal_common.mli | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tezt/lib_tezos/dal_common.ml b/tezt/lib_tezos/dal_common.ml index 49c88f906596..5fba53a195d7 100644 --- a/tezt/lib_tezos/dal_common.ml +++ b/tezt/lib_tezos/dal_common.ml @@ -68,10 +68,10 @@ module Parameters = struct attestation_threshold; } - let from_client client = + let from_client ?block client = let* json = Client.RPC.call_via_endpoint client - @@ RPC.get_chain_block_context_constants () + @@ RPC.get_chain_block_context_constants ?block () in from_protocol_parameters json |> return diff --git a/tezt/lib_tezos/dal_common.mli b/tezt/lib_tezos/dal_common.mli index ee0105fb5eb3..1dbd7da96d4a 100644 --- a/tezt/lib_tezos/dal_common.mli +++ b/tezt/lib_tezos/dal_common.mli @@ -40,7 +40,7 @@ module Parameters : sig val from_protocol_parameters : JSON.t -> t - val from_client : Client.t -> t Lwt.t + val from_client : ?block:string -> Client.t -> t Lwt.t val from_endpoint : Endpoint.t -> t Lwt.t -- GitLab From 7527014264d1a38d42c718d6b976fc7730250b3a Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Fri, 3 Oct 2025 08:33:23 +0200 Subject: [PATCH 06/15] DAL/Proto: Helper function when finalizing slot headers of a published level --- .../lib_protocol/dal_slot_storage.ml | 67 ++++++++++++------- 1 file changed, 42 insertions(+), 25 deletions(-) diff --git a/src/proto_alpha/lib_protocol/dal_slot_storage.ml b/src/proto_alpha/lib_protocol/dal_slot_storage.ml index 0bcaf20a4732..dc0f88dbbd54 100644 --- a/src/proto_alpha/lib_protocol/dal_slot_storage.ml +++ b/src/proto_alpha/lib_protocol/dal_slot_storage.ml @@ -111,6 +111,43 @@ let remove_old_headers ctxt ~published_level = | None -> return ctxt | Some level -> Storage.Dal.Slot.Headers.remove ctxt level +(* Finalize DAL slot headers for a given (single) published level: + - Fetch headers published at [published_level]. + - Compute their attestation status via [is_slot_attested]. + - Update the DAL skip-list and storage, mutating [ctxt]: + + Prune old headers in Storage.Dal.Slot.Headers per denunciation window, + + Write the new skip-list head to Storage.Dal.Slot.History, + + Append per-level cells to Storage.Dal.Slot.LevelHistories. + - Return the updated [ctxt] and the attestation bitset. *) +let finalize_slot_headers_for_published_level ctxt ~number_of_slots + ~attestation_lag ~published_level = + let open Lwt_result_syntax in + let* published_slots = find_slot_headers ctxt published_level in + let*! ctxt = remove_old_headers ctxt ~published_level in + let* ctxt, attestation, slot_headers_statuses = + match published_slots with + | None -> return (ctxt, Dal_attestation_repr.empty, []) + | Some published_slots -> + let slot_headers_statuses, attestation = + let is_slot_attested slot = + Raw_context.Dal.is_slot_index_attested + ctxt + slot.Dal_slot_repr.Header.id.index + in + compute_slot_headers_statuses ~is_slot_attested published_slots + in + return (ctxt, attestation, slot_headers_statuses) + in + let* ctxt = + update_skip_list + ctxt + ~slot_headers_statuses + ~published_level + ~number_of_slots + ~attestation_lag + in + return (ctxt, attestation) + let finalize_pending_slot_headers ctxt ~number_of_slots = let open Lwt_result_syntax in let {Level_repr.level = raw_level; _} = Raw_context.current_level ctxt in @@ -119,28 +156,8 @@ let finalize_pending_slot_headers ctxt ~number_of_slots = match Raw_level_repr.(sub raw_level attestation_lag) with | None -> return (ctxt, Dal_attestation_repr.empty) | Some published_level -> - let* published_slots = find_slot_headers ctxt published_level in - let*! ctxt = remove_old_headers ctxt ~published_level in - let* ctxt, attestation, slot_headers_statuses = - match published_slots with - | None -> return (ctxt, Dal_attestation_repr.empty, []) - | Some published_slots -> - let slot_headers_statuses, attestation = - let is_slot_attested slot = - Raw_context.Dal.is_slot_index_attested - ctxt - slot.Dal_slot_repr.Header.id.index - in - compute_slot_headers_statuses ~is_slot_attested published_slots - in - return (ctxt, attestation, slot_headers_statuses) - in - let* ctxt = - update_skip_list - ctxt - ~slot_headers_statuses - ~published_level - ~number_of_slots - ~attestation_lag - in - return (ctxt, attestation) + finalize_slot_headers_for_published_level + ctxt + ~published_level + ~number_of_slots + ~attestation_lag -- GitLab From b17fd6d1b0efe84e222495e236cc0f65120cc79b Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Fri, 3 Oct 2025 08:34:57 +0200 Subject: [PATCH 07/15] DAL/Proto: provide ~is_slot_attested from outside when finalizing a published level --- src/proto_alpha/lib_protocol/dal_slot_storage.ml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/proto_alpha/lib_protocol/dal_slot_storage.ml b/src/proto_alpha/lib_protocol/dal_slot_storage.ml index dc0f88dbbd54..b466e76acd2d 100644 --- a/src/proto_alpha/lib_protocol/dal_slot_storage.ml +++ b/src/proto_alpha/lib_protocol/dal_slot_storage.ml @@ -120,7 +120,7 @@ let remove_old_headers ctxt ~published_level = + Append per-level cells to Storage.Dal.Slot.LevelHistories. - Return the updated [ctxt] and the attestation bitset. *) let finalize_slot_headers_for_published_level ctxt ~number_of_slots - ~attestation_lag ~published_level = + ~is_slot_attested ~attestation_lag ~published_level = let open Lwt_result_syntax in let* published_slots = find_slot_headers ctxt published_level in let*! ctxt = remove_old_headers ctxt ~published_level in @@ -129,11 +129,6 @@ let finalize_slot_headers_for_published_level ctxt ~number_of_slots | None -> return (ctxt, Dal_attestation_repr.empty, []) | Some published_slots -> let slot_headers_statuses, attestation = - let is_slot_attested slot = - Raw_context.Dal.is_slot_index_attested - ctxt - slot.Dal_slot_repr.Header.id.index - in compute_slot_headers_statuses ~is_slot_attested published_slots in return (ctxt, attestation, slot_headers_statuses) @@ -156,8 +151,14 @@ let finalize_pending_slot_headers ctxt ~number_of_slots = match Raw_level_repr.(sub raw_level attestation_lag) with | None -> return (ctxt, Dal_attestation_repr.empty) | Some published_level -> + let is_slot_attested slot = + Raw_context.Dal.is_slot_index_attested + ctxt + slot.Dal_slot_repr.Header.id.index + in finalize_slot_headers_for_published_level ctxt ~published_level ~number_of_slots ~attestation_lag + ~is_slot_attested -- GitLab From c58c9c5d5b961643ccb3fbf1b91343d7fa488356 Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Fri, 3 Oct 2025 08:58:06 +0200 Subject: [PATCH 08/15] DAL/Proto: prepare for attestation_lag shrink during published-level finalization --- .../lib_protocol/dal_slot_storage.ml | 54 +++++++++++++++---- 1 file changed, 44 insertions(+), 10 deletions(-) diff --git a/src/proto_alpha/lib_protocol/dal_slot_storage.ml b/src/proto_alpha/lib_protocol/dal_slot_storage.ml index b466e76acd2d..2fe7440118a3 100644 --- a/src/proto_alpha/lib_protocol/dal_slot_storage.ml +++ b/src/proto_alpha/lib_protocol/dal_slot_storage.ml @@ -151,14 +151,48 @@ let finalize_pending_slot_headers ctxt ~number_of_slots = match Raw_level_repr.(sub raw_level attestation_lag) with | None -> return (ctxt, Dal_attestation_repr.empty) | Some published_level -> - let is_slot_attested slot = - Raw_context.Dal.is_slot_index_attested - ctxt - slot.Dal_slot_repr.Header.id.index + (* DAL/TODO: remove after S->T migration: + + Detect whether we are at the first block after the lag shrink. We do + this by comparing the published level we are about to process with the + published level of the current skip-list head (called + head_published_level below). + + Normal case: + - moving from head_published_level to head_published_level + 1 => there + is one published level to process + + Migration case: + - moving from head_published_level to head_published_level + k (k > 1) + => We should process [k] published levels at once to avoid introducing + gaps in the skip list. This requires updating the context correctly, in + particular the [LevelHistories] entry. + + Particular case: on startup, the head may be the genesis cell with + published_level = 0. *) + let* sl_history_head = get_slot_headers_history ctxt in + let Dal_slot_repr.Header.{published_level = sl_head_published_level; _} = + Dal_slot_repr.History.(content sl_history_head |> content_id).header_id + in + let published_level_gap_size = + Raw_level_repr.diff published_level sl_head_published_level + |> Int32.to_int in - finalize_slot_headers_for_published_level - ctxt - ~published_level - ~number_of_slots - ~attestation_lag - ~is_slot_attested + if Compare.Int.(published_level_gap_size <= 1) then + (* Normal path: process the next published level. *) + let is_slot_attested slot = + Raw_context.Dal.is_slot_index_attested + ctxt + slot.Dal_slot_repr.Header.id.index + in + finalize_slot_headers_for_published_level + ctxt + ~published_level + ~number_of_slots + ~attestation_lag + ~is_slot_attested + else + (* Migration path: there are missing published levels between the + skip-list head and [published_level] because [attestation_lag] + has shrunk at migration. We will backfill them. *) + failwith "TODO: implemented in next commits" -- GitLab From fcc637a0a8fd634050c1a7ae18bdf86a492645c9 Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Fri, 3 Oct 2025 09:30:39 +0200 Subject: [PATCH 09/15] DAL/Proto: fill skip-list gap when attestation_lag shrinks --- .../lib_protocol/dal_slot_storage.ml | 75 ++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/src/proto_alpha/lib_protocol/dal_slot_storage.ml b/src/proto_alpha/lib_protocol/dal_slot_storage.ml index 2fe7440118a3..a1b643dc59cb 100644 --- a/src/proto_alpha/lib_protocol/dal_slot_storage.ml +++ b/src/proto_alpha/lib_protocol/dal_slot_storage.ml @@ -143,6 +143,74 @@ let finalize_slot_headers_for_published_level ctxt ~number_of_slots in return (ctxt, attestation) +(* Handle the first block after an attestation_lag shrink (S -> T). We + "backfill" the missing published levels so the DAL skip-list has no gaps: + process the [published_level_gap_size - 1] intermediate published levels in + order, then the [target_published_level]. + + Semantics during backfill: do NOT proto-attest slots. *) +let finalize_slot_headers_at_lag_migration ~published_level_gap_size ctxt + ~target_published_level ~number_of_slots ~attestation_lag = + let open Lwt_result_syntax in + (* During migration, backfilled published levels must not be proto-attested. + We enforce a conservative attestation status (no proto attest, zero + shards). *) + let is_slot_attested slot = + let status = + Raw_context.Dal.is_slot_index_attested + ctxt + slot.Dal_slot_repr.Header.id.index + in + {status with attested_shards = 0; is_proto_attested = false} + in + (* Process published levels from oldest to newest: + - Start at [target_published_level - (gap_size - 1)] + - Incrementally finalize each published level up to + [target_published_level] + + Notes: + - [LevelHistories] is overwritten at each finalize; we collect the + per-level cells after each step and batch-write them once at the end. + - Only the final attestation bitset (for [target_published_level]) is + returned for inclusion in the block header. This should not be an issue, as + backfilled levels are non-attesting by design during migration. *) + let rec aux ctxt ~cells_of_pub_levels ~gap_size = + let gap_size = gap_size - 1 in + match Raw_level_repr.(sub target_published_level gap_size) with + | None -> + (* Defensive: not expected on our networks. *) + return (ctxt, Dal_attestation_repr.empty, cells_of_pub_levels) + | Some published_level -> + (* Finalize this published level. *) + let* ctxt, attestation_bitset = + finalize_slot_headers_for_published_level + ctxt + ~number_of_slots + ~is_slot_attested + ~published_level + ~attestation_lag:(attestation_lag + gap_size) + in + (* Collect skip-list cells produced at this step. *) + let* cells_of_pub_levels = + let+ cells_of_this_pub_level = find_level_histories ctxt in + cells_of_this_pub_level :: cells_of_pub_levels + in + if Compare.Int.(gap_size = 0) then + (* Done: [target_published_level] processed. *) + return (ctxt, attestation_bitset, cells_of_pub_levels) + else aux ~gap_size ctxt ~cells_of_pub_levels + in + (* Main entry for processing several published levels at migration. *) + let* ctxt, attestation_bitset, cells_of_pub_levels = + aux ctxt ~cells_of_pub_levels:[] ~gap_size:published_level_gap_size + in + (* Persist all collected per-level cells in one write. *) + let*! ctxt = + List.(filter_map (fun e -> e) cells_of_pub_levels |> concat) + |> Storage.Dal.Slot.LevelHistories.add ctxt + in + return (ctxt, attestation_bitset) + let finalize_pending_slot_headers ctxt ~number_of_slots = let open Lwt_result_syntax in let {Level_repr.level = raw_level; _} = Raw_context.current_level ctxt in @@ -195,4 +263,9 @@ let finalize_pending_slot_headers ctxt ~number_of_slots = (* Migration path: there are missing published levels between the skip-list head and [published_level] because [attestation_lag] has shrunk at migration. We will backfill them. *) - failwith "TODO: implemented in next commits" + finalize_slot_headers_at_lag_migration + ~published_level_gap_size + ctxt + ~target_published_level:published_level + ~number_of_slots + ~attestation_lag -- GitLab From c60127b9887fed4940af12f88624c94f431b561b Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Wed, 1 Oct 2025 13:12:39 +0200 Subject: [PATCH 10/15] Tezt/DAL: adapt migration RPC test --- tezt/tests/dal.ml | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/tezt/tests/dal.ml b/tezt/tests/dal.ml index ff399ca872c4..1c3de6358efa 100644 --- a/tezt/tests/dal.ml +++ b/tezt/tests/dal.ml @@ -5413,13 +5413,14 @@ module History_rpcs = struct publish ~max_level:migration_level starting_level Map_int.empty in - let* dal_parameters = Dal.Parameters.from_client client in - let lag = dal_parameters.attestation_lag in - + let old_lag = dal_parameters.attestation_lag in let number_of_slots = dal_parameters.number_of_slots in - Log.info "attestation_lag = %d, number_of_slots = %d" lag number_of_slots ; + Log.info + "old attestation_lag = %d, old number_of_slots = %d" + old_lag + number_of_slots ; - let last_attested_level = last_confirmed_published_level + lag in + let last_attested_level = last_confirmed_published_level + old_lag in (* The maximum level that needs to be reached (we use +2 to make last attested level final). *) let max_level = last_attested_level + 2 in @@ -5428,9 +5429,19 @@ module History_rpcs = struct wait_for_layer1_final_block dal_node last_attested_level in - let* first_level_new_proto = Client.level client in + let* new_proto_params = + Node.RPC.call node @@ RPC.get_chain_block_context_constants () + in + let new_lag = + JSON.( + new_proto_params |-> "dal_parametric" |-> "attestation_lag" |> as_int) + in + Log.info "migrated to next protocol; new attestation_lag = %d" new_lag ; + + let* second_level_new_proto = Client.level client in + assert (second_level_new_proto = migration_level + 1) ; Log.info "Publishing some commitments in the new protocol@." ; - let* commitments = publish ~max_level first_level_new_proto commitments in + let* commitments = publish ~max_level second_level_new_proto commitments in let module SeenIndexes = Set.Make (struct type t = int @@ -5456,12 +5467,13 @@ module History_rpcs = struct (match check_level with | Some level -> assert (level >= first_dal_level) ; - let expected_level = + let expected_cell_level = if level = first_dal_level then (* the "level" of genesis *) 0 - else level - lag + else if level > migration_level then level - new_lag + else level - old_lag in Check.( - (cell_level = expected_level) + (cell_level = expected_cell_level) int ~error_msg:"Unexpected cell level: got %L, expected %R") | None -> ()) ; -- GitLab From e683d7e4800cde01481b4939eb91caf11df56c95 Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Thu, 9 Oct 2025 06:36:55 +0200 Subject: [PATCH 11/15] Tezt/DAL: add support for cells_of_level RPC --- tezt/lib_tezos/RPC.ml | 15 +++++++++++++++ tezt/lib_tezos/RPC.mli | 6 ++++++ 2 files changed, 21 insertions(+) diff --git a/tezt/lib_tezos/RPC.ml b/tezt/lib_tezos/RPC.ml index c4e3feab23dc..56360b9f9c24 100644 --- a/tezt/lib_tezos/RPC.ml +++ b/tezt/lib_tezos/RPC.ml @@ -1912,6 +1912,21 @@ let get_chain_block_context_dal_commitments_history ?(chain = "main") ["chains"; chain; "blocks"; block; "context"; "dal"; "commitments_history"] Fun.id +let get_chain_block_context_dal_cells_of_level ?(chain = "main") + ?(block = "head") () = + make + GET + [ + "chains"; + chain; + "blocks"; + block; + "context"; + "dal"; + "skip_list_cells_of_level"; + ] + Fun.id + let get_chain_block_context_raw_json ?(chain = "main") ?(block = "head") ?(path = []) () = make diff --git a/tezt/lib_tezos/RPC.mli b/tezt/lib_tezos/RPC.mli index af5d340d6c19..04c7412de43b 100644 --- a/tezt/lib_tezos/RPC.mli +++ b/tezt/lib_tezos/RPC.mli @@ -1360,6 +1360,12 @@ val get_chain_block_context_issuance_expected_issuance : val get_chain_block_context_dal_commitments_history : ?chain:string -> ?block:string -> unit -> JSON.t t +(** Call RPC + /chains/[chain]/blocks/[block]/context/dal/skip_list_cells_of_level. + [chain] defaults to ["main"]. [block] defaults to ["head"]. *) +val get_chain_block_context_dal_cells_of_level : + ?chain:string -> ?block:string -> unit -> JSON.t t + (** Call RPC /chains/[chain]/blocks/[block]/context/raw/json. [chain] defaults to ["main"]. [block] defaults to ["head"]. *) -- GitLab From 8bf7ef917ceaf92e4b8f0624224983ce2c5beb11 Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Thu, 9 Oct 2025 06:43:20 +0200 Subject: [PATCH 12/15] [TMP] DAL/Node: consult skip-list for get_slot_commitment RPC --- src/lib_dal_node/RPC_server.ml | 15 ++------------- src/lib_dal_node/slot_manager.mli | 3 +++ 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/src/lib_dal_node/RPC_server.ml b/src/lib_dal_node/RPC_server.ml index 193bc1589fbf..8e99ee69e406 100644 --- a/src/lib_dal_node/RPC_server.ml +++ b/src/lib_dal_node/RPC_server.ml @@ -236,20 +236,9 @@ module Slots_handlers = struct let get_slot_commitment ctxt slot_level slot_index () () = call_handler1 (fun () -> - let open Lwt_result_syntax in let slot_id : Types.slot_id = {slot_level; slot_index} in - let* content = - Slot_manager.get_slot_content - ~reconstruct_if_missing:true - ctxt - slot_id - in - let cryptobox = Node_context.get_cryptobox ctxt in - let*? polynomial = - Slot_manager.polynomial_from_slot cryptobox content - in - let*? commitment = Slot_manager.commit cryptobox polynomial in - return commitment) + Slot_manager.get_commitment_from_slot_id ctxt slot_id + |> Errors.other_lwt_result) let get_slot_status ctxt slot_level slot_index () () = call_handler1 (fun () -> diff --git a/src/lib_dal_node/slot_manager.mli b/src/lib_dal_node/slot_manager.mli index 4c83796fb03e..a104c97a3dce 100644 --- a/src/lib_dal_node/slot_manager.mli +++ b/src/lib_dal_node/slot_manager.mli @@ -102,6 +102,9 @@ val get_slot_content : Types.slot_id -> (slot, [> Errors.other | Errors.not_found]) result Lwt.t +val get_commitment_from_slot_id : + Node_context.t -> Types.slot_id -> Cryptobox.commitment tzresult Lwt.t + (** [add_commitment_shards ~shards_proofs_precomputation node_store cryptobox commitment slot polynomial] registers the shards of the slot whose commitment is given. -- GitLab From 9267b1ce4e466df85993ecb2c11800ebf68858fc Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Thu, 9 Oct 2025 06:50:15 +0200 Subject: [PATCH 13/15] DAL/Node: add published_level to skip-list-slots table and migrate db --- src/lib_dal_node/block_handler.ml | 18 +++---- src/lib_dal_node/dal_store_sqlite3.ml | 48 +++++++++++-------- src/lib_dal_node/dal_store_sqlite3.mli | 12 ++--- ...add_published_level_to_skip_list_slots.sql | 15 ++++++ .../script-inputs/dal_store_migrations | 1 + src/lib_dal_node/slot_manager.ml | 30 ++++++------ src/lib_dal_node/store.ml | 8 ++-- src/lib_dal_node/store.mli | 6 +-- .../DAL Node- debug print store schemas.out | 15 +++--- 9 files changed, 89 insertions(+), 64 deletions(-) create mode 100644 src/lib_dal_node/migrations/001_add_published_level_to_skip_list_slots.sql diff --git a/src/lib_dal_node/block_handler.ml b/src/lib_dal_node/block_handler.ml index 2f8721ce2414..fc016babaa5d 100644 --- a/src/lib_dal_node/block_handler.ml +++ b/src/lib_dal_node/block_handler.ml @@ -183,14 +183,15 @@ let store_skip_list_cells ctxt cctxt dal_constants ~attested_level in let cells_of_level = List.map - (fun (hash, cell, slot_index, _cell_attestation_lag) -> + (fun (hash, cell, slot_index, cell_attestation_lag) -> ( Dal_proto_types.Skip_list_hash.of_proto Plugin.Skip_list.hash_encoding hash, Dal_proto_types.Skip_list_cell.of_proto Plugin.Skip_list.cell_encoding cell, - slot_index )) + slot_index, + cell_attestation_lag )) cells_of_level in let store = Node_context.get_store ctxt in @@ -364,8 +365,8 @@ let check_attesters_attested node_ctxt committee slot_to_committee parameters in return_unit -let process_block_data ctxt cctxt store proto_parameters block_level - (module Plugin : Dal_plugin.T) = +let process_block_data ctxt cctxt store ~prev_proto_parameters ~proto_parameters + block_level (module Plugin : Dal_plugin.T) = let open Lwt_result_syntax in let* block_info = (Plugin.block_info @@ -379,7 +380,7 @@ let process_block_data ctxt cctxt store proto_parameters block_level store_skip_list_cells ctxt cctxt - proto_parameters + prev_proto_parameters ~attested_level:block_level (module Plugin : Dal_plugin.T) [@profiler.record_s {verbosity = Notice} "store_skip_list_cells"] @@ -500,8 +501,8 @@ let process_block ctxt cctxt l1_crawler proto_parameters finalized_shell_header let store = Node_context.get_store ctxt in let block_level = finalized_shell_header.Block_header.level in let pred_level = Int32.pred block_level in - let*? (module Plugin) = - Node_context.get_plugin_for_level ctxt ~level:pred_level + let*? (module Plugin), prev_proto_parameters = + Node_context.get_plugin_and_parameters_for_level ctxt ~level:pred_level in let* () = if proto_parameters.Types.feature_enable then @@ -512,7 +513,8 @@ let process_block ctxt cctxt l1_crawler proto_parameters finalized_shell_header ctxt cctxt store - proto_parameters + ~prev_proto_parameters + ~proto_parameters block_level (module Plugin) [@profiler.record_s {verbosity = Notice} "process_block_data"] diff --git a/src/lib_dal_node/dal_store_sqlite3.ml b/src/lib_dal_node/dal_store_sqlite3.ml index b9960b20ea6a..97875135f917 100644 --- a/src/lib_dal_node/dal_store_sqlite3.ml +++ b/src/lib_dal_node/dal_store_sqlite3.ml @@ -67,7 +67,7 @@ module Q = struct You can review the result at [tezt/tests/expected/dal.ml/DAL Node- debug print store schemas.out]. *) - let version = 2 + let version = 1 let all : Dal_node_migrations.migration list = Dal_node_migrations.migrations version @@ -138,6 +138,8 @@ module Types = struct open Tezos_dal_node_services.Types + let published_level : level Caqti_type.t = int32 + let attested_level : level Caqti_type.t = int32 let dal_slot_index : slot_index Caqti_type.t = int16 @@ -176,14 +178,14 @@ module Skip_list_cells = struct let find_by_slot_id_opt : (level * slot_index, Skip_list_cell.t, [`One | `Zero]) t = let open Caqti_type.Std in - (t2 attested_level dal_slot_index ->? skip_list_cell) + (t2 published_level dal_slot_index ->? skip_list_cell) @@ {sql| SELECT cell FROM skip_list_cells WHERE hash = ( SELECT skip_list_cell_hash FROM skip_list_slots - WHERE attested_level = $1 AND slot_index = $2 + WHERE published_level = $1 AND slot_index = $2 )|sql} let find_by_level : @@ -192,24 +194,27 @@ module Skip_list_cells = struct [`Zero | `One | `Many] ) t = let open Caqti_type.Std in - (attested_level ->* t3 skip_list_cell skip_list_hash int) + (published_level ->* t3 skip_list_cell skip_list_hash int) @@ {sql| SELECT c.cell, c.hash, c.slot_index FROM skip_list_cells AS c JOIN skip_list_slots AS s ON s.skip_list_cell_hash = c.hash - WHERE s.attested_level = $1 + WHERE s.published_level = $1 ORDER BY s.slot_index DESC |sql} let insert_skip_list_slot : - (level * slot_index * Skip_list_hash.t, unit, [`Zero]) t = - (t3 attested_level dal_slot_index skip_list_hash ->. unit) + (level * slot_index * level * Skip_list_hash.t, unit, [`Zero]) t = + (t4 published_level dal_slot_index attested_level skip_list_hash ->. unit) @@ {sql| INSERT INTO skip_list_slots - (attested_level, slot_index, skip_list_cell_hash) - VALUES ($1, $2, $3) - ON CONFLICT(attested_level, slot_index) DO UPDATE SET skip_list_cell_hash = $3 + (published_level, slot_index, attested_level, skip_list_cell_hash) + VALUES ($1, $2, $3, $4) + ON CONFLICT(published_level, slot_index) + DO UPDATE SET + attested_level = excluded.attested_level, + skip_list_cell_hash = excluded.skip_list_cell_hash |sql} let insert_skip_list_cell : @@ -223,20 +228,20 @@ module Skip_list_cells = struct |sql} let delete_skip_list_cell : (level, unit, [`Zero]) t = - (attested_level ->. unit) + (published_level ->. unit) @@ {sql| DELETE FROM skip_list_cells WHERE hash IN ( SELECT skip_list_cell_hash FROM skip_list_slots - WHERE attested_level = $1) + WHERE published_level = $1) |sql} let delete_skip_list_slot : (level, unit, [`Zero]) t = - (attested_level ->. unit) + (published_level ->. unit) @@ {sql| DELETE FROM skip_list_slots - WHERE attested_level = $1 + WHERE published_level = $1 |sql} end @@ -244,13 +249,13 @@ module Skip_list_cells = struct with_connection store conn @@ fun conn -> Sqlite.Db.find_opt conn Q.find_opt skip_list_hash - let find_by_slot_id_opt ?conn store ~attested_level ~slot_index = + let find_by_slot_id_opt ?conn store ~published_level ~slot_index = with_connection store conn @@ fun conn -> - Sqlite.Db.find_opt conn Q.find_by_slot_id_opt (attested_level, slot_index) + Sqlite.Db.find_opt conn Q.find_by_slot_id_opt (published_level, slot_index) - let find_by_level ?conn store ~attested_level = + let find_by_level ?conn store ~published_level = with_connection store conn @@ fun conn -> - Sqlite.Db.collect_list conn Q.find_by_level attested_level + Sqlite.Db.collect_list conn Q.find_by_level published_level let remove ?conn store ~attested_level = let open Lwt_result_syntax in @@ -263,12 +268,15 @@ module Skip_list_cells = struct let open Lwt_result_syntax in with_connection store conn @@ fun conn -> List.iter_es - (fun (cell_hash, cell, slot_index) -> + (fun (cell_hash, cell, slot_index, attestation_lag) -> + let published_level = + Int32.(sub attested_level (of_int attestation_lag)) + in let* () = Sqlite.Db.exec conn Q.insert_skip_list_slot - (attested_level, slot_index, cell_hash) + (published_level, slot_index, attested_level, cell_hash) in Sqlite.Db.exec conn Q.insert_skip_list_cell (cell_hash, cell)) items diff --git a/src/lib_dal_node/dal_store_sqlite3.mli b/src/lib_dal_node/dal_store_sqlite3.mli index 544f4c25754d..7a58d07d91f8 100644 --- a/src/lib_dal_node/dal_store_sqlite3.mli +++ b/src/lib_dal_node/dal_store_sqlite3.mli @@ -48,23 +48,23 @@ module Skip_list_cells : sig Skip_list_hash.t -> Skip_list_cell.t option tzresult Lwt.t - (** [find_by_slot_id_opt ?conn store ~attested_level ~slot_index] returns the cell - associated to ([attested_level], [slot_index]) in the [store], if + (** [find_by_slot_id_opt ?conn store ~published_level ~slot_index] returns the cell + associated to ([published_level], [slot_index]) in the [store], if any. Uses the [conn] if provided (defaults to [None]). *) val find_by_slot_id_opt : ?conn:conn -> t -> - attested_level:int32 -> + published_level:int32 -> slot_index:Types.slot_index -> Skip_list_cell.t option tzresult Lwt.t - (** [find_by_level ?conn store ~attested_level] retrieves the tuples (cell * + (** [find_by_level ?conn store ~published_level] retrieves the tuples (cell * hash * slot_index) for the given attested level, if any. The results are sorted in decreasing order w.r.t. slot indices. *) val find_by_level : ?conn:conn -> t -> - attested_level:int32 -> + published_level:int32 -> (Dal_proto_types.Skip_list_cell.t * Dal_proto_types.Skip_list_hash.t * Types.slot_index) @@ -80,7 +80,7 @@ module Skip_list_cells : sig ?conn:conn -> t -> attested_level:int32 -> - (Skip_list_hash.t * Skip_list_cell.t * Types.slot_index) list -> + (Skip_list_hash.t * Skip_list_cell.t * Types.slot_index * int) list -> unit tzresult Lwt.t (** [remove ?conn store ~attested_level] removes any data related to diff --git a/src/lib_dal_node/migrations/001_add_published_level_to_skip_list_slots.sql b/src/lib_dal_node/migrations/001_add_published_level_to_skip_list_slots.sql new file mode 100644 index 000000000000..e25543d45444 --- /dev/null +++ b/src/lib_dal_node/migrations/001_add_published_level_to_skip_list_slots.sql @@ -0,0 +1,15 @@ +CREATE TABLE skip_list_slots_new ( + published_level INTEGER NOT NULL, + attested_level INTEGER NOT NULL, + slot_index INTEGER NOT NULL, + skip_list_cell_hash VARCHAR NOT NULL, + PRIMARY KEY(published_level, slot_index) +); + +INSERT INTO skip_list_slots_new (published_level, slot_index, attested_level, skip_list_cell_hash) +SELECT attested_level - 8, slot_index, attested_level, skip_list_cell_hash +FROM skip_list_slots; + +DROP TABLE skip_list_slots; + +ALTER TABLE skip_list_slots_new RENAME TO skip_list_slots; diff --git a/src/lib_dal_node/script-inputs/dal_store_migrations b/src/lib_dal_node/script-inputs/dal_store_migrations index 86aedc60d406..aeaa25b4c5ab 100644 --- a/src/lib_dal_node/script-inputs/dal_store_migrations +++ b/src/lib_dal_node/script-inputs/dal_store_migrations @@ -1 +1,2 @@ 7655cb4123007e77d2593aef3119382acb0f979665f2f25d00723f3591dcecde src/lib_dal_node/migrations/000_initial.sql +dc3bc8ce4010a0e0de6b435abc13839e28105522af45aac3b0c3efa6fdffaea9 src/lib_dal_node/migrations/001_add_published_level_to_skip_list_slots.sql diff --git a/src/lib_dal_node/slot_manager.ml b/src/lib_dal_node/slot_manager.ml index 7ac2e653b453..179b9ac5a3c2 100644 --- a/src/lib_dal_node/slot_manager.ml +++ b/src/lib_dal_node/slot_manager.ml @@ -356,20 +356,19 @@ let try_get_commitment_of_slot_id_from_memory ctxt slot_id = skip list cell stored in the SQLite store. Steps: - - Fetch the skip list cell for the given [attested_level] and slot index from - the SQLite store. + - Fetch the skip list cell for the given [slot_id] from the SQLite store. - Decode the cell using the DAL plugin. - Return the extracted slot header, if available. Returns [None] if the cell is not found in the store. *) let try_get_slot_header_from_indexed_skip_list (module Plugin : Dal_plugin.T) - ctxt ~attested_level slot_id = + ctxt slot_id = let open Lwt_result_syntax in let* cell_bytes_opt = Store.Skip_list_cells.find_by_slot_id_opt (Node_context.get_store ctxt) - ~attested_level - ~slot_index:slot_id.Types.Slot_id.slot_index + ~published_level:slot_id.Types.Slot_id.slot_level + ~slot_index:slot_id.slot_index in match cell_bytes_opt with | None -> return_none @@ -436,15 +435,12 @@ let try_get_commitment_of_slot_id_from_skip_list dal_plugin ctxt dal_constants let open Lwt_result_syntax in let*! published_slot_header_opt = let*! from_sqlite = - try_get_slot_header_from_indexed_skip_list - dal_plugin - ctxt - ~attested_level - slot_id + try_get_slot_header_from_indexed_skip_list dal_plugin ctxt slot_id in match from_sqlite with | Ok (Some _header as res) -> return res | _ -> + (* TODO: the attested level is wrong! *) try_get_slot_header_from_L1_skip_list dal_plugin ctxt @@ -472,21 +468,23 @@ let get_commitment_from_slot_id ctxt slot_id = | Some res -> return res | None -> ( let published_level = slot_id.Types.Slot_id.slot_level in - let*? dal_plugin, dal_constants = - Node_context.get_plugin_and_parameters_for_level - ctxt - ~level:published_level + let*? parameters = + Node_context.get_proto_parameters ctxt ~level:(`Level published_level) in let attested_level = Int32.add published_level - (Int32.of_int dal_constants.Types.attestation_lag) + (Int32.of_int parameters.Types.attestation_lag) + in + let*? dal_plugin = + Node_context.get_plugin_for_level ctxt ~level:attested_level in + let*! res = try_get_commitment_of_slot_id_from_skip_list dal_plugin ctxt - dal_constants + parameters slot_id ~attested_level in diff --git a/src/lib_dal_node/store.ml b/src/lib_dal_node/store.ml index 7c5dd959e308..5a45488aec27 100644 --- a/src/lib_dal_node/store.ml +++ b/src/lib_dal_node/store.ml @@ -908,18 +908,18 @@ module Skip_list_cells = struct t.skip_list_cells_store skip_list_hash - let find_by_slot_id_opt ?conn t ~attested_level ~slot_index = + let find_by_slot_id_opt ?conn t ~published_level ~slot_index = Dal_store_sqlite3.Skip_list_cells.find_by_slot_id_opt ?conn t.skip_list_cells_store - ~attested_level + ~published_level ~slot_index - let find_by_level ?conn t ~attested_level = + let find_by_level ?conn t ~published_level = Dal_store_sqlite3.Skip_list_cells.find_by_level ?conn t.skip_list_cells_store - ~attested_level + ~published_level let insert ?conn t ~attested_level items = Dal_store_sqlite3.Skip_list_cells.insert diff --git a/src/lib_dal_node/store.mli b/src/lib_dal_node/store.mli index a9c4f9b4ff83..a79edc3949ee 100644 --- a/src/lib_dal_node/store.mli +++ b/src/lib_dal_node/store.mli @@ -261,7 +261,7 @@ module Skip_list_cells : sig val find_by_slot_id_opt : ?conn:Sqlite.conn -> t -> - attested_level:int32 -> + published_level:int32 -> slot_index:Types.slot_index -> Dal_proto_types.Skip_list_cell.t option tzresult Lwt.t @@ -269,7 +269,7 @@ module Skip_list_cells : sig val find_by_level : ?conn:Sqlite.conn -> t -> - attested_level:int32 -> + published_level:int32 -> (Dal_proto_types.Skip_list_cell.t * Dal_proto_types.Skip_list_hash.t * Types.slot_index) @@ -284,7 +284,7 @@ module Skip_list_cells : sig ?conn:Dal_store_sqlite3.conn -> t -> attested_level:int32 -> - (Skip_list_hash.t * Skip_list_cell.t * Types.slot_index) list -> + (Skip_list_hash.t * Skip_list_cell.t * Types.slot_index * int) list -> unit tzresult Lwt.t (** [remove ?conn store ~attested_level] removes any data related to [attested_level] diff --git a/tezt/tests/expected/dal.ml/DAL Node- debug print store schemas.out b/tezt/tests/expected/dal.ml/DAL Node- debug print store schemas.out index 8b8347eba8f5..9ff6d16fc1a4 100644 --- a/tezt/tests/expected/dal.ml/DAL Node- debug print store schemas.out +++ b/tezt/tests/expected/dal.ml/DAL Node- debug print store schemas.out @@ -1,13 +1,14 @@ ./octez-dal-node debug print store schemas -CREATE TABLE skip_list_slots ( - attested_level INTEGER NOT NULL, - slot_index INTEGER NOT NULL, - skip_list_cell_hash VARCHAR NOT NULL, - PRIMARY KEY(attested_level, slot_index) -); - +There are 2 migrations to be done CREATE TABLE skip_list_cells ( hash VARCHAR NOT NULL PRIMARY KEY, cell VARCHAR NOT NULL +); + +CREATE TABLE "skip_list_slots" ( + published_level INTEGER NOT NULL, + slot_index INTEGER NOT NULL, + skip_list_cell_hash VARCHAR NOT NULL, + PRIMARY KEY(published_level, slot_index) ) -- GitLab From 55443bf943e07a1512ece0d7e2828a6c709df041 Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Thu, 9 Oct 2025 06:51:17 +0200 Subject: [PATCH 14/15] [TMP] DAL/Plugin: remove check on attested level in cells_of_level --- .../lib_dal/dal_plugin_registration.ml | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/proto_alpha/lib_dal/dal_plugin_registration.ml b/src/proto_alpha/lib_dal/dal_plugin_registration.ml index 5fbbc22dd778..3a9e4756ae16 100644 --- a/src/proto_alpha/lib_dal/dal_plugin_registration.ml +++ b/src/proto_alpha/lib_dal/dal_plugin_registration.ml @@ -414,17 +414,17 @@ module Plugin = struct let attestation_lag = H.attestation_lag_value cell_id.attestation_lag in - let* () = - let expected_attested_level = - Raw_level.( - add slot_id.published_level attestation_lag |> to_int32) - in - fail_unless - (* TODO: Needs to be adapted for migration block *) - (Int32.equal attested_level expected_attested_level) - (Attested_level_mismatch - {attested_level; published_level; attestation_lag}) - in + (* let* () = *) + (* let expected_attested_level = *) + (* Raw_level.( *) + (* add slot_id.published_level attestation_lag |> to_int32) *) + (* in *) + (* fail_unless *) + (* (\* TODO: Needs to be adapted for migration block *\) *) + (* (Int32.equal attested_level expected_attested_level) *) + (* (Attested_level_mismatch *) + (* {attested_level; published_level; attestation_lag}) *) + (* in *) return (hash, cell, slot_index, attestation_lag)) cells -- GitLab From 82e55c5937c6f559679e9251128e44bb080ad094 Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Thu, 9 Oct 2025 06:52:41 +0200 Subject: [PATCH 15/15] DAL/Tests: further update skip-list test --- tezt/tests/dal.ml | 131 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 112 insertions(+), 19 deletions(-) diff --git a/tezt/tests/dal.ml b/tezt/tests/dal.ml index 1c3de6358efa..bfc2079aba95 100644 --- a/tezt/tests/dal.ml +++ b/tezt/tests/dal.ml @@ -5374,7 +5374,7 @@ module History_rpcs = struct ~first_dal_level ~last_confirmed_published_level protocol dal_parameters client node dal_node = let module Map_int = Map.Make (Int) in - Log.info "slot_index = %d" slot_index ; + Log.info "slot_index = %d first_dal_level = %d" slot_index first_dal_level ; let client = Client.with_dal_node client ~dal_node in let slot_size = dal_parameters.Dal.Parameters.cryptobox.slot_size in let* starting_level = Client.level client in @@ -5397,10 +5397,6 @@ module History_rpcs = struct @@ Helpers.make_slot ~slot_size ("slot " ^ string_of_int level) in let* () = wait_mempool_injection in - Log.info - "Publish commitment %s at published_level %d@." - commitment - published_level ; let* () = bake_for client in let* _level = Node.wait_for_level node (level + 1) in publish @@ -5408,7 +5404,10 @@ module History_rpcs = struct (level + 1) (Map_int.add published_level commitment commitments) in - Log.info "Publishing some commitments in the previous protocol@." ; + Log.info + "Publishing commitments in the previous protocol, from level %d to %d" + (starting_level + 1) + (migration_level + 1) ; let* commitments = publish ~max_level:migration_level starting_level Map_int.empty in @@ -5420,15 +5419,6 @@ module History_rpcs = struct old_lag number_of_slots ; - let last_attested_level = last_confirmed_published_level + old_lag in - (* The maximum level that needs to be reached (we use +2 to make last - attested level final). *) - let max_level = last_attested_level + 2 in - - let wait_for_dal_node = - wait_for_layer1_final_block dal_node last_attested_level - in - let* new_proto_params = Node.RPC.call node @@ RPC.get_chain_block_context_constants () in @@ -5436,12 +5426,34 @@ module History_rpcs = struct JSON.( new_proto_params |-> "dal_parametric" |-> "attestation_lag" |> as_int) in - Log.info "migrated to next protocol; new attestation_lag = %d" new_lag ; + Log.info "Migrated to the next protocol; new attestation_lag = %d" new_lag ; + + let last_attested_level = last_confirmed_published_level + new_lag in + + let wait_for_dal_node = + wait_for_layer1_final_block dal_node last_attested_level + in let* second_level_new_proto = Client.level client in assert (second_level_new_proto = migration_level + 1) ; - Log.info "Publishing some commitments in the new protocol@." ; - let* commitments = publish ~max_level second_level_new_proto commitments in + Log.info + "Publish commitments in the new protocol, from level %d to %d" + (second_level_new_proto + 1) + (last_confirmed_published_level + 1) ; + let* commitments = + publish + ~max_level:last_confirmed_published_level + second_level_new_proto + commitments + in + let* () = + (* The maximum level that needs to be reached (we use +2 to make last + attested level final). *) + let max_level = last_attested_level + 2 in + let* current_level = Node.get_level node in + let count = max_level + 1 - current_level in + bake_for ~count client + in let module SeenIndexes = Set.Make (struct type t = int @@ -5562,11 +5574,92 @@ module History_rpcs = struct check_history (level + 1) in let* () = wait_for_dal_node in + Log.info "Check skip-list using commitments_history RPCs" ; let* () = check_history first_dal_level in + Check.( (!at_least_one_attested_status = true) bool ~error_msg:"No cell with the 'attested' status has been visited") ; + + let rec call_cells_of_level level = + if level > last_confirmed_published_level then unit + else + let* cells = + Node.RPC.call node + @@ RPC.get_chain_block_context_dal_cells_of_level + ~block:(string_of_int level) + () + in + let cells = JSON.as_list cells in + let num_cells = List.length cells in + let expected_num_cells = + if level < old_lag then 0 + else if level = migration_level + 1 then 4 * 32 + else 32 + in + Check.( + (num_cells = expected_num_cells) + int + ~error_msg:"Unexpected number of cells: got %L, expected %R") ; + call_cells_of_level (level + 1) + in + Log.info + "Call cells_of_level on each relevant level, and check the number of \ + cells returned" ; + let* () = call_cells_of_level 1 in + + (* Restart the DAL node. In this way, we don't have the slots in memory and + we need to retrieve them via the skip list. *) + let* () = + let* () = Dal_node.terminate dal_node in + Dal_node.run dal_node + in + let rec call_get_commitment level = + if level > last_confirmed_published_level then unit + else + let* commitment = + Dal_RPC.( + call dal_node + @@ get_level_index_commitment ~slot_level:level ~slot_index) + in + (* Log.info "commitment at published level %d: %s" level commitment ; *) + let expected_commitment = + match Map_int.find_opt level commitments with + | Some c -> c + | None -> Test.fail "Commitment not found at level %d" level + in + Check.( + (commitment = expected_commitment) + string + ~error_msg: + (let msg = sf "Unexpected commitment at level %d: " level in + msg ^ ": got %L, expected %R")) ; + call_get_commitment (level + 1) + in + Log.info "Check fetching commitments from the skip-list store" ; + let* () = call_get_commitment 2 in + + (* Remove the skip-list DB. In this way, commitment are retrieved from the L1 context. *) + let* () = + let* () = Dal_node.terminate dal_node in + let skip_list_db = + Format.sprintf "%s/store/skip_list_store" @@ Dal_node.data_dir dal_node + in + Log.info "Removing the skip list DB at %s" skip_list_db ; + let () = Sys.command ("rm -rf " ^ skip_list_db) |> ignore in + Dal_node.run dal_node + in + Log.info "Check fetching commitments from the L1 context" ; + let* () = call_get_commitment 2 in + + (* In case we cannot use the updated GET slot_commitment RPC, then we need + to go through a second DAL node: + - remove the slot and shard store; + - start a 2nd DAL node having the 1st DAL node as a backup + - call get_slot_content on the 2nd DAL node /levels//slots//content + - NOTE: the 2nd node should have the skip list locally, so it needs to + be running from the start *) unit let test_commitments_history_rpcs protocols = @@ -11725,7 +11818,7 @@ let tests_start_dal_node_around_migration ~migrate_from ~migrate_to = let register_migration ~migrate_from ~migrate_to = test_migration_plugin ~migration_level:4 ~migrate_from ~migrate_to ; History_rpcs.test_commitments_history_rpcs_with_migration - ~migration_level:10 + ~migration_level:11 ~migrate_from ~migrate_to ; tests_start_dal_node_around_migration ~migrate_from ~migrate_to ; -- GitLab