From 21596e688bbc620409c6234f14074e870db5f9f0 Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Thu, 2 Oct 2025 15:02:34 +0200 Subject: [PATCH 1/7] Tezt/lib_tezos: Dal.Parameters.from_client takes an optional argument ?block --- tezt/lib_tezos/dal_common.ml | 4 ++-- tezt/lib_tezos/dal_common.mli | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tezt/lib_tezos/dal_common.ml b/tezt/lib_tezos/dal_common.ml index 49c88f906596..5fba53a195d7 100644 --- a/tezt/lib_tezos/dal_common.ml +++ b/tezt/lib_tezos/dal_common.ml @@ -68,10 +68,10 @@ module Parameters = struct attestation_threshold; } - let from_client client = + let from_client ?block client = let* json = Client.RPC.call_via_endpoint client - @@ RPC.get_chain_block_context_constants () + @@ RPC.get_chain_block_context_constants ?block () in from_protocol_parameters json |> return diff --git a/tezt/lib_tezos/dal_common.mli b/tezt/lib_tezos/dal_common.mli index ee0105fb5eb3..1dbd7da96d4a 100644 --- a/tezt/lib_tezos/dal_common.mli +++ b/tezt/lib_tezos/dal_common.mli @@ -40,7 +40,7 @@ module Parameters : sig val from_protocol_parameters : JSON.t -> t - val from_client : Client.t -> t Lwt.t + val from_client : ?block:string -> Client.t -> t Lwt.t val from_endpoint : Endpoint.t -> t Lwt.t -- GitLab From f96ba56494f18b1667629b4c850b4da7d20bbe02 Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Fri, 10 Oct 2025 18:12:52 +0200 Subject: [PATCH 2/7] DAL/Tezt: rename main scenario function to main_scenario to avoid confusion --- tezt/tests/dal.ml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tezt/tests/dal.ml b/tezt/tests/dal.ml index 3942d158519e..e013925ccff4 100644 --- a/tezt/tests/dal.ml +++ b/tezt/tests/dal.ml @@ -5571,7 +5571,7 @@ let test_attestation_through_p2p ~batching_time_interval _protocol module History_rpcs = struct (* In the following function, no migration is performed (expect from genesis to alpha) when [migration_level] is equal to or smaller than 1. *) - let scenario ?(migration_level = 1) ~slot_index ~first_cell_level + let main_scenario ?(migration_level = 1) ~slot_index ~first_cell_level ~first_dal_level ~last_confirmed_published_level protocol dal_parameters client node dal_node = let module Map_int = Map.Make (Int) in @@ -5760,7 +5760,7 @@ module History_rpcs = struct let test_commitments_history_rpcs protocols = let scenario protocol dal_parameters _ client node dal_node = - scenario + main_scenario ~slot_index:3 ~first_cell_level:0 ~first_dal_level:1 @@ -5795,7 +5795,7 @@ module History_rpcs = struct doesn't have the DAL activated. *) (* We'll have 3 levels with a published and attested slot. *) let last_confirmed_published_level = migration_level + 3 in - scenario + main_scenario ~slot_index ~first_cell_level:0 ~first_dal_level:1 -- GitLab From 93c4a96305f086a19432b4c7b693dbcf6d401c7f Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Fri, 10 Oct 2025 18:14:24 +0200 Subject: [PATCH 3/7] Tezt: adapt test 'S023->Alpha: test commitments history with migration' for lag reduction --- tezt/tests/dal.ml | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/tezt/tests/dal.ml b/tezt/tests/dal.ml index e013925ccff4..d3b7d850257a 100644 --- a/tezt/tests/dal.ml +++ b/tezt/tests/dal.ml @@ -5640,7 +5640,32 @@ module History_rpcs = struct end) in let seen_indexes = ref SeenIndexes.empty in let at_least_one_attested_status = ref false in + let rec check_cell cell ~check_level = + let* lag_at_check_level = + let block = Option.map string_of_int check_level in + let* dal_parameters = Dal.Parameters.from_client ?block client in + return dal_parameters.attestation_lag + in + let* lag_at_pred_check_level = + let block = + Option.fold + ~none:None + ~some:(fun level -> + if level <= 1 then None else Some (string_of_int (level - 1))) + check_level + in + let* dal_parameters = Dal.Parameters.from_client ?block client in + return dal_parameters.attestation_lag + in + (* Select the right lag while taking care of the false returned lag in the + migration block (pevious lag used for validation but parameters patched + at block finalization). *) + let lag = + if lag_at_check_level = lag_at_pred_check_level then lag_at_check_level + else lag_at_pred_check_level + in + let skip_list_kind = JSON.(cell |-> "kind" |> as_string) in let expected_skip_list_kind = "dal_skip_list" in Check.( @@ -5654,17 +5679,19 @@ module History_rpcs = struct seen_indexes := SeenIndexes.add cell_index !seen_indexes ; let content = JSON.(skip_list |-> "content") in let cell_level = JSON.(content |-> "level" |> as_int) in + (match check_level with | Some level -> assert (level >= first_dal_level) ; - let expected_level = + let expected_published_level = if level = first_dal_level then (* the "level" of genesis *) 0 else level - lag in Check.( - (cell_level = expected_level) + (cell_level = expected_published_level) int - ~error_msg:"Unexpected cell level: got %L, expected %R") + ~error_msg: + "Unexpected cell's published level: got %L, expected %R") | None -> ()) ; let cell_slot_index = JSON.(content |-> "index" |> as_int) in let () = @@ -12101,9 +12128,9 @@ let tests_start_dal_node_around_migration ~migrate_from ~migrate_to = tests ~migrate_from ~migrate_to ~check_rpc:true let register_migration ~migrate_from ~migrate_to = - test_migration_plugin ~migration_level:4 ~migrate_from ~migrate_to ; + test_migration_plugin ~migration_level:11 ~migrate_from ~migrate_to ; History_rpcs.test_commitments_history_rpcs_with_migration - ~migration_level:10 + ~migration_level:11 ~migrate_from ~migrate_to ; tests_start_dal_node_around_migration ~migrate_from ~migrate_to ; -- GitLab From 6e8a415320e0ee0276e4dc4e186be6a11bbe52a3 Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Thu, 9 Oct 2025 10:51:22 +0200 Subject: [PATCH 4/7] DAL/Proto: Helper function when finalizing slot headers of a published level --- .../lib_protocol/dal_slot_storage.ml | 71 ++++++++++++------- 1 file changed, 44 insertions(+), 27 deletions(-) diff --git a/src/proto_alpha/lib_protocol/dal_slot_storage.ml b/src/proto_alpha/lib_protocol/dal_slot_storage.ml index 0bcaf20a4732..84e60ebf89d4 100644 --- a/src/proto_alpha/lib_protocol/dal_slot_storage.ml +++ b/src/proto_alpha/lib_protocol/dal_slot_storage.ml @@ -111,36 +111,53 @@ let remove_old_headers ctxt ~published_level = | None -> return ctxt | Some level -> Storage.Dal.Slot.Headers.remove ctxt level +(* Finalize DAL slot headers for a given (single) published level: + - Fetch headers published at [published_level]. + - Compute their attestation status via [is_slot_attested]. + - Update the DAL skip-list and storage, mutating [ctxt]: + + Prune old headers in Storage.Dal.Slot.Headers per denunciation window, + + Write the new skip-list head to Storage.Dal.Slot.History, + + Append per-level cells to Storage.Dal.Slot.LevelHistories. + - Return the updated [ctxt] and the attestation bitset. *) +let finalize_slot_headers_for_published_level ctxt ~number_of_slots + ~attestation_lag published_level = + let open Lwt_result_syntax in + let* published_slots = find_slot_headers ctxt published_level in + let*! ctxt = remove_old_headers ctxt ~published_level in + let* ctxt, attestation, slot_headers_statuses = + match published_slots with + | None -> return (ctxt, Dal_attestation_repr.empty, []) + | Some published_slots -> + let slot_headers_statuses, attestation = + let is_slot_attested slot = + Raw_context.Dal.is_slot_index_attested + ctxt + slot.Dal_slot_repr.Header.id.index + in + compute_slot_headers_statuses ~is_slot_attested published_slots + in + return (ctxt, attestation, slot_headers_statuses) + in + let* ctxt = + update_skip_list + ctxt + ~slot_headers_statuses + ~published_level + ~number_of_slots + ~attestation_lag + in + return (ctxt, attestation) + let finalize_pending_slot_headers ctxt ~number_of_slots = let open Lwt_result_syntax in let {Level_repr.level = raw_level; _} = Raw_context.current_level ctxt in let Constants_parametric_repr.{dal; _} = Raw_context.constants ctxt in - let attestation_lag = dal.attestation_lag in - match Raw_level_repr.(sub raw_level attestation_lag) with + let curr_attestation_lag = dal.attestation_lag in + match Raw_level_repr.(sub raw_level curr_attestation_lag) with | None -> return (ctxt, Dal_attestation_repr.empty) | Some published_level -> - let* published_slots = find_slot_headers ctxt published_level in - let*! ctxt = remove_old_headers ctxt ~published_level in - let* ctxt, attestation, slot_headers_statuses = - match published_slots with - | None -> return (ctxt, Dal_attestation_repr.empty, []) - | Some published_slots -> - let slot_headers_statuses, attestation = - let is_slot_attested slot = - Raw_context.Dal.is_slot_index_attested - ctxt - slot.Dal_slot_repr.Header.id.index - in - compute_slot_headers_statuses ~is_slot_attested published_slots - in - return (ctxt, attestation, slot_headers_statuses) - in - let* ctxt = - update_skip_list - ctxt - ~slot_headers_statuses - ~published_level - ~number_of_slots - ~attestation_lag - in - return (ctxt, attestation) + finalize_slot_headers_for_published_level + ctxt + published_level + ~number_of_slots + ~attestation_lag:curr_attestation_lag -- GitLab From 13232bf5b2f23bdf88935f6599d70d2555409906 Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Thu, 9 Oct 2025 10:52:08 +0200 Subject: [PATCH 5/7] DAL/Proto: provide ~is_slot_attested from outside when finalizing a published level --- src/proto_alpha/lib_protocol/dal_slot_storage.ml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/proto_alpha/lib_protocol/dal_slot_storage.ml b/src/proto_alpha/lib_protocol/dal_slot_storage.ml index 84e60ebf89d4..94ceb88321e0 100644 --- a/src/proto_alpha/lib_protocol/dal_slot_storage.ml +++ b/src/proto_alpha/lib_protocol/dal_slot_storage.ml @@ -120,7 +120,7 @@ let remove_old_headers ctxt ~published_level = + Append per-level cells to Storage.Dal.Slot.LevelHistories. - Return the updated [ctxt] and the attestation bitset. *) let finalize_slot_headers_for_published_level ctxt ~number_of_slots - ~attestation_lag published_level = + ~attestation_lag ~is_slot_attested published_level = let open Lwt_result_syntax in let* published_slots = find_slot_headers ctxt published_level in let*! ctxt = remove_old_headers ctxt ~published_level in @@ -129,11 +129,6 @@ let finalize_slot_headers_for_published_level ctxt ~number_of_slots | None -> return (ctxt, Dal_attestation_repr.empty, []) | Some published_slots -> let slot_headers_statuses, attestation = - let is_slot_attested slot = - Raw_context.Dal.is_slot_index_attested - ctxt - slot.Dal_slot_repr.Header.id.index - in compute_slot_headers_statuses ~is_slot_attested published_slots in return (ctxt, attestation, slot_headers_statuses) @@ -156,8 +151,14 @@ let finalize_pending_slot_headers ctxt ~number_of_slots = match Raw_level_repr.(sub raw_level curr_attestation_lag) with | None -> return (ctxt, Dal_attestation_repr.empty) | Some published_level -> + let is_slot_attested slot = + Raw_context.Dal.is_slot_index_attested + ctxt + slot.Dal_slot_repr.Header.id.index + in finalize_slot_headers_for_published_level ctxt published_level ~number_of_slots ~attestation_lag:curr_attestation_lag + ~is_slot_attested -- GitLab From 1762a6b14565f701e6e14841e0f8ff79f720a287 Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Thu, 9 Oct 2025 10:54:57 +0200 Subject: [PATCH 6/7] DAL/Proto: prepare for attestation_lag shrink during published-level finalization --- .../lib_protocol/dal_slot_storage.ml | 70 ++++++++++++++++--- 1 file changed, 60 insertions(+), 10 deletions(-) diff --git a/src/proto_alpha/lib_protocol/dal_slot_storage.ml b/src/proto_alpha/lib_protocol/dal_slot_storage.ml index 94ceb88321e0..6bf96e619abc 100644 --- a/src/proto_alpha/lib_protocol/dal_slot_storage.ml +++ b/src/proto_alpha/lib_protocol/dal_slot_storage.ml @@ -151,14 +151,64 @@ let finalize_pending_slot_headers ctxt ~number_of_slots = match Raw_level_repr.(sub raw_level curr_attestation_lag) with | None -> return (ctxt, Dal_attestation_repr.empty) | Some published_level -> - let is_slot_attested slot = - Raw_context.Dal.is_slot_index_attested - ctxt - slot.Dal_slot_repr.Header.id.index + (* DAL/TODO: remove after P1->P2 migration: + + Detect whether we are at the first block after the lag shrink. We do + this by comparing the current attestation lag read from the protocol + parameters with the attestation lag used for the head of the DAL skip + list. + + Normal case: + + - This is the case when curr_attestation_lag = prev_attestation_lag + + Migration case: + + - This is the case when k = prev_attestation_lag - curr_attestation_lag + > 0. + + => We should process [k + 1] published levels at once to avoid + introducing a gap of [k] levels without cells in the skip list. This + requires updating the context correctly, in particular the + [LevelHistories] entry. *) + let* sl_history_head = get_slot_headers_history ctxt in + let Dal_slot_repr.History. + {header_id = _; attestation_lag = prev_attestation_lag} = + Dal_slot_repr.History.(content sl_history_head |> content_id) + in + let prev_attestation_lag = + Dal_slot_repr.History.attestation_lag_value prev_attestation_lag in - finalize_slot_headers_for_published_level - ctxt - published_level - ~number_of_slots - ~attestation_lag:curr_attestation_lag - ~is_slot_attested + + let reset_dummy_genesis = + Dal_slot_repr.History.(equal sl_history_head genesis) + in + if + Compare.Int.( + curr_attestation_lag = prev_attestation_lag || reset_dummy_genesis) + then + (* Normal path: process the next published level, or the first published + level if the previous genesis cell was the dummy value. *) + let is_slot_attested slot = + Raw_context.Dal.is_slot_index_attested + ctxt + slot.Dal_slot_repr.Header.id.index + in + finalize_slot_headers_for_published_level + ctxt + published_level + ~number_of_slots + ~attestation_lag:curr_attestation_lag + ~is_slot_attested + else + let () = + assert (Compare.Int.(curr_attestation_lag < prev_attestation_lag)) + in + (* Migration path: there are missing published levels between the + skip-list head and [published_level] because attestation_lag has + shrunk at migration from [prev_attestation_lag] to + [curr_attestation_lag]. + + We will backfill the missing [prev_attestation_lag - + curr_attestation_lag] levels with 32 cells each. *) + failwith "TODO: implemented in next commits" -- GitLab From e2f0e33595ac1e89ec4a11106f65b1e3c8e7e332 Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Thu, 16 Oct 2025 11:23:30 +0200 Subject: [PATCH 7/7] DAL/Proto: fill skip-list gap when attestation_lag shrinks --- .../lib_protocol/dal_slot_storage.ml | 90 ++++++++++++++++++- 1 file changed, 88 insertions(+), 2 deletions(-) diff --git a/src/proto_alpha/lib_protocol/dal_slot_storage.ml b/src/proto_alpha/lib_protocol/dal_slot_storage.ml index 6bf96e619abc..d3914d486840 100644 --- a/src/proto_alpha/lib_protocol/dal_slot_storage.ml +++ b/src/proto_alpha/lib_protocol/dal_slot_storage.ml @@ -143,6 +143,88 @@ let finalize_slot_headers_for_published_level ctxt ~number_of_slots in return (ctxt, attestation) +(* Handle the first block after an attestation_lag shrink (P1 -> P2). We + "backfill" the missing published levels so the DAL skip-list has no gaps: + process the [prev_attestation_lag - curr_attestation_lag] intermediate + published levels in order, then the "normal" [target_published_level]. + + The function assumes that prev_attestation_lag > curr_attestation_lag. + + Semantics during backfill: do NOT proto-attest slots. *) +let finalize_slot_headers_at_lag_migration ctxt ~target_published_level + ~number_of_slots ~prev_attestation_lag ~curr_attestation_lag = + let open Lwt_result_syntax in + (* During migration, backfilled published levels must not be proto-attested. + We enforce a conservative attestation status (no proto attest, zero + shards). *) + let is_slot_attested slot = + let status = + Raw_context.Dal.is_slot_index_attested + ctxt + slot.Dal_slot_repr.Header.id.index + in + {status with attested_shards = 0; is_proto_attested = false} + in + (* Process published levels from oldest to newest: + + - Set [current_gap = prev_attestation_lag - curr_attestation_lag] + + - Start at [target_published_level - current_gap]. + + This is equal to [current_level - prev_attestation_lag], since + [target_published_level = current_level - curr_attestation_lag]. + + - Incrementally finalize each published level up to + [target_published_level] by reducing the gap + + - Make sure to use the right (intermediate) attestation_lag for each + processed intermediate level (we'll decrease the lag from + [prev_attestation_lag] to [curr_attestation_lag] one by one). + + Notes: + - [LevelHistories] is overwritten at each finalize; we collect the + per-level cells after each step and batch-write them once at the end. + - Only the final attestation bitset (for [target_published_level]) is + returned for inclusion in the block header. This should not be an issue, as + backfilled levels are non-attesting by design during migration. *) + let rec aux ctxt ~cells_of_pub_levels ~current_gap = + match Raw_level_repr.(sub target_published_level current_gap) with + | None -> + (* Defensive: not expected on our networks. *) + return (ctxt, Dal_attestation_repr.empty, cells_of_pub_levels) + | Some published_level -> + (* Finalize this published level. *) + let* ctxt, attestation_bitset = + finalize_slot_headers_for_published_level + ~attestation_lag:(curr_attestation_lag + current_gap) + ctxt + ~number_of_slots + ~is_slot_attested + published_level + in + (* Collect skip-list cells produced at this step. *) + let* cells_of_pub_levels = + let+ cells_of_this_pub_level = find_level_histories ctxt in + cells_of_this_pub_level :: cells_of_pub_levels + in + if Compare.Int.(current_gap = 0) then + (* Done: [target_published_level] processed. *) + return (ctxt, attestation_bitset, cells_of_pub_levels) + else aux ctxt ~cells_of_pub_levels ~current_gap:(current_gap - 1) + in + (* Main entry for processing several published levels at migration. *) + let current_gap = prev_attestation_lag - curr_attestation_lag in + let* ctxt, attestation_bitset, cells_of_pub_levels = + aux ctxt ~cells_of_pub_levels:[] ~current_gap + in + (* Persist all collected per-level cells in one write. *) + let*! ctxt = + List.(filter_map (fun e -> e) cells_of_pub_levels |> concat) + |> List.rev + |> Storage.Dal.Slot.LevelHistories.add ctxt + in + return (ctxt, attestation_bitset) + let finalize_pending_slot_headers ctxt ~number_of_slots = let open Lwt_result_syntax in let {Level_repr.level = raw_level; _} = Raw_context.current_level ctxt in @@ -179,7 +261,6 @@ let finalize_pending_slot_headers ctxt ~number_of_slots = let prev_attestation_lag = Dal_slot_repr.History.attestation_lag_value prev_attestation_lag in - let reset_dummy_genesis = Dal_slot_repr.History.(equal sl_history_head genesis) in @@ -211,4 +292,9 @@ let finalize_pending_slot_headers ctxt ~number_of_slots = We will backfill the missing [prev_attestation_lag - curr_attestation_lag] levels with 32 cells each. *) - failwith "TODO: implemented in next commits" + finalize_slot_headers_at_lag_migration + ~prev_attestation_lag + ~curr_attestation_lag + ctxt + ~target_published_level:published_level + ~number_of_slots -- GitLab