diff --git a/src/proto_alpha/lib_protocol/dal_slot_storage.ml b/src/proto_alpha/lib_protocol/dal_slot_storage.ml index 0bcaf20a47321d89c90be770b765035909c553c7..d3914d486840640fdb576c741d2b22ed10e3c450 100644 --- a/src/proto_alpha/lib_protocol/dal_slot_storage.ml +++ b/src/proto_alpha/lib_protocol/dal_slot_storage.ml @@ -111,36 +111,190 @@ let remove_old_headers ctxt ~published_level = | None -> return ctxt | Some level -> Storage.Dal.Slot.Headers.remove ctxt level +(* Finalize DAL slot headers for a given (single) published level: + - Fetch headers published at [published_level]. + - Compute their attestation status via [is_slot_attested]. + - Update the DAL skip-list and storage, mutating [ctxt]: + + Prune old headers in Storage.Dal.Slot.Headers per denunciation window, + + Write the new skip-list head to Storage.Dal.Slot.History, + + Append per-level cells to Storage.Dal.Slot.LevelHistories. + - Return the updated [ctxt] and the attestation bitset. *) +let finalize_slot_headers_for_published_level ctxt ~number_of_slots + ~attestation_lag ~is_slot_attested published_level = + let open Lwt_result_syntax in + let* published_slots = find_slot_headers ctxt published_level in + let*! ctxt = remove_old_headers ctxt ~published_level in + let* ctxt, attestation, slot_headers_statuses = + match published_slots with + | None -> return (ctxt, Dal_attestation_repr.empty, []) + | Some published_slots -> + let slot_headers_statuses, attestation = + compute_slot_headers_statuses ~is_slot_attested published_slots + in + return (ctxt, attestation, slot_headers_statuses) + in + let* ctxt = + update_skip_list + ctxt + ~slot_headers_statuses + ~published_level + ~number_of_slots + ~attestation_lag + in + return (ctxt, attestation) + +(* Handle the first block after an attestation_lag shrink (P1 -> P2). We + "backfill" the missing published levels so the DAL skip-list has no gaps: + process the [prev_attestation_lag - curr_attestation_lag] intermediate + published levels in order, then the "normal" [target_published_level]. + + The function assumes that prev_attestation_lag > curr_attestation_lag. + + Semantics during backfill: do NOT proto-attest slots. *) +let finalize_slot_headers_at_lag_migration ctxt ~target_published_level + ~number_of_slots ~prev_attestation_lag ~curr_attestation_lag = + let open Lwt_result_syntax in + (* During migration, backfilled published levels must not be proto-attested. + We enforce a conservative attestation status (no proto attest, zero + shards). *) + let is_slot_attested slot = + let status = + Raw_context.Dal.is_slot_index_attested + ctxt + slot.Dal_slot_repr.Header.id.index + in + {status with attested_shards = 0; is_proto_attested = false} + in + (* Process published levels from oldest to newest: + + - Set [current_gap = prev_attestation_lag - curr_attestation_lag] + + - Start at [target_published_level - current_gap]. + + This is equal to [current_level - prev_attestation_lag], since + [target_published_level = current_level - curr_attestation_lag]. + + - Incrementally finalize each published level up to + [target_published_level] by reducing the gap + + - Make sure to use the right (intermediate) attestation_lag for each + processed intermediate level (we'll decrease the lag from + [prev_attestation_lag] to [curr_attestation_lag] one by one). + + Notes: + - [LevelHistories] is overwritten at each finalize; we collect the + per-level cells after each step and batch-write them once at the end. + - Only the final attestation bitset (for [target_published_level]) is + returned for inclusion in the block header. This should not be an issue, as + backfilled levels are non-attesting by design during migration. *) + let rec aux ctxt ~cells_of_pub_levels ~current_gap = + match Raw_level_repr.(sub target_published_level current_gap) with + | None -> + (* Defensive: not expected on our networks. *) + return (ctxt, Dal_attestation_repr.empty, cells_of_pub_levels) + | Some published_level -> + (* Finalize this published level. *) + let* ctxt, attestation_bitset = + finalize_slot_headers_for_published_level + ~attestation_lag:(curr_attestation_lag + current_gap) + ctxt + ~number_of_slots + ~is_slot_attested + published_level + in + (* Collect skip-list cells produced at this step. *) + let* cells_of_pub_levels = + let+ cells_of_this_pub_level = find_level_histories ctxt in + cells_of_this_pub_level :: cells_of_pub_levels + in + if Compare.Int.(current_gap = 0) then + (* Done: [target_published_level] processed. *) + return (ctxt, attestation_bitset, cells_of_pub_levels) + else aux ctxt ~cells_of_pub_levels ~current_gap:(current_gap - 1) + in + (* Main entry for processing several published levels at migration. *) + let current_gap = prev_attestation_lag - curr_attestation_lag in + let* ctxt, attestation_bitset, cells_of_pub_levels = + aux ctxt ~cells_of_pub_levels:[] ~current_gap + in + (* Persist all collected per-level cells in one write. *) + let*! ctxt = + List.(filter_map (fun e -> e) cells_of_pub_levels |> concat) + |> List.rev + |> Storage.Dal.Slot.LevelHistories.add ctxt + in + return (ctxt, attestation_bitset) + let finalize_pending_slot_headers ctxt ~number_of_slots = let open Lwt_result_syntax in let {Level_repr.level = raw_level; _} = Raw_context.current_level ctxt in let Constants_parametric_repr.{dal; _} = Raw_context.constants ctxt in - let attestation_lag = dal.attestation_lag in - match Raw_level_repr.(sub raw_level attestation_lag) with + let curr_attestation_lag = dal.attestation_lag in + match Raw_level_repr.(sub raw_level curr_attestation_lag) with | None -> return (ctxt, Dal_attestation_repr.empty) | Some published_level -> - let* published_slots = find_slot_headers ctxt published_level in - let*! ctxt = remove_old_headers ctxt ~published_level in - let* ctxt, attestation, slot_headers_statuses = - match published_slots with - | None -> return (ctxt, Dal_attestation_repr.empty, []) - | Some published_slots -> - let slot_headers_statuses, attestation = - let is_slot_attested slot = - Raw_context.Dal.is_slot_index_attested - ctxt - slot.Dal_slot_repr.Header.id.index - in - compute_slot_headers_statuses ~is_slot_attested published_slots - in - return (ctxt, attestation, slot_headers_statuses) + (* DAL/TODO: remove after P1->P2 migration: + + Detect whether we are at the first block after the lag shrink. We do + this by comparing the current attestation lag read from the protocol + parameters with the attestation lag used for the head of the DAL skip + list. + + Normal case: + + - This is the case when curr_attestation_lag = prev_attestation_lag + + Migration case: + + - This is the case when k = prev_attestation_lag - curr_attestation_lag + > 0. + + => We should process [k + 1] published levels at once to avoid + introducing a gap of [k] levels without cells in the skip list. This + requires updating the context correctly, in particular the + [LevelHistories] entry. *) + let* sl_history_head = get_slot_headers_history ctxt in + let Dal_slot_repr.History. + {header_id = _; attestation_lag = prev_attestation_lag} = + Dal_slot_repr.History.(content sl_history_head |> content_id) + in + let prev_attestation_lag = + Dal_slot_repr.History.attestation_lag_value prev_attestation_lag in - let* ctxt = - update_skip_list + let reset_dummy_genesis = + Dal_slot_repr.History.(equal sl_history_head genesis) + in + if + Compare.Int.( + curr_attestation_lag = prev_attestation_lag || reset_dummy_genesis) + then + (* Normal path: process the next published level, or the first published + level if the previous genesis cell was the dummy value. *) + let is_slot_attested slot = + Raw_context.Dal.is_slot_index_attested + ctxt + slot.Dal_slot_repr.Header.id.index + in + finalize_slot_headers_for_published_level ctxt - ~slot_headers_statuses - ~published_level + published_level + ~number_of_slots + ~attestation_lag:curr_attestation_lag + ~is_slot_attested + else + let () = + assert (Compare.Int.(curr_attestation_lag < prev_attestation_lag)) + in + (* Migration path: there are missing published levels between the + skip-list head and [published_level] because attestation_lag has + shrunk at migration from [prev_attestation_lag] to + [curr_attestation_lag]. + + We will backfill the missing [prev_attestation_lag - + curr_attestation_lag] levels with 32 cells each. *) + finalize_slot_headers_at_lag_migration + ~prev_attestation_lag + ~curr_attestation_lag + ctxt + ~target_published_level:published_level ~number_of_slots - ~attestation_lag - in - return (ctxt, attestation) diff --git a/tezt/lib_tezos/dal_common.ml b/tezt/lib_tezos/dal_common.ml index 49c88f9065966831a23da330859fa8234cb07e52..5fba53a195d7dceccf093d72982b08af58ba7b01 100644 --- a/tezt/lib_tezos/dal_common.ml +++ b/tezt/lib_tezos/dal_common.ml @@ -68,10 +68,10 @@ module Parameters = struct attestation_threshold; } - let from_client client = + let from_client ?block client = let* json = Client.RPC.call_via_endpoint client - @@ RPC.get_chain_block_context_constants () + @@ RPC.get_chain_block_context_constants ?block () in from_protocol_parameters json |> return diff --git a/tezt/lib_tezos/dal_common.mli b/tezt/lib_tezos/dal_common.mli index ee0105fb5eb398eb88f1748002caece386d21441..1dbd7da96d4a67f8c02c385250a0c69b76962f07 100644 --- a/tezt/lib_tezos/dal_common.mli +++ b/tezt/lib_tezos/dal_common.mli @@ -40,7 +40,7 @@ module Parameters : sig val from_protocol_parameters : JSON.t -> t - val from_client : Client.t -> t Lwt.t + val from_client : ?block:string -> Client.t -> t Lwt.t val from_endpoint : Endpoint.t -> t Lwt.t diff --git a/tezt/tests/dal.ml b/tezt/tests/dal.ml index 3942d158519edb5a383474600c026f5f876c80a0..d3b7d850257a4aa471ef1d7b14d27f4816c2c7e1 100644 --- a/tezt/tests/dal.ml +++ b/tezt/tests/dal.ml @@ -5571,7 +5571,7 @@ let test_attestation_through_p2p ~batching_time_interval _protocol module History_rpcs = struct (* In the following function, no migration is performed (expect from genesis to alpha) when [migration_level] is equal to or smaller than 1. *) - let scenario ?(migration_level = 1) ~slot_index ~first_cell_level + let main_scenario ?(migration_level = 1) ~slot_index ~first_cell_level ~first_dal_level ~last_confirmed_published_level protocol dal_parameters client node dal_node = let module Map_int = Map.Make (Int) in @@ -5640,7 +5640,32 @@ module History_rpcs = struct end) in let seen_indexes = ref SeenIndexes.empty in let at_least_one_attested_status = ref false in + let rec check_cell cell ~check_level = + let* lag_at_check_level = + let block = Option.map string_of_int check_level in + let* dal_parameters = Dal.Parameters.from_client ?block client in + return dal_parameters.attestation_lag + in + let* lag_at_pred_check_level = + let block = + Option.fold + ~none:None + ~some:(fun level -> + if level <= 1 then None else Some (string_of_int (level - 1))) + check_level + in + let* dal_parameters = Dal.Parameters.from_client ?block client in + return dal_parameters.attestation_lag + in + (* Select the right lag while taking care of the false returned lag in the + migration block (pevious lag used for validation but parameters patched + at block finalization). *) + let lag = + if lag_at_check_level = lag_at_pred_check_level then lag_at_check_level + else lag_at_pred_check_level + in + let skip_list_kind = JSON.(cell |-> "kind" |> as_string) in let expected_skip_list_kind = "dal_skip_list" in Check.( @@ -5654,17 +5679,19 @@ module History_rpcs = struct seen_indexes := SeenIndexes.add cell_index !seen_indexes ; let content = JSON.(skip_list |-> "content") in let cell_level = JSON.(content |-> "level" |> as_int) in + (match check_level with | Some level -> assert (level >= first_dal_level) ; - let expected_level = + let expected_published_level = if level = first_dal_level then (* the "level" of genesis *) 0 else level - lag in Check.( - (cell_level = expected_level) + (cell_level = expected_published_level) int - ~error_msg:"Unexpected cell level: got %L, expected %R") + ~error_msg: + "Unexpected cell's published level: got %L, expected %R") | None -> ()) ; let cell_slot_index = JSON.(content |-> "index" |> as_int) in let () = @@ -5760,7 +5787,7 @@ module History_rpcs = struct let test_commitments_history_rpcs protocols = let scenario protocol dal_parameters _ client node dal_node = - scenario + main_scenario ~slot_index:3 ~first_cell_level:0 ~first_dal_level:1 @@ -5795,7 +5822,7 @@ module History_rpcs = struct doesn't have the DAL activated. *) (* We'll have 3 levels with a published and attested slot. *) let last_confirmed_published_level = migration_level + 3 in - scenario + main_scenario ~slot_index ~first_cell_level:0 ~first_dal_level:1 @@ -12101,9 +12128,9 @@ let tests_start_dal_node_around_migration ~migrate_from ~migrate_to = tests ~migrate_from ~migrate_to ~check_rpc:true let register_migration ~migrate_from ~migrate_to = - test_migration_plugin ~migration_level:4 ~migrate_from ~migrate_to ; + test_migration_plugin ~migration_level:11 ~migrate_from ~migrate_to ; History_rpcs.test_commitments_history_rpcs_with_migration - ~migration_level:10 + ~migration_level:11 ~migrate_from ~migrate_to ; tests_start_dal_node_around_migration ~migrate_from ~migrate_to ;