From ad51c9883c0fa6acbc7dd7bfd2236494f18fc754 Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Wed, 29 Oct 2025 11:15:27 +0100 Subject: [PATCH 1/2] Store/snapshot: minor refactoring --- src/lib_store/unix/snapshots.ml | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index 1792c9f3050d..507ed9a64c0a 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -2402,6 +2402,23 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct in return (export_block, pred_block, minimum_level_needed) + let retrieve_floating_block_list chain_store ~from_block ~to_block = + let open Lwt_result_syntax in + (* Invariant: the list of blocks returned by the + [Store.Chain_traversal.path] contains metadata for the block + target and its predecessor as the metadata were read during the + early steps of the snapshot export and are stored in the + cache. Having metadata for theses blocks is mandatory. *) + let*! o = Store.Chain_traversal.path chain_store ~from_block ~to_block in + match o with + | None -> tzfail Cannot_retrieve_block_interval + | Some floating_blocks -> + (* Don't forget to add the first block as + [Chain_traversal.path] does not include the lower-bound + block *) + let floating_blocks = from_block :: floating_blocks in + return_some floating_blocks + (* Returns the list of cemented files to export and an optional list of remaining blocks. If the export block is cemented, we need to cut the cycle containing the export block accordingly and retrieve the @@ -2412,7 +2429,7 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct let* o = Cemented_block_store.load_table src_cemented_dir in match o with | None -> return ([], None) - | Some table_arr -> ( + | Some table_arr -> let table_len = Array.length table_arr in let table = Array.to_list table_arr in (* Check whether the export_block is in the cemented blocks *) @@ -2467,20 +2484,13 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct Store.Block.read_block_by_level chain_store caboose_level else return first_block_in_cycle in - let*! o = - Store.Chain_traversal.path + let* floating_blocks = + retrieve_floating_block_list chain_store ~from_block:first_block ~to_block:export_block in - match o with - | None -> tzfail Cannot_retrieve_block_interval - | Some floating_blocks -> - (* Don't forget to add the first block as - [Chain_traversal.path] does not include the lower-bound - block *) - let floating_blocks = first_block :: floating_blocks in - return (filtered_table, Some floating_blocks)) + return (filtered_table, floating_blocks) (* Ensures that the history mode requested to export is compatible with the current storage. *) -- GitLab From dca201abbf3c99f8aa7d3441ae6cab49ce3a90dc Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Wed, 29 Oct 2025 11:16:24 +0100 Subject: [PATCH 2/2] Store: Ensure savepoint metadata while exporting from cemented block --- src/lib_store/unix/snapshots.ml | 93 ++++++++++++++++++++++++++++++--- 1 file changed, 85 insertions(+), 8 deletions(-) diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index 507ed9a64c0a..af98b67412f1 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -2405,10 +2405,10 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct let retrieve_floating_block_list chain_store ~from_block ~to_block = let open Lwt_result_syntax in (* Invariant: the list of blocks returned by the - [Store.Chain_traversal.path] contains metadata for the block - target and its predecessor as the metadata were read during the - early steps of the snapshot export and are stored in the - cache. Having metadata for theses blocks is mandatory. *) + [Store.Chain_traversal.path] contains metadata for the block target and + its predecessor as the metadata were read during the early steps of the + snapshot export and are stored in the cache. Having metadata for theses + blocks is mandatory. *) let*! o = Store.Chain_traversal.path chain_store ~from_block ~to_block in match o with | None -> tzfail Cannot_retrieve_block_interval @@ -2463,9 +2463,82 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct let extra_cycle = List.hd extra_cycles |> WithExceptions.Option.get ~loc:__LOC__ in - (* If the export block is the last block in cycle, append the cycle *) + (* If the export block is the last block of the extra_cycle, we need + to put the complete extra_cycle in the floating store to ensure + block metadata. *) if Compare.Int32.(export_block_level = extra_cycle.end_level) then - return (filtered_table @ [extra_cycle], Some []) + let block_store = Store.Unsafe.get_block_store chain_store in + let cemented_store = + Block_store.cemented_block_store block_store + in + let* from_block = + let* res = + Cemented_block_store.get_cemented_block_by_level + ~read_metadata:true + cemented_store + extra_cycle.start_level + in + match res with + | Some block -> Store.Unsafe.block_of_repr block |> return + | None -> tzfail Cannot_retrieve_block_interval + in + let* to_block = + let* res = + Cemented_block_store.get_cemented_block_by_level + ~read_metadata:true + cemented_store + extra_cycle.end_level + in + match res with + | Some block -> Store.Unsafe.block_of_repr block |> return + | None -> tzfail Cannot_retrieve_block_interval + in + let* floating_blocks = + retrieve_floating_block_list chain_store ~from_block ~to_block + in + return (filtered_table, floating_blocks) + else if Compare.Int32.(export_block_level = extra_cycle.start_level) + then + (* When the export level is set to the very first block of a + cycle, it's preceding block (that needs to be exported with + metadata) is stored in the preceding cycle -- without metadata. + To handle that, the two last cycles are stored in the floating + store of the exported snapshot. *) + let block_store = Store.Unsafe.get_block_store chain_store in + let cemented_store = + Block_store.cemented_block_store block_store + in + let last_filtered_cycle, remaining_filtered_cycles = + match List.rev filtered_table with + | hd :: tl -> (hd, tl) + | [] -> (* Invariant: the table is not empty here*) assert false + in + let* from_block = + let* res = + Cemented_block_store.get_cemented_block_by_level + ~read_metadata:true + cemented_store + last_filtered_cycle.start_level + in + match res with + | Some block -> Store.Unsafe.block_of_repr block |> return + | None -> tzfail Cannot_retrieve_block_interval + in + let* to_block = + let* res = + Cemented_block_store.get_cemented_block_by_level + ~read_metadata:true + cemented_store + export_block_level + in + match res with + | Some block -> Store.Unsafe.block_of_repr block |> return + | None -> tzfail Cannot_retrieve_block_interval + in + let* floating_blocks = + retrieve_floating_block_list chain_store ~from_block ~to_block + in + return (remaining_filtered_cycles, floating_blocks) else let* first_block = let* first_block_in_cycle = @@ -2593,8 +2666,12 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct Lwt_stream.of_list (List.filter_map (fun b -> - if Store.Block.level b = export_pred_level then - Some (Store.Unsafe.repr_of_block b) + if Store.Block.level b = export_pred_level then ( + let b = Store.Unsafe.repr_of_block b in + (* Enforce metadata in predecessor's block. The invariant might + be broken in case of Store cache issue. *) + assert (Option.is_some (Block_repr.metadata b)) ; + Some b) else Some {(Store.Unsafe.repr_of_block b) with metadata = None}) floating_blocks) in -- GitLab