diff --git a/CHANGES.rst b/CHANGES.rst index b3aff39c18b302473c6638d3112c2aea095f6f54..38e35a63c6ae4cf9a15638b1dfc80a6a702458a8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -49,6 +49,12 @@ Node operations are now always resolved in favor of the preexisting operation, regardless of contents. (MR :gl:`!18208`) +- ** Breaking change ** Bumped the snapshot version from 8 to 9, to support + early baking, just after importing a snapshot. Snapshots of version 8 exported + with previous versions of Octez (v22) can still be imported. Snapshots of + version 9 are not retro-compatible with previous Octez versions (MR + :gl:`!18387`). + Client ------ diff --git a/src/lib_store/store.mli b/src/lib_store/store.mli index 4bddb419f855b8d133544d30dde75f01bd15ba66..ade68f459c8655a6e5d885a7b19b3d12cf2cb049 100644 --- a/src/lib_store/store.mli +++ b/src/lib_store/store.mli @@ -668,12 +668,11 @@ module Chain : sig invariant holds: [checkpoint.level >= all_head.last_preserved_block_level] - The checkpoint will tend to designate the highest block among - all chain head's [last_preserved_block_level] in a normal - mode. This is not always true. i.e. after a snapshot import - where the checkpoint will be set as the imported block and when - the [target] block is reached, the checkpoint will be set at - this point. *) + Normally, the checkpoint is the block with the highest level among those located + at the [last_preserved_block_level] of each head of the chain. This is not + true after a snapshot import: then, the checkpoint will be set as the + predecessor of the imported block; and when the [target] block is reached, + the checkpoint will be set at this point. *) val checkpoint : chain_store -> block_descriptor Lwt.t (** [target chain_store] returns the target block associated to the diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index baaffd481a857343e89dda118bbe6493def05952..0f98f669b04d4be75f5f50e4b06a22cab08534ef 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -629,12 +629,15 @@ module Version = struct * - 6: new context representation introduced by irmin 3.7.2 * - 7: fix tar snapshots corrupted generation * - 8: change cemented files offset format to 64 bits + * - 9: export target predecessor contains metadata *) + let v8_version = 8 + (* Used for old snapshot format versions *) - let legacy_version = 7 + let legacy_version = v8_version - let current_version = 8 + let current_version = 9 (* List of versions that are supported *) let supported_versions = @@ -2144,6 +2147,7 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct (export_block_descr, (Block_repr.hash first_block, first_block_level))) else let exception Done in + let export_pred_level = Int32.sub (Store.Block.level export_block) 1l in let f block = (* FIXME: we also write potential branches, it will eventually be GCed *) @@ -2151,7 +2155,11 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct if Block_hash.equal limit_hash (Block_repr.hash block) then raise Done else return_unit else - let block = (* Prune everything *) {block with metadata = None} in + let block = + (* Prune everything but the predecessor's metadata *) + if Block_repr.level block = export_pred_level then block + else {block with metadata = None} + in let*! () = bpush#push block in return_unit in @@ -2254,7 +2262,8 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct - the target_block is not the genesis - the target_block and its predecessor are known - the context of the predecessor of the target_block must be known - - at least max_op_ttl(target_block) headers must be available + - at least max_op_ttl(target_block) headers must be available for the + predecessor of the target *) let check_export_block_validity chain_store block = let open Lwt_result_syntax in @@ -2306,8 +2315,8 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct savepoint_level > Int32.pred block_level && not pred_context_exists) (Invalid_export_block {block = Some block_hash; reason = `Pruned_pred}) in - let* block_metadata = - let*! o = Store.Block.get_block_metadata_opt chain_store block in + let* pred_block_metadata = + let*! o = Store.Block.get_block_metadata_opt chain_store pred_block in match o with | None -> tzfail @@ -2316,13 +2325,15 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct in let*! _, caboose_level = Store.Chain.caboose chain_store in (* We will need the following blocks - [ (target_block - max_op_ttl(target_block)) ; ... ; target_block ] *) - let block_max_op_ttl = Store.Block.max_operations_ttl block_metadata in + [ (pred(target_block) - max_op_ttl(target_block)) ; ... ; pred(target_block) ] *) + let block_max_op_ttl = Store.Block.max_operations_ttl pred_block_metadata in let*! genesis_block = Store.Chain.genesis_block chain_store in let genesis_level = Store.Block.level genesis_block in let minimum_level_needed = Compare.Int32.( - max genesis_level Int32.(sub block_level (of_int block_max_op_ttl))) + max + genesis_level + Int32.(sub (sub block_level 1l) (of_int block_max_op_ttl))) in let* () = fail_when @@ -2540,10 +2551,13 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct in (* Prune all blocks except for the export_block's predecessor *) let floating_block_stream = + let export_pred_level = Int32.sub (Store.Block.level export_block) 1l in Lwt_stream.of_list (List.filter_map (fun b -> - Some {(Store.Unsafe.repr_of_block b) with metadata = None}) + if Store.Block.level b = export_pred_level then + Some (Store.Unsafe.repr_of_block b) + else Some {(Store.Unsafe.repr_of_block b) with metadata = None}) floating_blocks) in (* Protocols *) @@ -4224,6 +4238,10 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct | Rolling _ -> return (Rolling None) | Full _ -> return (Full None) in + (* TODO/FIXME: https://gitlab.com/tezos/tezos/-/issues/8005 + remove the v8 import backward compatibility as soon as v9 + (and v23) are mandatory.*) + let is_v8_import = snapshot_version = Version.v8_version in let*! () = Event.(emit restoring_floating_blocks) () in let* () = Animation.display_progress @@ -4243,7 +4261,8 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct validation_store.resulting_context_hash ~predecessor_header:block_data.predecessor_header ~protocol_levels - ~history_mode) + ~history_mode + ~is_v8_import) in let* () = reading_thread in let*! () = Event.(emit floating_blocks_restored) () in @@ -4252,9 +4271,8 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct return_unit end -(* [snapshot_file_kind ~snapshot_path] returns the kind of a - snapshot. We assume that a snapshot is valid if the medata can be - read. *) +(* [snapshot_file_kind ~snapshot_path] returns the kind of a snapshot. We assume + that a snapshot is valid if the metadata can be read. *) let snapshot_file_kind ~snapshot_path = let open Lwt_result_syntax in let is_valid_uncompressed_snapshot file = diff --git a/src/lib_store/unix/store.ml b/src/lib_store/unix/store.ml index 348e424b4684e1946d68e4cf7589bf5c78a9c3dd..3ec47b2a0971a8afe984cbc32a52e7e4ee95b513 100644 --- a/src/lib_store/unix/store.ml +++ b/src/lib_store/unix/store.ml @@ -3743,7 +3743,7 @@ module Unsafe = struct let restore_from_snapshot ?(notify = fun () -> Lwt.return_unit) store_dir ~genesis ~genesis_context_hash ~floating_blocks_stream ~new_head_with_metadata ~new_head_resulting_context_hash - ~predecessor_header ~protocol_levels ~history_mode = + ~predecessor_header ~protocol_levels ~history_mode ~is_v8_import = let open Lwt_result_syntax in let chain_id = Chain_id.of_block_hash genesis.Genesis.block in let chain_dir = Naming.chain_dir store_dir chain_id in @@ -3765,9 +3765,16 @@ module Unsafe = struct (Naming.current_head_file chain_dir) (Block.descriptor new_head_with_metadata) in - (* Checkpoint is the new head *) + (* Checkpoint is the predecessor of the target block as we have both the + associated context and the block's metadata. *) + let new_checkpoint = + if is_v8_import then new_head_descr + else + ( Block_header.hash predecessor_header, + predecessor_header.Block_header.shell.level ) + in let* () = - Stored_data.write_file (Naming.checkpoint_file chain_dir) new_head_descr + Stored_data.write_file (Naming.checkpoint_file chain_dir) new_checkpoint in (* Cementing highwatermark is set to None *) let* () = @@ -3776,9 +3783,16 @@ module Unsafe = struct None in let* () = Stored_data.write_file (Naming.target_file chain_dir) None in - (* Savepoint is the head *) + (* Savepoint is the predecessor of the target block as we have both the + associated context and the block's metadata. *) + let new_savepoint = + if is_v8_import then new_head_descr + else + ( Block_header.hash predecessor_header, + predecessor_header.Block_header.shell.level ) + in let* () = - Stored_data.write_file (Naming.savepoint_file chain_dir) new_head_descr + Stored_data.write_file (Naming.savepoint_file chain_dir) new_savepoint in (* Depending on the history mode, set the caboose properly *) let* caboose_descr = diff --git a/src/lib_store/unix/store.mli b/src/lib_store/unix/store.mli index 36d117727bff8c22dec2c1bfb1ec2f54af754812..605c3f225966a9f432a4544dd6129af56d2cf511 100644 --- a/src/lib_store/unix/store.mli +++ b/src/lib_store/unix/store.mli @@ -670,12 +670,11 @@ module Chain : sig invariant holds: [checkpoint.level >= all_head.last_preserved_block_level] - The checkpoint will tend to designate the highest block among - all chain head's [last_preserved_block_level] in a normal - mode. This is not always true. i.e. after a snapshot import - where the checkpoint will be set as the imported block and when - the [target] block is reached, the checkpoint will be set at - this point. *) + The checkpoint will tend to designate the highest block among all chain + head's [last_preserved_block_level] in a normal mode. This is not always + true. i.e. after a snapshot import where the checkpoint will be set as the + predecessor of the imported block and when the [target] block is reached, + the checkpoint will be set at this point. *) val checkpoint : chain_store -> block_descriptor Lwt.t (** [target chain_store] returns the target block associated to the @@ -1134,12 +1133,12 @@ module Unsafe : sig locked_f:(chain_store -> 'a tzresult Lwt.t) -> 'a tzresult Lwt.t - (** [restore_from_snapshot ?notify ~store_dir ~context_index - ~genesis ~genesis_context_hash ~floating_blocks_stream - ~new_head_with_metadata ~new_head_resulting_context_hash - ~predecessor_header ~protocol_levels ~history_mode] initialises - a coherent store in [store_dir] with all the given info - retrieved from a snapshot. *) + (** [restore_from_snapshot ?notify ~store_dir ~context_index ~genesis + ~genesis_context_hash ~floating_blocks_stream ~new_head_with_metadata + ~new_head_resulting_context_hash ~predecessor_header ~protocol_levels + ~history_mode ~is_legacy_import] initialises a coherent store in + [store_dir] with all the given info retrieved from a snapshot. + [is_v8_import] allows backward compatibility with v8 snapshot. *) val restore_from_snapshot : ?notify:(unit -> unit Lwt.t) -> [`Store_dir] Naming.directory -> @@ -1151,5 +1150,6 @@ module Unsafe : sig predecessor_header:Block_header.t -> protocol_levels:Protocol_levels.protocol_info Protocol_levels.t -> history_mode:History_mode.t -> + is_v8_import:bool -> unit tzresult Lwt.t end diff --git a/src/lib_store/unix/test/test.ml b/src/lib_store/unix/test/test.ml index ddaf0d33496f6e53e802cf8c47614e13d9b4dd56..5989664603765a79889def13e3cdca69a63554f8 100644 --- a/src/lib_store/unix/test/test.ml +++ b/src/lib_store/unix/test/test.ml @@ -27,7 +27,7 @@ _______ Component: Store - Invocation: dune exec src/lib_store/unix/test/main.exe --file test.ml + Invocation: dune exec src/lib_store/unix/test/main.exe -- --file test.ml Subject: Store tests ( snapshots, reconstruct, history_mode_switch ) *) diff --git a/src/lib_store/unix/test/test_snapshots.ml b/src/lib_store/unix/test/test_snapshots.ml index 97b7613614786a96d96be53a04e8d905569738f6..474292e9e4e4d9c8a6ce279f1e1f53589f60f037 100644 --- a/src/lib_store/unix/test/test_snapshots.ml +++ b/src/lib_store/unix/test/test_snapshots.ml @@ -65,7 +65,7 @@ let check_import_invariants ~test_descr ~rolling let expected_present, expected_absent = List.partition (fun b -> - Compare.Int32.(Store.Block.level b <= snd checkpoint) + Compare.Int32.(Store.Block.level b <= Store.Block.level head) && Compare.Int32.(Store.Block.level b >= snd caboose)) previously_baked_blocks in @@ -79,18 +79,26 @@ let check_import_invariants ~test_descr ~rolling (* Check that the descriptors are consistent *) let* expected_caboose_level = if rolling then + let* head_pred = + Store.Block.read_predecessor imported_chain_store head + in (* In rolling: we expected to have at least the max_op_ttl - blocks from the head *) + blocks from the predecessor of the head *) let* metadata = - Store.Block.get_block_metadata imported_chain_store head + Store.Block.get_block_metadata imported_chain_store head_pred in let max_op_ttl = Store.Block.max_operations_ttl metadata in - return Int32.(sub (Store.Block.level head) (of_int max_op_ttl)) + return Int32.(sub (Store.Block.level head_pred) (of_int max_op_ttl)) else return 0l in + (* As the savepoint and checkpoint are targeting the predecessor of the + snapshot's target, we rely on the target's predecessor. *) + let exported_block_predecessor_level = + Int32.sub (Store.Block.level exported_block) 1l + in Assert.equal ~msg:("savepoint consistency: " ^ test_descr) - (Store.Block.level exported_block) + exported_block_predecessor_level (snd savepoint) ; Assert.equal ~msg:("checkpoint consistency: " ^ test_descr) diff --git a/tezt/tests/storage_snapshots.ml b/tezt/tests/storage_snapshots.ml index bf0ef687ea0869a558da91dbc0fc79312f418b6d..48dce27f8632593ca0544aa1445dfe7af8c9127b 100644 --- a/tezt/tests/storage_snapshots.ml +++ b/tezt/tests/storage_snapshots.ml @@ -123,8 +123,13 @@ let check_consistency_after_import node ~expected_head ~expected_checkpoint let check_blocks_availability node ~history_mode ~head ~savepoint ~caboose = (* The metadata of genesis is available anyway *) Log.info "Checking blocks availability for %s" (Node.name node) ; - let* (_ : RPC.block_metadata) = - Node.RPC.call node @@ RPC.get_chain_block_metadata ~block:"0" () + let* () = + if caboose > 0 then unit + else + let* (_ : RPC.block_metadata) = + Node.RPC.call node @@ RPC.get_chain_block_metadata ~block:"0" () + in + unit in let iter_block_range_s a b f = Lwt_list.iter_s f (range a b |> List.rev |> List.map string_of_int) @@ -221,11 +226,17 @@ let export_import_and_check node ~export_level ~history_mode ~export_format node_arguments in let* () = Node.wait_for_ready fresh_node in + (* As the savepoint and checkpoint are targeting the predecessor of the + snapshot's target, we rely on the target's predecessor. *) + let export_level_predecessor = export_level - 1 in let expected_checkpoint, expected_savepoint, expected_caboose = match history_mode with - | Node.Full_history -> (export_level, export_level, 0) + | Node.Full_history -> + (export_level_predecessor, export_level_predecessor, 0) | Node.Rolling_history -> - (export_level, export_level, max 0 (export_level - max_op_ttl)) + ( export_level_predecessor, + export_level_predecessor, + max 0 (export_level_predecessor - max_op_ttl) ) in let* () = check_consistency_after_import @@ -418,8 +429,13 @@ let test_drag_after_rolling_import = let blocks_to_bake = ((blocks_preservation_cycles + additional_cycles) * blocks_per_cycle) - 1 in + (* As the savepoint and checkpoint are targeting the predecessor of the + snapshot's target, we rely on the target's predecessor. *) + let export_level_predecessor = export_level - 1 in let expected_checkpoint, expected_savepoint, expected_caboose = - (export_level, export_level, max 0 (export_level - max_op_ttl)) + ( export_level_predecessor, + export_level_predecessor, + max 0 (export_level_predecessor - max_op_ttl) ) in let* () = check_consistency_after_import @@ -546,7 +562,7 @@ let test_info_command = (* This is expected to be updated as soon as a new snapshot version is released (referring to the Snapshot.Version.current_version from `lib_store/unix/snapshots`)*) - let expected_version = 8 in + let expected_version = 9 in Log.info "Checks the human formatted output" ; (* Get the info line, which is the second line. *) let* () =