From 27945ad14faaf9470c50f8aeb4340f50aa295554 Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Mon, 5 Dec 2022 14:29:47 +0100 Subject: [PATCH 1/9] Store: refactor snapshot's version --- src/lib_store/unix/snapshots.ml | 6 ++---- src/lib_store/unix/snapshots.mli | 3 --- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index fc324c4e693b..eb91e5dab354 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -616,8 +616,6 @@ module Version = struct | Some _ -> return_false end -let current_version = Version.current_version - (* The [default_index_log_size] defines the default maximal value for the log_size, used by the index for storing in memory values, while importing a snapshot. The current limit of 30M aims to target @@ -1387,7 +1385,7 @@ module Raw_exporter : EXPORTER = struct Naming.snapshot_version_file snapshot_tmp_dir |> Naming.file_path in let version_json = - Data_encoding.Json.construct Version.encoding current_version + Data_encoding.Json.construct Version.encoding Version.current_version in let* () = Lwt_utils_unix.Json.write_file version_file version_json in return @@ -1622,7 +1620,7 @@ module Tar_exporter : EXPORTER = struct Naming.snapshot_version_file snapshot_tmp_dir |> Naming.file_path in let version_json = - Data_encoding.Json.construct Version.encoding current_version + Data_encoding.Json.construct Version.encoding Version.current_version in let* () = Lwt_utils_unix.Json.write_file version_file version_json in let*! () = diff --git a/src/lib_store/unix/snapshots.mli b/src/lib_store/unix/snapshots.mli index aa5a9c403715..0ad17bb1481d 100644 --- a/src/lib_store/unix/snapshots.mli +++ b/src/lib_store/unix/snapshots.mli @@ -139,9 +139,6 @@ type error += | Inconsistent_imported_block of Block_hash.t * Block_hash.t | Wrong_snapshot_file of {filename : string} -(** Current version of snapshots *) -val current_version : int - type snapshot_format = Tar | Raw val pp_snapshot_format : Format.formatter -> snapshot_format -> unit -- GitLab From 04145a2a43627c3e5469660f23345181b7103a2a Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Mon, 5 Dec 2022 14:31:43 +0100 Subject: [PATCH 2/9] Store: refactor snapshot's header read --- src/lib_store/unix/snapshots.ml | 70 ++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 28 deletions(-) diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index eb91e5dab354..205527b10e68 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -2750,9 +2750,9 @@ module type IMPORTER = sig snapshot_path:string -> dst_store_dir:[`Store_dir] Naming.directory -> Chain_id.t -> - t Lwt.t + t tzresult Lwt.t - val load_snapshot_header : t -> snapshot_header tzresult Lwt.t + val snapshot_header : t -> snapshot_header val load_block_data : t -> block_data tzresult Lwt.t @@ -2791,6 +2791,8 @@ end module Raw_importer : IMPORTER = struct type t = { + version : Version.t; + metadata : metadata; snapshot_dir : [`Snapshot_dir] Naming.directory; snapshot_cemented_dir : [`Cemented_blocks_dir] Naming.directory; snapshot_protocol_dir : [`Protocol_dir] Naming.directory; @@ -2800,15 +2802,29 @@ module Raw_importer : IMPORTER = struct dst_chain_dir : [`Chain_dir] Naming.directory; } + let load_snapshot_header ~snapshot_path = + let (module Loader) = + (module Make_snapshot_loader (Raw_loader) : Snapshot_loader) + in + Loader.load_snapshot_header ~snapshot_path + + let snapshot_header {version; metadata; _} = (version, metadata) + let init ~snapshot_path ~dst_store_dir chain_id = + let open Lwt_result_syntax in let snapshot_dir = Naming.snapshot_dir ~snapshot_path () in let snapshot_cemented_dir = Naming.cemented_blocks_dir snapshot_dir in let snapshot_protocol_dir = Naming.protocol_store_dir snapshot_dir in let dst_chain_dir = Naming.chain_dir dst_store_dir chain_id in let dst_cemented_dir = Naming.cemented_blocks_dir dst_chain_dir in let dst_protocol_dir = Naming.protocol_store_dir dst_store_dir in - Lwt.return + let* version, metadata = + load_snapshot_header ~snapshot_path:(snapshot_dir |> Naming.(dir_path)) + in + return { + version; + metadata; snapshot_dir; snapshot_cemented_dir; snapshot_protocol_dir; @@ -2818,13 +2834,6 @@ module Raw_importer : IMPORTER = struct dst_chain_dir; } - let load_snapshot_header t = - let (module Loader) = - (module Make_snapshot_loader (Raw_loader) : Snapshot_loader) - in - Loader.load_snapshot_header - ~snapshot_path:Naming.(t.snapshot_dir |> dir_path) - let load_block_data t = let open Lwt_result_syntax in let file = Naming.(snapshot_block_data_file t.snapshot_dir |> file_path) in @@ -3067,6 +3076,8 @@ end module Tar_importer : IMPORTER = struct type t = { + version : Version.t; + metadata : metadata; snapshot_file : [`Snapshot_file] Naming.file; snapshot_tar : [`Tar_archive] Naming.directory; snapshot_cemented_blocks_dir : [`Cemented_blocks_dir] Naming.directory; @@ -3079,8 +3090,16 @@ module Tar_importer : IMPORTER = struct files : Onthefly.file list; } + let load_snapshot_header ~snapshot_path = + let (module Loader) = + (module Make_snapshot_loader (Tar_loader) : Snapshot_loader) + in + Loader.load_snapshot_header ~snapshot_path + + let snapshot_header {version; metadata; _} = (version, metadata) + let init ~snapshot_path ~dst_store_dir chain_id = - let open Lwt_syntax in + let open Lwt_result_syntax in let snapshot_dir = Naming.snapshot_dir ~snapshot_path:(Filename.dirname snapshot_path) () in @@ -3096,10 +3115,15 @@ module Tar_importer : IMPORTER = struct let dst_chain_dir = Naming.chain_dir dst_store_dir chain_id in let dst_cemented_dir = Naming.cemented_blocks_dir dst_chain_dir in let dst_protocol_dir = Naming.protocol_store_dir dst_store_dir in - let* tar = Onthefly.open_in ~file:(Naming.file_path snapshot_file) in - let* files = Onthefly.list_files tar in - Lwt.return + let* version, metadata = + load_snapshot_header ~snapshot_path:(snapshot_file |> Naming.(file_path)) + in + let*! tar = Onthefly.open_in ~file:(Naming.file_path snapshot_file) in + let*! files = Onthefly.list_files tar in + return { + version; + metadata; snapshot_file; snapshot_tar; snapshot_cemented_blocks_dir; @@ -3111,13 +3135,6 @@ module Tar_importer : IMPORTER = struct files; } - let load_snapshot_header t = - let (module Loader) = - (module Make_snapshot_loader (Tar_loader) : Snapshot_loader) - in - Loader.load_snapshot_header - ~snapshot_path:Naming.(t.snapshot_file |> file_path) - let load_block_data t = let open Lwt_result_syntax in let filename = @@ -3392,8 +3409,6 @@ end module type Snapshot_importer = sig type t - val read_snapshot_header : t -> snapshot_header tzresult Lwt.t - val import : snapshot_path:string -> ?patch_context: @@ -3421,8 +3436,6 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct let close = Importer.close - let read_snapshot_header = Importer.load_snapshot_header - let restore_cemented_blocks ?(check_consistency = true) ~dst_chain_dir ~genesis_hash ~progress_display_mode snapshot_importer = let open Lwt_result_syntax in @@ -3694,7 +3707,7 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct ~in_memory ~progress_display_mode (genesis : Genesis.t) = let open Lwt_result_syntax in let chain_id = Chain_id.of_block_hash genesis.Genesis.block in - let*! snapshot_importer = init ~snapshot_path ~dst_store_dir chain_id in + let* snapshot_importer = init ~snapshot_path ~dst_store_dir chain_id in let dst_store_dir = Naming.dir_path dst_store_dir in let* () = fail_when @@ -3722,8 +3735,9 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct (Sys.file_exists snapshot_path) (Snapshot_file_not_found snapshot_path) in - let* snapshot_header = Importer.load_snapshot_header snapshot_importer in - let snapshot_version, snapshot_metadata = snapshot_header in + let ((snapshot_version, snapshot_metadata) as snapshot_header) = + Importer.snapshot_header snapshot_importer + in let* () = fail_unless (Version.is_supported snapshot_version) -- GitLab From 363af6c1849a29581572ce71ae208b0953df4427 Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Tue, 22 Nov 2022 17:02:31 +0100 Subject: [PATCH 3/9] Context/Store: remove legacy (v2/v3) snasphot import support --- src/lib_context/disk/context.ml | 26 +-- src/lib_context/disk/context.mli | 1 - src/lib_context/dump/context_dump.ml | 246 +--------------------- src/lib_context/dump/context_dump_intf.ml | 147 ------------- src/lib_context/test/test_context.ml | 1 - src/lib_store/unix/snapshots.ml | 22 +- 6 files changed, 19 insertions(+), 424 deletions(-) diff --git a/src/lib_context/disk/context.ml b/src/lib_context/disk/context.ml index bc4c72d53899..7db44f898a01 100644 --- a/src/lib_context/disk/context.ml +++ b/src/lib_context/disk/context.ml @@ -120,7 +120,6 @@ module type TEZOS_CONTEXT_UNIX = sig expected_context_hash:Tezos_crypto.Context_hash.t -> nb_context_elements:int -> fd:Lwt_unix.file_descr -> - legacy:bool -> in_memory:bool -> progress_display_mode:Animation.progress_display_mode -> unit tzresult Lwt.t @@ -1131,7 +1130,6 @@ module Make (Encoding : module type of Tezos_context_encoding.Context) = struct open Tezos_context_dump module Context_dumper = Context_dump.Make (Dumpable_context) - module Context_dumper_legacy = Context_dump.Make_legacy (Dumpable_context) (* provides functions dump_context and restore_context *) let dump_context idx data ~fd ~on_disk ~progress_display_mode = @@ -1148,20 +1146,12 @@ module Make (Encoding : module type of Tezos_context_encoding.Context) = struct Lwt.return res let restore_context idx ~expected_context_hash ~nb_context_elements ~fd - ~legacy ~in_memory ~progress_display_mode = - if legacy then - Context_dumper_legacy.restore_context_fd - idx - ~expected_context_hash - ~fd - ~nb_context_elements - ~progress_display_mode - else - Context_dumper.restore_context_fd - idx - ~in_memory - ~expected_context_hash - ~fd - ~nb_context_elements - ~progress_display_mode + ~in_memory ~progress_display_mode = + Context_dumper.restore_context_fd + idx + ~in_memory + ~expected_context_hash + ~fd + ~nb_context_elements + ~progress_display_mode end diff --git a/src/lib_context/disk/context.mli b/src/lib_context/disk/context.mli index fb4e531b64bd..3dcbd32eb6a2 100644 --- a/src/lib_context/disk/context.mli +++ b/src/lib_context/disk/context.mli @@ -64,7 +64,6 @@ module type TEZOS_CONTEXT_UNIX = sig expected_context_hash:Tezos_crypto.Context_hash.t -> nb_context_elements:int -> fd:Lwt_unix.file_descr -> - legacy:bool -> in_memory:bool -> progress_display_mode:Animation.progress_display_mode -> unit tzresult Lwt.t diff --git a/src/lib_context/dump/context_dump.ml b/src/lib_context/dump/context_dump.ml index ea494beabf95..1b7ed7dcd4d7 100644 --- a/src/lib_context/dump/context_dump.ml +++ b/src/lib_context/dump/context_dump.ml @@ -143,194 +143,6 @@ let set_mbytes written context_fd buf b = if Buffer.length buf > 1_000_000 then flush written context_fd buf else Lwt.return_unit -module Make_legacy (I : Dump_interface) = struct - type command = - | Root - | Node_seq of (string * I.Kinded_hash.t, error trace) Seq_es.t - | Blob of bytes - | Eoc of {info : I.commit_info; parents : I.Commit_hash.t list} - | Eof - - (* Command encoding. *) - - let get_char rbuf = - let open Lwt_result_syntax in - let* s = read_string ~len:1 rbuf in - return @@ EndianString.BigEndian.get_int8 s 0 - - let get_int4 rbuf = - let open Lwt_result_syntax in - let* s = read_string ~len:4 rbuf in - return @@ EndianString.BigEndian.get_int32 s 0 - - (* To decode a variable size string we need to: 1/ read the length of the - string, encoded on 4 bytes; 2/ reset the offset to the beginning of the string - encoding. *) - let get_length_and_reset_offset rbuf = - let open Lwt_result_syntax in - let* l = get_int4 rbuf in - let length = Int32.to_int l in - let fd, buf, ofs, total = !rbuf in - rbuf := (fd, buf, ofs - 4, total) ; - return (length + 4) - - let read_variable_length_string rbuf = - let open Lwt_result_syntax in - let* length_name = get_length_and_reset_offset rbuf in - let b = Bytes.create length_name in - let+ () = read_mbytes rbuf b in - let name = Data_encoding.(Binary.of_bytes_exn string) b in - (length_name, name) - - let read_fixed_length_hash rbuf = - let open Lwt_result_syntax in - let length_hash = 1 + 4 + 32 (*enum + size + hash*) in - let b = Bytes.create length_hash in - let+ () = read_mbytes rbuf b in - let hash = Data_encoding.Binary.of_bytes_exn I.Kinded_hash.encoding b in - (length_hash, hash) - - let read_seq rbuf total = - let open Lwt_result_syntax in - let step i = - if i >= total then return_none - else - let* length_name, name = read_variable_length_string rbuf in - let* length_hash, hash = read_fixed_length_hash rbuf in - let node = (name, hash) in - let i = i + length_name + length_hash in - return_some (node, i) - in - Seq_es.unfold_es step 0 - - let eoc_encoding_raw = - let open Data_encoding in - obj2 - (req "info" I.commit_info_encoding) - (req "parents" (list I.Commit_hash.encoding)) - - let get_command rbuf = - let open Lwt_result_syntax in - let* t = get_int64 rbuf in - let total = Int64.to_int t in - let* t = get_char rbuf in - let tag = Char.chr t in - let read_empty () = - let len = total - 1 in - let b = Bytes.create len in - let+ () = read_mbytes rbuf b in - Data_encoding.Binary.of_bytes_exn Data_encoding.empty b - in - match tag with - | 'r' -> - let+ () = read_empty () in - Root - | 'e' -> - let+ () = read_empty () in - Eof - | 'c' -> - let len = total - 1 in - let b = Bytes.create len in - let+ () = read_mbytes rbuf b in - let info, parents = - Data_encoding.Binary.of_bytes_exn eoc_encoding_raw b - in - Eoc {info; parents} - | 'b' -> - let len = total - 1 in - let b = Bytes.create len in - let+ () = read_mbytes rbuf b in - let data = Data_encoding.Binary.of_bytes_exn Data_encoding.bytes b in - Blob data - | 'n' -> - let* s = get_int4 rbuf in - let list_size = Int32.to_int s in - let data = read_seq rbuf list_size in - return (Node_seq data) - | _ -> tzfail Restore_context_failure - - (* Restoring *) - - let restore_context_fd index ~expected_context_hash ~fd ~nb_context_elements - ~progress_display_mode = - let open Lwt_result_syntax in - let read = ref 0 in - let rbuf = ref (fd, Bytes.empty, 0, read) in - (* Editing the repository *) - let add_blob t blob = - let*! tree = I.add_bytes t blob in - return tree - in - let add_dir t keys = - let* r = I.add_dir t keys in - match r with - | None -> tzfail Restore_context_failure - | Some tree -> return tree - in - let restore () = - let first_pass () = - let* r = get_command rbuf in - match r with - | Root -> return_unit - | _ -> tzfail Inconsistent_context_dump - in - let rec second_pass batch ctxt context_hash notify = - let*! () = notify () in - let* c = get_command rbuf in - match c with - | Node_seq contents -> - let* tree = add_dir batch contents in - second_pass batch (I.update_context ctxt tree) context_hash notify - | Blob data -> - let* tree = add_blob batch data in - second_pass batch (I.update_context ctxt tree) context_hash notify - | Eoc {info; parents} -> ( - let*! b = I.set_context ~info ~parents ctxt context_hash in - match b with - | false -> tzfail Inconsistent_context_dump - | true -> return_unit) - | _ -> tzfail Inconsistent_context_dump - in - let check_eof () = - let* e = get_command rbuf in - match e with - | Eof -> return_unit - | _ -> tzfail Inconsistent_context_dump - in - let* block_data = first_pass () in - let* () = - Animation.display_progress - ~every:1000 - ~progress_display_mode - ~pp_print_step:(fun fmt i -> - Format.fprintf - fmt - "Writing context: %dK/%dK (%d%%) elements, %s read" - (i / 1_000) - (nb_context_elements / 1_000) - (100 * i / nb_context_elements) - (if !read > 1_048_576 then - Format.asprintf "%dMiB" (!read / 1_048_576) - else Format.asprintf "%dKiB" (!read / 1_024))) - (fun notify -> - I.batch index (fun batch -> - second_pass - batch - (I.make_context index) - expected_context_hash - notify)) - in - let* () = check_eof () in - return block_data - in - Lwt.catch - (fun () -> restore ()) - (function - | Unix.Unix_error (e, _, _) -> - tzfail @@ System_read_error (Unix.error_message e) - | err -> Lwt.fail err) -end - module Make (I : Dump_interface) = struct type command = | Root @@ -543,7 +355,7 @@ module Make (I : Dump_interface) = struct return_unit | _ -> tzfail Inconsistent_context_dump in - let* block_data = first_pass () in + let* () = first_pass () in let* () = Animation.display_progress ~every:1000 @@ -567,7 +379,7 @@ module Make (I : Dump_interface) = struct notify)) in let* () = check_eof () in - return block_data + return_unit in Lwt.catch (fun () -> restore ()) @@ -576,57 +388,3 @@ module Make (I : Dump_interface) = struct tzfail @@ System_read_error (Unix.error_message e) | err -> Lwt.fail err) end - -(* Legacy errors*) -type error += - | Inconsistent_snapshot_file - | Inconsistent_snapshot_data - | Invalid_snapshot_version of string * string list - -let () = - let open Data_encoding in - register_error_kind - `Permanent - ~id:"context_dump.inconsistent_snapshot_file" - ~title:"Inconsistent snapshot file" - ~description:"Error while opening snapshot file" - ~pp:(fun ppf () -> - Format.fprintf - ppf - "Failed to read snapshot file. The provided file is inconsistent.") - empty - (function Inconsistent_snapshot_file -> Some () | _ -> None) - (fun () -> Inconsistent_snapshot_file) ; - register_error_kind - `Permanent - ~id:"context_dump.inconsistent_snapshot_data" - ~title:"Inconsistent snapshot data" - ~description:"The data provided by the snapshot is inconsistent" - ~pp:(fun ppf () -> - Format.fprintf - ppf - "The data provided by the snapshot file is inconsistent (context_hash \ - does not correspond for block).") - empty - (function Inconsistent_snapshot_data -> Some () | _ -> None) - (fun () -> Inconsistent_snapshot_data) ; - register_error_kind - `Permanent - ~id:"context_dump.invalid_snapshot_version" - ~title:"Invalid snapshot version" - ~description:"The version of the snapshot to import is not valid" - ~pp:(fun ppf (found, expected) -> - Format.fprintf - ppf - "The snapshot to import has version \"%s\" but one of %a was expected." - found - Format.( - pp_print_list - ~pp_sep:(fun ppf () -> fprintf ppf ", ") - (fun ppf version -> fprintf ppf "\"%s\"" version)) - expected) - (obj2 (req "found" string) (req "expected" (list string))) - (function - | Invalid_snapshot_version (found, expected) -> Some (found, expected) - | _ -> None) - (fun (found, expected) -> Invalid_snapshot_version (found, expected)) diff --git a/src/lib_context/dump/context_dump_intf.ml b/src/lib_context/dump/context_dump_intf.ml index 77dd9d436f81..939d1f7e6762 100644 --- a/src/lib_context/dump/context_dump_intf.ml +++ b/src/lib_context/dump/context_dump_intf.ml @@ -216,151 +216,4 @@ module type Context_dump = sig and type context := I.context and type block_header := I.Block_header.t and type context_hash := I.Commit_hash.t - - (* Preserve a legacy importer for now. *) - module Make_legacy (I : Dump_interface) : sig - type index - - type context - - type block_header - - type context_hash - - val restore_context_fd : - index -> - expected_context_hash:context_hash -> - fd:Lwt_unix.file_descr -> - nb_context_elements:int -> - progress_display_mode:Animation.progress_display_mode -> - unit tzresult Lwt.t - end - with type index := I.index - and type context := I.context - and type block_header := I.Block_header.t - and type context_hash := I.Commit_hash.t -end - -(* Legacy *) - -module type Dump_interface_legacy = sig - type index - - type context - - type tree - - type hash - - type contents := bytes - - type step := string - - type commit_info - - type batch - - val batch : index -> (batch -> 'a Lwt.t) -> 'a Lwt.t - - val commit_info_encoding : commit_info Data_encoding.t - - val hash_equal : hash -> hash -> bool - - module Block_header : sig - type t = Block_header.t - - val to_bytes : t -> Bytes.t - - val of_bytes : Bytes.t -> t option - - val equal : t -> t -> bool - - val encoding : t Data_encoding.t - end - - module Pruned_block : sig - type t - - val to_bytes : t -> Bytes.t - - val of_bytes : Bytes.t -> t option - - val header : t -> Block_header.t - - val encoding : t Data_encoding.t - end - - module Block_data : sig - type t - - val to_bytes : t -> Bytes.t - - val of_bytes : Bytes.t -> t option - - val header : t -> Block_header.t - - val encoding : t Data_encoding.t - end - - module Protocol_data : sig - type t - - val to_bytes : t -> Bytes.t - - val of_bytes : Bytes.t -> t option - - val encoding : t Data_encoding.t - - val encoding_1_0_0 : t Data_encoding.t - end - - module Commit_hash : sig - type t - - val to_bytes : t -> Bytes.t - - val of_bytes : Bytes.t -> t tzresult - - val encoding : t Data_encoding.t - end - - module Kinded_hash : Kinded_hash with type hash := hash - - (* commit manipulation (for parents) *) - val context_parents : context -> Commit_hash.t list - - (* Commit info *) - val context_info : context -> commit_info - - (* block header manipulation *) - val get_context : index -> Block_header.t -> context option Lwt.t - - val set_context : - info:commit_info -> - parents:Commit_hash.t list -> - context -> - Block_header.t -> - bool Lwt.t - - (* for dumping *) - val context_tree : context -> tree - - (** Visit each branch and leaf of the given tree {i exactly once}, in - depth-first post-order traversal. Branch children are visited in ascending - key order. Memory usage is linear in the size of the tree. *) - val tree_iteri_unique : - (int -> - [`Branch of (step * Kinded_hash.t) list | `Leaf of contents] -> - unit Lwt.t) -> - tree -> - unit Lwt.t - - (* for restoring *) - val make_context : index -> context - - val update_context : context -> tree -> context - - val add_bytes : batch -> bytes -> tree Lwt.t - - val add_dir : batch -> (step * Kinded_hash.t) list -> tree option Lwt.t end diff --git a/src/lib_context/test/test_context.ml b/src/lib_context/test/test_context.ml index 0fd11c9c1456..05f89711e1e1 100644 --- a/src/lib_context/test/test_context.ml +++ b/src/lib_context/test/test_context.ml @@ -819,7 +819,6 @@ struct ~expected_context_hash:target_context_hash ~nb_context_elements ~fd:context_fd - ~legacy:false ~in_memory:true ~progress_display_mode:Animation.Auto) (fun () -> Lwt_unix.close context_fd)) diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index 205527b10e68..1915f988b7d0 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -551,7 +551,7 @@ let () = Format.fprintf ppf "Failed to read snapshot file %s. The provided file is inconsistent or \ - is from Octez 9.7 (or before) and it cannot be imported anymore." + is from Octez 12 (or before) and it cannot be imported anymore." filename) Data_encoding.(obj1 (req "filename" string)) (function Wrong_snapshot_file {filename} -> Some filename | _ -> None) @@ -593,11 +593,14 @@ module Version = struct * - 3: snapshot exported with storage 0.0.7 * - 4: snapshot exported with storage 0.0.8 to current * - 5: snapshot exported with new protocol tables *) + + let legacy_version = 4 + let current_version = 5 (* List of versions that are supported *) let supported_versions = - [(2, `Legacy); (3, `Legacy); (4, `Current); (5, `Current)] + [(legacy_version, `Legacy); (current_version, `Current)] let is_supported version = match List.assq_opt version supported_versions with @@ -2761,7 +2764,6 @@ module type IMPORTER = sig Context.index -> expected_context_hash:Context_hash.t -> nb_context_elements:int -> - legacy:bool -> in_memory:bool -> progress_display_mode:Animation.progress_display_mode -> unit tzresult Lwt.t @@ -2843,7 +2845,7 @@ module Raw_importer : IMPORTER = struct | None -> tzfail (Cannot_read {kind = `Block_data; path = file}) let restore_context t context_index ~expected_context_hash - ~nb_context_elements ~legacy ~in_memory ~progress_display_mode = + ~nb_context_elements ~in_memory ~progress_display_mode = let open Lwt_result_syntax in let context_file_path = Naming.(snapshot_context_file t.snapshot_dir |> file_path) @@ -2872,7 +2874,6 @@ module Raw_importer : IMPORTER = struct ~expected_context_hash ~fd ~nb_context_elements - ~legacy ~in_memory ~progress_display_mode in @@ -3154,7 +3155,7 @@ module Tar_importer : IMPORTER = struct | None -> tzfail (Cannot_read {kind = `Block_data; path = filename}) let restore_context t context_index ~expected_context_hash - ~nb_context_elements ~legacy ~in_memory ~progress_display_mode = + ~nb_context_elements ~in_memory ~progress_display_mode = let open Lwt_result_syntax in let filename = Naming.(snapshot_context_file t.snapshot_tar |> file_path) in let* header = @@ -3169,7 +3170,6 @@ module Tar_importer : IMPORTER = struct ~expected_context_hash ~nb_context_elements ~fd - ~legacy ~in_memory ~progress_display_mode @@ -3579,7 +3579,7 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct let restore_and_apply_context snapshot_importer protocol_levels ?user_expected_block ~context_index ~user_activated_upgrades - ~user_activated_protocol_overrides ~operation_metadata_size_limit ~legacy + ~user_activated_protocol_overrides ~operation_metadata_size_limit ~in_memory ~progress_display_mode snapshot_metadata genesis chain_id = let open Lwt_result_syntax in (* Start by committing genesis *) @@ -3640,7 +3640,6 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct context_index ~expected_context_hash ~nb_context_elements:snapshot_metadata.context_elements - ~legacy ~in_memory ~progress_display_mode in @@ -3771,8 +3770,6 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct let*! () = import_log_notice ~snapshot_header snapshot_path user_expected_block in - let* legacy = Version.is_legacy snapshot_version in - let indexing_strategy = if legacy then `Always else `Minimal in let patch_context = Option.map (fun f ctxt -> @@ -3785,7 +3782,7 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct let*! context_index = Context.init ~readonly:false - ~indexing_strategy + ~indexing_strategy:`Minimal ~index_log_size:default_index_log_size ?patch_context dst_context_dir @@ -3808,7 +3805,6 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct snapshot_metadata genesis chain_id - ~legacy ~in_memory in (* Restore store *) -- GitLab From b1c411dbb262de557a0a6f1caacf5b94024981c8 Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Mon, 5 Dec 2022 14:56:56 +0100 Subject: [PATCH 4/9] Store/snasphots: cleaner way to handle legacy snapshots --- src/lib_store/unix/snapshots.ml | 126 +++++++++++++++++--------------- 1 file changed, 67 insertions(+), 59 deletions(-) diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index 1915f988b7d0..07ad26005a5f 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -2890,37 +2890,44 @@ module Raw_importer : IMPORTER = struct Naming.(snapshot_protocol_levels_file t.snapshot_dir |> encoded_file_path) in let*! table_bytes = Lwt_utils_unix.read_file protocol_tbl_filename in - match - Data_encoding.Binary.of_string_opt Protocol_levels.encoding table_bytes - with - | Some table -> return table - | None -> ( - (* Try with legacy's encoding *) + let* res = + let* is_legacy = Version.is_legacy t.version in + if is_legacy then + (* Use the legacy encoding *) match Data_encoding.Binary.of_string_opt Protocol_levels.Legacy.encoding table_bytes with | Some table -> - Protocol_levels.Legacy.fold_es - (fun proto_level activation_block map -> - let protocol_info = - { - Protocol_levels.protocol = - activation_block.Protocol_levels.Legacy.protocol; - activation_block = activation_block.block; - (* Only snapshot with legacy semantics will have - legacy encoding. *) - expect_predecessor_context = false; - } - in - return (Protocol_levels.add proto_level protocol_info map)) - table - Protocol_levels.empty - | None -> - tzfail - (Cannot_read - {kind = `Protocol_table; path = protocol_tbl_filename})) + let* res = + Protocol_levels.Legacy.fold_es + (fun proto_level activation_block map -> + let protocol_info = + { + Protocol_levels.protocol = + activation_block.Protocol_levels.Legacy.protocol; + activation_block = activation_block.block; + (* Only snapshot with legacy semantics will have + legacy encoding. *) + expect_predecessor_context = false; + } + in + return (Protocol_levels.add proto_level protocol_info map)) + table + Protocol_levels.empty + in + return_some res + | None -> return_none + else + Data_encoding.Binary.of_string_opt Protocol_levels.encoding table_bytes + |> return + in + match res with + | Some v -> return v + | None -> + tzfail + (Cannot_read {kind = `Protocol_table; path = protocol_tbl_filename}) let load_and_validate_protocol_filenames t = let open Lwt_result_syntax in @@ -3183,44 +3190,45 @@ module Tar_importer : IMPORTER = struct in match o with | Some str -> ( - let res = - Data_encoding.Binary.read_opt - Protocol_levels.encoding - str - 0 - (String.length str) - in - match res with - | Some (_ofs, res) -> return res - | None -> ( - (* Try with legacy's encoding *) + let* res = + let* is_legacy = Version.is_legacy t.version in + if is_legacy then + (* Use the legacy encoding *) match - Data_encoding.Binary.read_opt + Data_encoding.Binary.of_string_opt Protocol_levels.Legacy.encoding str - 0 - (String.length str) with - | Some (_ofs, table) -> - Protocol_levels.Legacy.fold_es - (fun proto_level activation_block map -> - let protocol_info = - { - Protocol_levels.protocol = - activation_block.Protocol_levels.Legacy.protocol; - activation_block = activation_block.block; - (* Only snapshot with legacy semantics will have - legacy encoding. *) - expect_predecessor_context = false; - } - in - return (Protocol_levels.add proto_level protocol_info map)) - table - Protocol_levels.empty - | None -> - tzfail - (Cannot_read - {kind = `Protocol_table; path = protocol_tbl_filename}))) + | Some table -> + let* res = + Protocol_levels.Legacy.fold_es + (fun proto_level activation_block map -> + let protocol_info = + { + Protocol_levels.protocol = + activation_block.Protocol_levels.Legacy.protocol; + activation_block = activation_block.block; + (* Only snapshot with legacy semantics will have + legacy encoding. *) + expect_predecessor_context = false; + } + in + return (Protocol_levels.add proto_level protocol_info map)) + table + Protocol_levels.empty + in + return_some res + | None -> return_none + else + Data_encoding.Binary.of_string_opt Protocol_levels.encoding str + |> return + in + match res with + | Some v -> return v + | None -> + tzfail + (Cannot_read + {kind = `Protocol_table; path = protocol_tbl_filename})) | None -> tzfail (Cannot_read {kind = `Protocol_table; path = protocol_tbl_filename}) -- GitLab From cf16768d89216f1911e04bc5dd048ec0dcd4d822 Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Mon, 5 Dec 2022 16:19:31 +0100 Subject: [PATCH 5/9] Store/snapshot: check resulting context hash at import --- src/lib_store/unix/snapshots.ml | 196 +++++++++++++++++++++++++------- 1 file changed, 154 insertions(+), 42 deletions(-) diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index 07ad26005a5f..72ef50a797d5 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -710,9 +710,59 @@ type block_data = { predecessor_header : Block_header.t; predecessor_block_metadata_hash : Block_metadata_hash.t option; predecessor_ops_metadata_hash : Operation_metadata_list_list_hash.t option; + resulting_context_hash : Context_hash.t; } let block_data_encoding = + let open Data_encoding in + conv + (fun { + block_header; + operations; + predecessor_header; + predecessor_block_metadata_hash; + predecessor_ops_metadata_hash; + resulting_context_hash; + } -> + ( operations, + block_header, + predecessor_header, + predecessor_block_metadata_hash, + predecessor_ops_metadata_hash, + resulting_context_hash )) + (fun ( operations, + block_header, + predecessor_header, + predecessor_block_metadata_hash, + predecessor_ops_metadata_hash, + resulting_context_hash ) -> + { + block_header; + operations; + predecessor_header; + predecessor_block_metadata_hash; + predecessor_ops_metadata_hash; + resulting_context_hash; + }) + (obj6 + (req "operations" (list (list (dynamic_size Operation.encoding)))) + (req "block_header" (dynamic_size Block_header.encoding)) + (req "predecessor_header" (dynamic_size Block_header.encoding)) + (opt "predecessor_block_metadata_hash" Block_metadata_hash.encoding) + (opt + "predecessor_ops_metadata_hash" + Operation_metadata_list_list_hash.encoding) + (req " resulting_context_hash" Context_hash.encoding)) + +type legacy_block_data = { + block_header : Block_header.t; + operations : Operation.t list list; + predecessor_header : Block_header.t; + predecessor_block_metadata_hash : Block_metadata_hash.t option; + predecessor_ops_metadata_hash : Operation_metadata_list_list_hash.t option; +} + +let legacy_block_data_encoding = let open Data_encoding in conv (fun { @@ -1325,6 +1375,7 @@ module type EXPORTER = sig predecessor_block_metadata_hash:Block_metadata_hash.t option -> predecessor_ops_metadata_hash:Operation_metadata_list_list_hash.t option -> export_block:Store.Block.t -> + resulting_context_hash:Context_hash.t -> unit Lwt.t val dump_context : @@ -1400,7 +1451,7 @@ module Raw_exporter : EXPORTER = struct } let write_block_data t ~predecessor_header ~predecessor_block_metadata_hash - ~predecessor_ops_metadata_hash ~export_block = + ~predecessor_ops_metadata_hash ~export_block ~resulting_context_hash = let open Lwt_syntax in let block_data = { @@ -1409,6 +1460,7 @@ module Raw_exporter : EXPORTER = struct predecessor_header; predecessor_block_metadata_hash; predecessor_ops_metadata_hash; + resulting_context_hash; } in let bytes = @@ -1645,7 +1697,7 @@ module Tar_exporter : EXPORTER = struct } let write_block_data t ~predecessor_header ~predecessor_block_metadata_hash - ~predecessor_ops_metadata_hash ~export_block = + ~predecessor_ops_metadata_hash ~export_block ~resulting_context_hash = let block_data = { block_header = Store.Block.header export_block; @@ -1653,6 +1705,7 @@ module Tar_exporter : EXPORTER = struct predecessor_header; predecessor_block_metadata_hash; predecessor_ops_metadata_hash; + resulting_context_hash; } in let bytes = @@ -2340,9 +2393,13 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct let* pred_resulting_context = Store.Block.resulting_context_hash chain_store pred_block in + let* resulting_context_hash = + Store.Block.resulting_context_hash chain_store export_block + in return ( export_mode, export_block, + resulting_context_hash, pred_block, pred_resulting_context, protocol_levels, @@ -2350,6 +2407,7 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct in let* ( export_mode, export_block, + resulting_context_hash, pred_block, pred_resulting_context, protocol_levels, @@ -2363,6 +2421,7 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct return ( export_mode, export_block, + resulting_context_hash, pred_block, pred_resulting_context, protocol_levels, @@ -2430,9 +2489,13 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct let* pred_resulting_context = Store.Block.resulting_context_hash chain_store pred_block in + let* resulting_context_hash = + Store.Block.resulting_context_hash chain_store export_block + in return ( export_mode, export_block, + resulting_context_hash, pred_block, pred_resulting_context, protocol_levels, @@ -2447,6 +2510,7 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct in let* ( export_mode, export_block, + resulting_context_hash, pred_block, pred_resulting_context, protocol_levels, @@ -2494,6 +2558,7 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct return ( export_mode, export_block, + resulting_context_hash, pred_block, pred_resulting_context, protocol_levels, @@ -2527,6 +2592,7 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct protect (fun () -> let* ( export_mode, export_block, + resulting_context_hash, pred_block, pred_resulting_context, protocol_levels, @@ -2563,6 +2629,7 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct ~predecessor_block_metadata_hash ~predecessor_ops_metadata_hash ~export_block + ~resulting_context_hash in let* written_context_elements = Exporter.dump_context @@ -2842,7 +2909,39 @@ module Raw_importer : IMPORTER = struct let*! block_data = Lwt_utils_unix.read_file file in match Data_encoding.Binary.of_string_opt block_data_encoding block_data with | Some block_data -> return block_data - | None -> tzfail (Cannot_read {kind = `Block_data; path = file}) + | None -> ( + let* is_legacy = Version.is_legacy t.version in + let* res = + if is_legacy then + Data_encoding.Binary.of_string_opt + legacy_block_data_encoding + block_data + |> Option.map + (fun + { + block_header; + operations; + predecessor_header; + predecessor_block_metadata_hash; + predecessor_ops_metadata_hash; + } + -> + { + block_header; + operations; + predecessor_header; + predecessor_block_metadata_hash; + predecessor_ops_metadata_hash; + resulting_context_hash = Context_hash.zero; + }) + |> return + else + return + @@ Data_encoding.Binary.of_string_opt block_data_encoding block_data + in + match res with + | Some v -> return v + | None -> tzfail (Cannot_read {kind = `Block_data; path = file})) let restore_context t context_index ~expected_context_hash ~nb_context_elements ~in_memory ~progress_display_mode = @@ -3148,17 +3247,38 @@ module Tar_importer : IMPORTER = struct let filename = Naming.(snapshot_block_data_file t.snapshot_tar |> file_path) in - let* o = - let*! o = Onthefly.load_from_filename t.tar ~filename in - match o with - | Some str -> ( - match Data_encoding.Binary.of_string_opt block_data_encoding str with - | Some metadata -> return_some metadata - | None -> return_none) - | None -> return_none - in + let*! o = Onthefly.load_from_filename t.tar ~filename in match o with - | Some metadata -> return metadata + | Some str -> ( + let* is_legacy = Version.is_legacy t.version in + let* res = + if is_legacy then + Data_encoding.Binary.of_string_opt legacy_block_data_encoding str + |> Option.map + (fun + { + block_header; + operations; + predecessor_header; + predecessor_block_metadata_hash; + predecessor_ops_metadata_hash; + } + -> + { + block_header; + operations; + predecessor_header; + predecessor_block_metadata_hash; + predecessor_ops_metadata_hash; + resulting_context_hash = Context_hash.zero; + }) + |> return + else + return @@ Data_encoding.Binary.of_string_opt block_data_encoding str + in + match res with + | Some v -> return v + | None -> tzfail (Cannot_read {kind = `Block_data; path = filename})) | None -> tzfail (Cannot_read {kind = `Block_data; path = filename}) let restore_context t context_index ~expected_context_hash @@ -3566,23 +3686,21 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct in Event.(emit import_loading ()) - let check_context_hash_consistency ~expect_predecessor_context - validation_store block_header = - (* FIXME: provide the resulting block context in the snapshot and - check against it when the [expect_predecessor_context] - semantics is used. *) + let check_context_hash_consistency ~expected_context_hash validation_store = fail_unless - (expect_predecessor_context - || Context_hash.equal - validation_store - .Tezos_validation.Block_validation.resulting_context_hash - block_header.Block_header.shell.context) + (Context_hash.equal + validation_store.Block_validation.resulting_context_hash + expected_context_hash + || (* This is needed as the former snapshot's version does not + provide enough data to perform this check with the new + context hash semantics (a block header contains the + predecessor's context hash). In that particular case, we + are not able to perform this validity check. *) + Context_hash.equal expected_context_hash Context_hash.zero) (Inconsistent_context_hash { - expected = block_header.Block_header.shell.context; - got = - validation_store - .Tezos_validation.Block_validation.resulting_context_hash; + expected = expected_context_hash; + got = validation_store.Block_validation.resulting_context_hash; }) let restore_and_apply_context snapshot_importer protocol_levels @@ -3600,6 +3718,7 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct in let* ({ block_header; + resulting_context_hash; operations; predecessor_header; predecessor_block_metadata_hash; @@ -3626,7 +3745,7 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct (Inconsistent_imported_block (block_header_hash, snapshot_metadata.block_hash)) in - let expected_context_hash, expect_predecessor_context = + let imported_context_hash = match Protocol_levels.find predecessor_header.shell.proto_level @@ -3635,29 +3754,23 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct | None -> Stdlib.failwith "unknown protocol" | Some {Protocol_levels.expect_predecessor_context; _} -> if expect_predecessor_context then - (block_header.Block_header.shell.context, expect_predecessor_context) - else - ( predecessor_header.Block_header.shell.context, - expect_predecessor_context ) + block_header.Block_header.shell.context + else predecessor_header.Block_header.shell.context in - let predecessor_resulting_context_hash = expected_context_hash in - (* Restore context *) let* () = Importer.restore_context snapshot_importer context_index - ~expected_context_hash + ~expected_context_hash:imported_context_hash ~nb_context_elements:snapshot_metadata.context_elements ~in_memory ~progress_display_mode in let* predecessor_context = - let*! o = - Context.checkout context_index predecessor_resulting_context_hash - in + let*! o = Context.checkout context_index imported_context_hash in match o with | Some ch -> return ch - | None -> tzfail (Inconsistent_context predecessor_resulting_context_hash) + | None -> tzfail (Inconsistent_context imported_context_hash) in let predecessor_context = Tezos_shell_context.Shell_context.wrap_disk_context predecessor_context @@ -3669,7 +3782,7 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct chain_id; predecessor_block_header = predecessor_header; predecessor_context; - predecessor_resulting_context_hash; + predecessor_resulting_context_hash = imported_context_hash; predecessor_block_metadata_hash; predecessor_ops_metadata_hash; user_activated_upgrades; @@ -3699,9 +3812,8 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct in let* () = check_context_hash_consistency - ~expect_predecessor_context + ~expected_context_hash:resulting_context_hash block_validation_result.validation_store - block_header in return (block_data, genesis_ctxt_hash, block_validation_result) -- GitLab From 2acf6fa9b36b9bb41c4b0d273f60df59796c6193 Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Tue, 6 Dec 2022 10:20:05 +0100 Subject: [PATCH 6/9] Store: fix snapshot import invariant --- src/lib_store/unix/snapshots.ml | 1 + src/lib_store/unix/store.ml | 35 +++++++++++++++++++++++++++------ src/lib_store/unix/store.mli | 7 ++++--- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index 72ef50a797d5..03ff2af798b5 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -4007,6 +4007,7 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct ~new_head_with_metadata ~new_head_resulting_context_hash: validation_store.resulting_context_hash + ~predecessor_header:block_data.predecessor_header ~protocol_levels ~history_mode) in diff --git a/src/lib_store/unix/store.ml b/src/lib_store/unix/store.ml index b8577713600a..587e4e2e93b4 100644 --- a/src/lib_store/unix/store.ml +++ b/src/lib_store/unix/store.ml @@ -3295,8 +3295,8 @@ module Unsafe = struct let restore_from_snapshot ?(notify = fun () -> Lwt.return_unit) store_dir ~genesis ~genesis_context_hash ~floating_blocks_stream - ~new_head_with_metadata ~new_head_resulting_context_hash ~protocol_levels - ~history_mode = + ~new_head_with_metadata ~new_head_resulting_context_hash + ~predecessor_header ~protocol_levels ~history_mode = let open Lwt_result_syntax in let chain_id = Chain_id.of_block_hash genesis.Genesis.block in let chain_dir = Naming.chain_dir store_dir chain_id in @@ -3384,13 +3384,36 @@ module Unsafe = struct Block_store.load chain_dir ~genesis_block ~readonly:false in (* Store the floating (in the correct order!) *) - (* No context under the head will be accessible: we do not need - to store consistent resulting_context_hash *) + let predecessor_block_hash = Block_header.hash predecessor_header in let*! () = Lwt_stream.iter_s (fun block -> - let*! _ = - Block_store.store_block block_store block Context_hash.zero + let hash = Block_repr.hash block in + let*! (_ : unit tzresult) = + (* Depending on the block's context hash semantic, we + store the right context hash for the predecessor of + the imported block so that the invariant which states + that the savepoint can be the target of a snapshot + export is maintained. This is needed only when the + block contains the resulting context hash. + For other blocks, and as their contexts won't be + accessible, we do not need to store consistent + resulting context hash. *) + if Block_hash.equal predecessor_block_hash hash then + let predecessor_context_hash = + match + Protocol_levels.find + predecessor_header.shell.proto_level + protocol_levels + with + | None -> Stdlib.failwith "unknown protocol" + | Some {Protocol_levels.expect_predecessor_context; _} -> + if expect_predecessor_context then + Block_repr.context new_head_with_metadata + else predecessor_header.Block_header.shell.context + in + Block_store.store_block block_store block predecessor_context_hash + else Block_store.store_block block_store block Context_hash.zero in notify ()) floating_blocks_stream diff --git a/src/lib_store/unix/store.mli b/src/lib_store/unix/store.mli index 342e54f53aeb..0586c2ebd98c 100644 --- a/src/lib_store/unix/store.mli +++ b/src/lib_store/unix/store.mli @@ -1102,9 +1102,9 @@ module Unsafe : sig (** [restore_from_snapshot ?notify ~store_dir ~context_index ~genesis ~genesis_context_hash ~floating_blocks_stream ~new_head_with_metadata ~new_head_resulting_context_hash - ~protocol_levels ~history_mode] initialises a coherent store in - [store_dir] with all the given info retrieved from a - snapshot. *) + ~predecessor_header ~protocol_levels ~history_mode] initialises + a coherent store in [store_dir] with all the given info + retrieved from a snapshot. *) val restore_from_snapshot : ?notify:(unit -> unit Lwt.t) -> [`Store_dir] Naming.directory -> @@ -1113,6 +1113,7 @@ module Unsafe : sig floating_blocks_stream:Block_repr.block Lwt_stream.t -> new_head_with_metadata:Block_repr.block -> new_head_resulting_context_hash:Context_hash.t -> + predecessor_header:Block_header.t -> protocol_levels:Protocol_levels.protocol_info Protocol_levels.t -> history_mode:History_mode.t -> unit tzresult Lwt.t -- GitLab From b5b7b3ca616b27cb963197695c40c9bd2cfeb280 Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Tue, 6 Dec 2022 09:22:04 +0100 Subject: [PATCH 7/9] Changelog: remove snapshots v2 and v3 compatibility --- CHANGES.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index c9a7636971e1..91caa65578dd 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -79,6 +79,10 @@ Node of version ``4`` exported with previous versions of Octez can still be imported. Snapshots of version ``5`` are not backward compatible. +- Removed the compatibility with storage snapshots of version ``2`` + and ``3``. These snapshot versions from Octez 12 cannot be imported + anymore. + Client ------ -- GitLab From 12a9d2b276e90deedb0988ac456ffe5a21fb6d39 Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Wed, 7 Dec 2022 11:01:12 +0100 Subject: [PATCH 8/9] Store/snapshots: fix pred context existance check --- src/lib_store/unix/snapshots.ml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index 03ff2af798b5..bc84188790ec 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -2156,8 +2156,17 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct | Some pred_block -> return pred_block in (* Make sure that the predecessor's context is known *) - let*! pred_context_exists = - Store.Block.context_exists chain_store pred_block + let* pred_context_exists = + let protocol_level = Store.Block.proto_level block in + let* expect_predecessor_context_hash = + Store.Chain.expect_predecessor_context_hash chain_store ~protocol_level + in + let*! exists = + if expect_predecessor_context_hash then + Store.Block.context_exists chain_store block + else Store.Block.context_exists chain_store pred_block + in + return exists in (* We also need the predecessor not to be pruned *) let* () = -- GitLab From 0b7757eac1f683ae3e8832597e366b5b8bcbdf65 Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Wed, 7 Dec 2022 11:01:50 +0100 Subject: [PATCH 9/9] Tezt: add a test to export a freshly imported snapshot --- tezt/tests/multinode_snapshot.ml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tezt/tests/multinode_snapshot.ml b/tezt/tests/multinode_snapshot.ml index 1003d3ec8d7d..36728f7cb023 100644 --- a/tezt/tests/multinode_snapshot.ml +++ b/tezt/tests/multinode_snapshot.ml @@ -374,6 +374,23 @@ let test_storage_snapshot = (* ########################################################################### *) Log.info "Check consistency 1" ; + (* Make sure that a snapshot freshly imported, that has its + savepoint set on the head, can export a valid snapshot. *) + let* () = + let node = node_rolling in + Log.info + "Check fresh import can be exported (node_rolling, %s)." + (Node.name node) ; + let* () = + export + node + Rolling_history + ~export_level:snapshot_level + ~snapshot:"from_fresh_import.rolling" + in + unit + in + (* Check consistency of imported snapshots Do not factorize calls to ease debugging -- GitLab