From 43395fbd39c299452c126332e0709f2f40702eb9 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Thu, 1 Aug 2024 14:23:55 +0100 Subject: [PATCH 1/8] Store: Expose the get_and_upgrade_offsets function --- src/lib_store/unix/cemented_block_store.ml | 3 --- src/lib_store/unix/cemented_block_store.mli | 5 +++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/lib_store/unix/cemented_block_store.ml b/src/lib_store/unix/cemented_block_store.ml index d2db03e8c90a..2f207655371f 100644 --- a/src/lib_store/unix/cemented_block_store.ml +++ b/src/lib_store/unix/cemented_block_store.ml @@ -1079,9 +1079,6 @@ let check_indexes_consistency ?(post_step = fun () -> Lwt.return_unit) in return_unit -(** [get_and_upgrade_offsets fd nb_blocks] obtains the list of [nb_blocks] offsets from the - beginning of the file given by file descriptor [fd] and upgrades their sizes from 32-bits to - 64-bits, adjusting their values for the following blocks in the cemented blocks file. *) let get_and_upgrade_offsets fd nb_blocks = let open Lwt_syntax in (* Obtain the list of offsets (32-bit format) *) diff --git a/src/lib_store/unix/cemented_block_store.mli b/src/lib_store/unix/cemented_block_store.mli index 47211a4e389b..1773bba1a936 100644 --- a/src/lib_store/unix/cemented_block_store.mli +++ b/src/lib_store/unix/cemented_block_store.mli @@ -310,6 +310,11 @@ val check_indexes_consistency : t -> unit tzresult Lwt.t +(** [get_and_upgrade_offsets fd nb_blocks] obtains the list of [nb_blocks] offsets from the + beginning of the file given by file descriptor [fd] and upgrades their sizes from 32-bits to + 64-bits, adjusting their values for the following blocks in the cemented blocks file. *) +val get_and_upgrade_offsets : Lwt_unix.file_descr -> int -> bytes Lwt.t + (* Given the chain directory with cemented files, upgrade all of them to have 64-bit offsets at the beginning, instead of 32-bits. *) val v_3_2_upgrade : [`Chain_dir] Naming.directory -> unit tzresult Lwt.t -- GitLab From 7f2845517014182f3f43d8cbbeffc767867a302d Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Thu, 1 Aug 2024 14:26:08 +0100 Subject: [PATCH 2/8] Store: Increase snapshot version --- src/lib_store/unix/snapshots.ml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index 9d60bbed768e..18a3701101da 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -597,16 +597,17 @@ module Version = struct context (storage 2.0) * - 6: new context representation introduced by irmin 3.7.2 * - 7: fix tar snapshots corrupted generation + * - 8: change cemented files offset format to 64 bits *) (* Used for old snapshot format versions *) - (* Keeping this for future use. *) - let _legacy_version = 4 + let legacy_version = 7 - let current_version = 7 + let current_version = 8 (* List of versions that are supported *) - let supported_versions = [(current_version, `Current)] + let supported_versions = + [(legacy_version, `Legacy_format); (current_version, `Current)] let is_supported version = match List.assq_opt version supported_versions with -- GitLab From a665426115a5b5674ca886d4ea0866246631dcdf Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Thu, 1 Aug 2024 14:27:40 +0100 Subject: [PATCH 3/8] Store: Upgrade copy mechanism for Tar snapshot import --- src/lib_store/unix/snapshots.ml | 80 ++++++++++++++++++++++++++++----- 1 file changed, 68 insertions(+), 12 deletions(-) diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index 18a3701101da..8cbe1df8574c 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -615,8 +615,7 @@ module Version = struct | None -> false (* Returns true if the given version uses a legacy data format. *) - (* Keeping this for future use. *) - let _is_legacy_format version = + let is_legacy_format version = let open Lwt_result_syntax in match List.assq_opt version supported_versions with | None -> @@ -972,6 +971,13 @@ module Onthefly : sig (* [copy_to_file tar file ~dst] copies the [file] from the [tar] into new file designated by [dst]. *) val copy_to_file : i -> file -> dst:string -> unit Lwt.t + + (* [copy_and_upgrade_legacy_file tar filename ~dst ~nb_blocks] loads + the file [filename] from the given [tar], upgrades the [~nb_blocks] offset + bytes from a 32 to a 64-bit format (in case the snapshot uses a legacy + format) and then it copies the rest of the file in the [~dst] file. *) + val copy_and_upgrade_legacy_file : + i -> file -> dst:string -> nb_blocks:int -> unit Lwt.t end = struct include Tar @@ -1396,6 +1402,31 @@ end = struct Lwt.finalize (fun () -> copy_n tar.fd fd header.Tar.Header.file_size) (fun () -> Lwt_unix.close fd) + + let copy_and_upgrade_legacy_file tar {header; data_ofs} ~dst ~nb_blocks = + let open Lwt_syntax in + let* _ = Lwt_unix.LargeFile.lseek tar.fd data_ofs SEEK_SET in + let* upgraded_fd = + Lwt_unix.openfile + dst + Unix.[O_WRONLY; O_CREAT; O_TRUNC] + snapshot_rw_file_perm + in + Lwt.finalize + (fun () -> + (* Should use the Cemented_block_store function - need to expose after merging MR (?) *) + let* bytes = + Cemented_block_store.get_and_upgrade_offsets tar.fd nb_blocks + in + (* Write the 64-bit offsets list into the new file *) + let block = Cstruct.of_bytes bytes in + let* () = Writer.really_write upgraded_fd block in + (* Copy the rest of the block file containing INITIAL_SIZE - 4 * NB_BLOCKS bytes. *) + let remaining_size = + Int64.(sub header.Tar.Header.file_size (of_int (4 * nb_blocks))) + in + copy_n tar.fd upgraded_fd remaining_size) + (fun () -> Lwt_unix.close upgraded_fd) end module type EXPORTER = sig @@ -3462,16 +3493,41 @@ module Tar_importer : IMPORTER = struct | Some file -> return file | None -> tzfail (Cannot_read {kind = `Cemented_cycle; path = filename}) in - let*! () = - Onthefly.copy_to_file - t.tar - tar_file - ~dst: - (Filename.concat - (Naming.dir_path t.dst_cemented_dir) - (Filename.basename file)) - in - return_unit + let* is_legacy_format = Version.is_legacy_format t.version in + if is_legacy_format then + (* Need to upgrade the offset format of cemented files from 32 to 64 bits. *) + let* nb_blocks = + match String.split_on_char '_' file with + | [x_str; y_str] -> + let x = int_of_string x_str in + let y = int_of_string y_str in + return (y - x + 1) + | _ -> + (* Invalid cemented blocks file *) + failwith "Wrong filename format: %s" file + in + let*! () = + Onthefly.copy_and_upgrade_legacy_file + t.tar + tar_file + ~dst: + (Filename.concat + (Naming.dir_path t.dst_cemented_dir) + (Filename.basename file)) + ~nb_blocks + in + return_unit + else + let*! () = + Onthefly.copy_to_file + t.tar + tar_file + ~dst: + (Filename.concat + (Naming.dir_path t.dst_cemented_dir) + (Filename.basename file)) + in + return_unit let restore_floating_blocks t genesis_hash = let open Lwt_result_syntax in -- GitLab From 2f09e1e846b4d326d9b5d97e23658f4c00d51d97 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Thu, 1 Aug 2024 14:28:02 +0100 Subject: [PATCH 4/8] Store: Upgrade copy mechanism for Raw snapshot import --- src/lib_store/unix/snapshots.ml | 77 +++++++++++++++++++++++++++++---- 1 file changed, 68 insertions(+), 9 deletions(-) diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index 8cbe1df8574c..d44007c57ad8 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -3185,18 +3185,77 @@ module Raw_importer : IMPORTER = struct else return_true) files + let copy_and_upgrade_legacy_file ~src ~dst ~file = + let open Lwt_result_syntax in + let*! fd = Lwt_unix.openfile src [Unix.O_RDONLY; O_CLOEXEC] 0o444 in + (* Destination *) + let*! upgraded_fd = + Lwt_unix.openfile dst [Unix.O_WRONLY; Unix.O_CREAT; Unix.O_TRUNC] 0o644 + in + (* Total number of blocks stored in the cemented blocks file *) + let* nb_blocks = + match String.split_on_char '_' file with + | [x_str; y_str] -> + let x = int_of_string x_str in + let y = int_of_string y_str in + return (y - x + 1) + | _ -> + (* Invalid cemented blocks file *) + failwith "Wrong filename format: %s" file + in + Lwt.finalize + (fun () -> + let*! bytes = + Cemented_block_store.get_and_upgrade_offsets fd nb_blocks + in + (* Write the 64-bit offsets list into the new file *) + let*! () = + Lwt_utils_unix.write_bytes + ~pos:0 + ~len:(8 * nb_blocks) + upgraded_fd + bytes + in + (* Copy the rest of the block file *) + let buffer = Bytes.create cemented_buffer_size in + let copy_bytes fd_in fd_out = + let rec loop () = + let*! bytes_read = + Lwt_unix.read fd_in buffer 0 cemented_buffer_size + in + if bytes_read = 0 then Lwt.return_unit + else + let*! _bytes_written = + Lwt_unix.write fd_out buffer 0 bytes_read + in + loop () + in + loop () + in + let*! () = copy_bytes fd upgraded_fd in + return_unit) + (fun () -> + let*! () = Lwt_unix.close fd in + let*! () = Lwt_unix.close upgraded_fd in + Lwt.return_unit) + let restore_cemented_cycle t ~file = - let open Lwt_syntax in + let open Lwt_result_syntax in let src = Filename.concat (Naming.dir_path t.snapshot_cemented_dir) file in let dst = Filename.concat (Naming.dir_path t.dst_cemented_dir) file in - let* () = - Lwt_utils_unix.copy_file_raw - ~buffer_size:cemented_buffer_size - ~src - ~dst - () - in - return_ok_unit + let* is_legacy_format = Version.is_legacy_format t.version in + if is_legacy_format then + (* Need to upgrade the offset format of cemented files from 32 to 64 bits. *) + copy_and_upgrade_legacy_file ~src ~dst ~file + else + let*! () = + Lwt_utils_unix.copy_file_raw + ~buffer_size:cemented_buffer_size + ~src + ~dst + () + in + return_unit let restore_floating_blocks t genesis_hash = let open Lwt_result_syntax in -- GitLab From 02fb0a2da849e39cd9d5d86a7893e29e0e82f8ee Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Fri, 2 Aug 2024 11:31:44 +0100 Subject: [PATCH 5/8] Tezt: bump snapshot version to 8 --- tezt/tests/storage_snapshots.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tezt/tests/storage_snapshots.ml b/tezt/tests/storage_snapshots.ml index 3e123e6930d5..fbae52d84f9a 100644 --- a/tezt/tests/storage_snapshots.ml +++ b/tezt/tests/storage_snapshots.ml @@ -543,7 +543,7 @@ let test_info_command = (* This is expected to be updated as soon as a new snapshot version is released (referring to the Snapshot.Version.current_version from `lib_store/unix/snapshots`)*) - let expected_version = 7 in + let expected_version = 8 in Log.info "Checks the human formatted output" ; (* Get the info line, which is the second line. *) let* () = -- GitLab From 5cb5d41ff2ef2cfd0c30b179032c5fdc35dd1b70 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Tue, 6 Aug 2024 09:25:33 +0100 Subject: [PATCH 6/8] Store: Add error for bad cemented_blocks_files names --- src/lib_store/unix/snapshots.ml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index d44007c57ad8..7357e9477a23 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -3199,9 +3199,7 @@ module Raw_importer : IMPORTER = struct let x = int_of_string x_str in let y = int_of_string y_str in return (y - x + 1) - | _ -> - (* Invalid cemented blocks file *) - failwith "Wrong filename format: %s" file + | _ -> tzfail (Invalid_cemented_file file) in Lwt.finalize (fun () -> @@ -3561,9 +3559,7 @@ module Tar_importer : IMPORTER = struct let x = int_of_string x_str in let y = int_of_string y_str in return (y - x + 1) - | _ -> - (* Invalid cemented blocks file *) - failwith "Wrong filename format: %s" file + | _ -> tzfail (Invalid_cemented_file file) in let*! () = Onthefly.copy_and_upgrade_legacy_file -- GitLab From 8e83080994af06c12e2e779e9282e9fa71623bc7 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Tue, 6 Aug 2024 09:41:24 +0100 Subject: [PATCH 7/8] Store: Add event message for upgrading snapshot version --- src/lib_store/shared/store_events.ml | 8 ++++++++ src/lib_store/unix/snapshots.ml | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/src/lib_store/shared/store_events.ml b/src/lib_store/shared/store_events.ml index 37efe592878b..354944850999 100644 --- a/src/lib_store/shared/store_events.ml +++ b/src/lib_store/shared/store_events.ml @@ -627,3 +627,11 @@ let upgrade_cemented_file_skip = ~msg:"File {path} does not have 32-bit offsets, skipping upgrade" ~pp1:Format.pp_print_string ("path", Data_encoding.string) + +let import_legacy_snapshot_version = + declare_1 + ~section + ~level:Notice + ~name:"import_legacy_snapshot_version" + ~msg:"Importing a legacy snapshot version: {legacy_version}." + ("legacy_version", Data_encoding.int31) diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index 7357e9477a23..43a8c58a9c42 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -4008,6 +4008,12 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct (Snapshot_file_not_found snapshot_path) in let snapshot_version = Importer.snapshot_version snapshot_importer in + let* is_legacy_format = Version.is_legacy_format snapshot_version in + let*! () = + if is_legacy_format then + Store_events.(emit import_legacy_snapshot_version snapshot_version) + else Lwt.return_unit + in let snapshot_metadata = Importer.snapshot_metadata snapshot_importer in let* () = fail_unless -- GitLab From 01e6ca459dd030af2dfc37c3955bd03e1c637322 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Fri, 9 Aug 2024 10:25:08 +0100 Subject: [PATCH 8/8] Store: Add TODO to remove legacy copy file --- src/lib_store/unix/snapshots.ml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index 43a8c58a9c42..69c574202338 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -975,7 +975,8 @@ module Onthefly : sig (* [copy_and_upgrade_legacy_file tar filename ~dst ~nb_blocks] loads the file [filename] from the given [tar], upgrades the [~nb_blocks] offset bytes from a 32 to a 64-bit format (in case the snapshot uses a legacy - format) and then it copies the rest of the file in the [~dst] file. *) + format) and then it copies the rest of the file in the [~dst] file. + TODO: this function should be removed when the upgrade is finished. *) val copy_and_upgrade_legacy_file : i -> file -> dst:string -> nb_blocks:int -> unit Lwt.t end = struct -- GitLab