From 75018437f9f7a7b43aedbf4b81f694663cc233d0 Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Tue, 8 Jul 2025 13:56:32 +0200 Subject: [PATCH 1/2] Store/snapshots: remove obsolete legacy store/snapshot upgrade --- src/lib_store/unix/snapshots.ml | 149 ++++---------------------------- 1 file changed, 18 insertions(+), 131 deletions(-) diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index 0f98f669b04d..b3e1e9579f32 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -1005,14 +1005,6 @@ module Onthefly : sig (* [copy_to_file tar file ~dst] copies the [file] from the [tar] into new file designated by [dst]. *) val copy_to_file : i -> file -> dst:string -> unit Lwt.t - - (* [copy_and_upgrade_legacy_file tar filename ~dst ~nb_blocks] loads - the file [filename] from the given [tar], upgrades the [~nb_blocks] offset - bytes from a 32 to a 64-bit format (in case the snapshot uses a legacy - format) and then it copies the rest of the file in the [~dst] file. - TODO: this function should be removed when the upgrade is finished. *) - val copy_and_upgrade_legacy_file : - i -> file -> dst:string -> nb_blocks:int -> unit Lwt.t end = struct include Tar @@ -1437,31 +1429,6 @@ end = struct Lwt.finalize (fun () -> copy_n tar.fd fd header.Tar.Header.file_size) (fun () -> Lwt_unix.close fd) - - let copy_and_upgrade_legacy_file tar {header; data_ofs} ~dst ~nb_blocks = - let open Lwt_syntax in - let* _ = Lwt_unix.LargeFile.lseek tar.fd data_ofs SEEK_SET in - let* upgraded_fd = - Lwt_unix.openfile - dst - Unix.[O_WRONLY; O_CREAT; O_TRUNC] - snapshot_rw_file_perm - in - Lwt.finalize - (fun () -> - (* Should use the Cemented_block_store function - need to expose after merging MR (?) *) - let* bytes = - Cemented_block_store.get_and_upgrade_offsets tar.fd nb_blocks - in - (* Write the 64-bit offsets list into the new file *) - let block = Cstruct.of_bytes bytes in - let* () = Writer.really_write upgraded_fd block in - (* Copy the rest of the block file containing INITIAL_SIZE - 4 * NB_BLOCKS bytes. *) - let remaining_size = - Int64.(sub header.Tar.Header.file_size (of_int (4 * nb_blocks))) - in - copy_n tar.fd upgraded_fd remaining_size) - (fun () -> Lwt_unix.close upgraded_fd) end module type EXPORTER = sig @@ -3240,75 +3207,18 @@ module Raw_importer : IMPORTER = struct else return_true) files - let copy_and_upgrade_legacy_file ~src ~dst ~file = - let open Lwt_result_syntax in - let*! fd = Lwt_unix.openfile src [Unix.O_RDONLY; O_CLOEXEC] 0o444 in - (* Destination *) - let*! upgraded_fd = - Lwt_unix.openfile dst [Unix.O_WRONLY; Unix.O_CREAT; Unix.O_TRUNC] 0o644 - in - (* Total number of blocks stored in the cemented blocks file *) - let* nb_blocks = - match String.split_on_char '_' file with - | [x_str; y_str] -> - let x = int_of_string x_str in - let y = int_of_string y_str in - return (y - x + 1) - | _ -> tzfail (Invalid_cemented_file file) - in - Lwt.finalize - (fun () -> - let*! bytes = - Cemented_block_store.get_and_upgrade_offsets fd nb_blocks - in - (* Write the 64-bit offsets list into the new file *) - let*! () = - Lwt_utils_unix.write_bytes - ~pos:0 - ~len:(8 * nb_blocks) - upgraded_fd - bytes - in - (* Copy the rest of the block file *) - let buffer = Bytes.create cemented_buffer_size in - let copy_bytes fd_in fd_out = - let rec loop () = - let*! bytes_read = - Lwt_unix.read fd_in buffer 0 cemented_buffer_size - in - if bytes_read = 0 then Lwt.return_unit - else - let*! _bytes_written = - Lwt_unix.write fd_out buffer 0 bytes_read - in - loop () - in - loop () - in - let*! () = copy_bytes fd upgraded_fd in - return_unit) - (fun () -> - let*! () = Lwt_unix.close fd in - let*! () = Lwt_unix.close upgraded_fd in - Lwt.return_unit) - let restore_cemented_cycle t ~file = let open Lwt_result_syntax in let src = Filename.concat (Naming.dir_path t.snapshot_cemented_dir) file in let dst = Filename.concat (Naming.dir_path t.dst_cemented_dir) file in - let* is_legacy_format = Version.is_legacy_format t.version in - if is_legacy_format then - (* Need to upgrade the offset format of cemented files from 32 to 64 bits. *) - copy_and_upgrade_legacy_file ~src ~dst ~file - else - let*! () = - Lwt_utils_unix.copy_file_raw - ~buffer_size:cemented_buffer_size - ~src - ~dst - () - in - return_unit + let*! () = + Lwt_utils_unix.copy_file_raw + ~buffer_size:cemented_buffer_size + ~src + ~dst + () + in + return_unit let restore_floating_blocks t genesis_hash = let open Lwt_result_syntax in @@ -3618,39 +3528,16 @@ module Tar_importer : IMPORTER = struct | Some file -> return file | None -> tzfail (Cannot_read {kind = `Cemented_cycle; path = filename}) in - let* is_legacy_format = Version.is_legacy_format t.version in - if is_legacy_format then - (* Need to upgrade the offset format of cemented files from 32 to 64 bits. *) - let* nb_blocks = - match String.split_on_char '_' file with - | [x_str; y_str] -> - let x = int_of_string x_str in - let y = int_of_string y_str in - return (y - x + 1) - | _ -> tzfail (Invalid_cemented_file file) - in - let*! () = - Onthefly.copy_and_upgrade_legacy_file - t.tar - tar_file - ~dst: - (Filename.concat - (Naming.dir_path t.dst_cemented_dir) - (Filename.basename file)) - ~nb_blocks - in - return_unit - else - let*! () = - Onthefly.copy_to_file - t.tar - tar_file - ~dst: - (Filename.concat - (Naming.dir_path t.dst_cemented_dir) - (Filename.basename file)) - in - return_unit + let*! () = + Onthefly.copy_to_file + t.tar + tar_file + ~dst: + (Filename.concat + (Naming.dir_path t.dst_cemented_dir) + (Filename.basename file)) + in + return_unit let restore_floating_blocks t genesis_hash = let open Lwt_result_syntax in -- GitLab From 21c8a57fcfd01ae578504e0370f2dde4bba861db Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Tue, 8 Jul 2025 14:01:12 +0200 Subject: [PATCH 2/2] Store/snapshots: enforce v8 as legacy version --- src/lib_store/unix/snapshots.ml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index b3e1e9579f32..cc14eed977cb 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -4128,7 +4128,9 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct (* TODO/FIXME: https://gitlab.com/tezos/tezos/-/issues/8005 remove the v8 import backward compatibility as soon as v9 (and v23) are mandatory.*) - let is_v8_import = snapshot_version = Version.v8_version in + let is_v8_import = + is_legacy_format && snapshot_version = Version.v8_version + in let*! () = Event.(emit restoring_floating_blocks) () in let* () = Animation.display_progress -- GitLab