From a157f93b6b6b51ce60d486b9bcc382639c67e725 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Mon, 22 Jul 2024 15:38:50 +0100 Subject: [PATCH 1/7] Store: Refactor the block offset calculation in cemented files --- src/lib_store/unix/cemented_block_store.ml | 125 ++++++++++++++------- 1 file changed, 86 insertions(+), 39 deletions(-) diff --git a/src/lib_store/unix/cemented_block_store.ml b/src/lib_store/unix/cemented_block_store.ml index 1fad8deccebc..ca5956b90fed 100644 --- a/src/lib_store/unix/cemented_block_store.ml +++ b/src/lib_store/unix/cemented_block_store.ml @@ -32,6 +32,81 @@ open Store_errors is an absolute offset in the file. are prefixed by 4 bytes of length *) + +module type OffsetCore = sig + type t + + (* Number of bytes needed for its representation in the cemented file *) + val offset_length : int + + (* Retrieve the type 'a offset from a block of bytes *) + val from_bytes : bytes -> int -> t + + (* Set the bytes with the correct value *) + val set_bytes : bytes -> int -> t -> unit + + (* Transform the offset to int *) + val to_int : t -> int + + (* Transform from int to offset *) + val of_int : int -> t + + (* Encoding of the type *) + val encoding : t Data_encoding.t +end + +module MakeOffset (OffsetCore : OffsetCore) = struct + include OffsetCore + + (** [get_offset fd block_number] returns the offset on position + [block_number] in the cemented file given by file descriptor [fd] using the + appropriate functions from the [OffsetCore] module, for transformation from [bytes] *) + let get_offset fd block_number = + let open Lwt_syntax in + let* _ofs = + Lwt_unix.lseek fd (block_number * offset_length) Unix.SEEK_SET + in + let offset_buffer = Bytes.create offset_length in + (* We read the (absolute) offset at the position in the offset array *) + let* () = + Lwt_utils_unix.read_bytes ~pos:0 ~len:offset_length fd offset_buffer + in + let ofs = from_bytes offset_buffer 0 in + return @@ to_int ofs + + (** [load_offset_region fd nb_blocks] retrieves the list + [nb_blocks] of offsets from the beginnig of a cemented file given + by [fd], with the help of [OffsetCore] bit manipulation tools. *) + let load_offset_region fd nb_blocks = + let open Lwt_syntax in + let len_offset = nb_blocks * offset_length in + let bytes = Bytes.create len_offset in + let+ () = Lwt_utils_unix.read_bytes ~len:len_offset fd bytes in + Array.map to_int + @@ Data_encoding.Binary.of_bytes_exn + Data_encoding.(Variable.array ~max_length:nb_blocks encoding) + bytes +end + +module Int32Core = struct + type t = int32 + + let offset_length = 4 + + let from_bytes = Bytes.get_int32_be + + let set_bytes = Bytes.set_int32_be + + let to_int ofs = + Option.value (Int32.unsigned_to_int ofs) ~default:(Int32.to_int ofs) + + let of_int = Int32.of_int + + let encoding = Data_encoding.int32 +end + +module Int32_offset = MakeOffset (Int32Core) + (* On-disk index of block's hashes to level *) module Cemented_block_level_index = Index_unix.Make (Block_key) (Block_level) (Index.Cache.Unbounded) @@ -369,10 +444,6 @@ let close cemented_store = terminated which means potential new reads won't be scheduled. *) Metadata_fd_cache.clear cemented_store.metadata_fd_cache -(* FIXME: https://gitlab.com/tezos/tezos/-/issues/7034 Cemented file - cannot exceed 4Gib. *) -let offset_length = 4 (* file offset *) - let find_block_file cemented_store block_level = try if Compare.Int32.(block_level < 0l) then None @@ -578,26 +649,7 @@ let cement_blocks_metadata cemented_store blocks = let read_block fd block_number = let open Lwt_syntax in - let* _ofs = Lwt_unix.lseek fd (block_number * offset_length) Unix.SEEK_SET in - let offset_buffer = Bytes.create offset_length in - (* We read the (absolute) offset at the position in the offset array *) - let* () = - Lwt_utils_unix.read_bytes ~pos:0 ~len:offset_length fd offset_buffer - in - let offset = - let ofs = Bytes.get_int32_be offset_buffer 0 in - (* We interpret the offset, written as an int32, as an unsigned - int32. This is allowed by the encoded scheme and allows one - additional bit to encode the offset. It enables dealing with - files up to 4Gib. *) - match Int32.unsigned_to_int ofs with - | Some v -> v - | None -> - (* It will be [None] on 32-bit machines which is not - supported. We default to [Int32.to_int] instead of [assert - false] *) - Int32.to_int ofs - in + let* offset = Int32_offset.get_offset fd block_number in let* _ofs = Lwt_unix.lseek fd offset Unix.SEEK_SET in (* We move the cursor to the element's position *) let* block, _len = Block_repr_unix.read_next_block_exn fd in @@ -661,7 +713,7 @@ let cement_blocks ?(check_consistency = true) (cemented_store : t) ~write_metadata ({chunk_length; reading_sequence} : chunk_iterator) = let open Lwt_result_syntax in let nb_blocks = chunk_length in - let preamble_length = nb_blocks * offset_length in + let preamble_length = nb_blocks * Int32_offset.offset_length in let* () = fail_when (nb_blocks = 0) (Cannot_cement_blocks `Empty) in let* first_block_level = let* _block_hash, _n, block_bytes = @@ -781,10 +833,13 @@ let cement_blocks ?(check_consistency = true) (cemented_store : t) Block_repr_unix.prune_raw_block_bytes block_bytes in (* We start by blitting the corresponding offset in the preamble part *) - Bytes.set_int32_be - offsets_buffer - (i * offset_length) - (Int32.of_int current_offset) ; + let () = + Int32_offset.( + set_bytes + offsets_buffer + (i * offset_length) + (of_int current_offset)) + in (* We write the block in the file *) let*! () = Lwt_utils_unix.write_bytes @@ -1031,15 +1086,7 @@ let check_indexes_consistency ?(post_step = fun () -> Lwt.return_unit) Int32.to_int (cemented_blocks_file_length cemented_blocks_file) in - (* Load the offset region *) - let len_offset = nb_blocks * offset_length in - let bytes = Bytes.create len_offset in - let*! () = Lwt_utils_unix.read_bytes ~len:len_offset fd bytes in - let offsets = - Data_encoding.Binary.of_bytes_exn - Data_encoding.(Variable.array ~max_length:nb_blocks int32) - bytes - in + let*! offsets = Int32_offset.load_offset_region fd nb_blocks in (* Cursor is now after the offset region *) let rec iter_blocks ?pred_block n = if n = nb_blocks then return_unit @@ -1047,7 +1094,7 @@ let check_indexes_consistency ?(post_step = fun () -> Lwt.return_unit) let*! cur_offset = Lwt_unix.lseek fd 0 Unix.SEEK_CUR in let* () = fail_unless - Compare.Int32.(Int32.of_int cur_offset = offsets.(n)) + (cur_offset = offsets.(n)) (Inconsistent_cemented_store (Bad_offset {level = n; cycle = Naming.file_path file})) -- GitLab From f3dcd8d14a4f8cb9db12054381da993edf1470e5 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Mon, 22 Jul 2024 15:42:25 +0100 Subject: [PATCH 2/7] Store: Add compatibility with 64-bit offsets for reading of cemented files --- src/lib_store/unix/cemented_block_store.ml | 62 +++++++++++++++++++--- 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/src/lib_store/unix/cemented_block_store.ml b/src/lib_store/unix/cemented_block_store.ml index ca5956b90fed..37cc24d3eed1 100644 --- a/src/lib_store/unix/cemented_block_store.ml +++ b/src/lib_store/unix/cemented_block_store.ml @@ -105,7 +105,24 @@ module Int32Core = struct let encoding = Data_encoding.int32 end +module Int64Core = struct + type t = int64 + + let offset_length = 8 + + let from_bytes = Bytes.get_int64_be + + let set_bytes = Bytes.set_int64_be + + let to_int = Int64.to_int + + let of_int = Int64.of_int + + let encoding = Data_encoding.int64 +end + module Int32_offset = MakeOffset (Int32Core) +module Int64_offset = MakeOffset (Int64Core) (* On-disk index of block's hashes to level *) module Cemented_block_level_index = @@ -492,9 +509,14 @@ let find_block_file cemented_store block_level = (* Hypothesis: the table is ordered. *) let compute_location cemented_store block_level = let open Option_syntax in - let+ {start_level; file; _} = find_block_file cemented_store block_level in + let+ {start_level; end_level; file} = + find_block_file cemented_store block_level + in let level_delta = Int32.(to_int (sub block_level start_level)) in - (file, level_delta) + let nb_blocks = + Int32.to_int @@ cemented_blocks_file_length {start_level; file; end_level} + in + (file, level_delta, nb_blocks) let is_cemented cemented_store hash = try @@ -579,7 +601,7 @@ let read_block_metadata ?location cemented_store block_level = in match location with | None -> return_none - | Some (cemented_file, _block_number) -> + | Some (cemented_file, _block_number, _nb_blocks) -> let metadata_file = Naming.( cemented_store.cemented_blocks_dir |> cemented_blocks_metadata_dir @@ -647,9 +669,35 @@ let cement_blocks_metadata cemented_store blocks = let*! () = Lwt_unix.rename tmp_metadata_file_path metadata_file_path in return_unit -let read_block fd block_number = +(** [is_using_64_bit_offsets fd nb_blocks] checks whether the cemented file + given by [fd] is formatted with 32 or 64 bit offsets; the decision + is taken based on whether the first offset points correctly to the first + block in the file or not; + - offset = first 64 bits decoded as an int64 + - first_block_offset = 8 (bytes) * [nb_blocks] (first block offset, given + that the file has 64-bit offsets) + Check whether (offset = first_block_offset) holds. If not, then we definitely + use 32-bit offsets. If it does, then we cannot have 32-bit offsets, as we cannot have + the first 4 bytes decoded as int32 to 4 * num_blocks and also the first 8 bytes + decoded as int64 to 8 * num_blocks. *) +let is_using_64_bit_offsets fd nb_blocks = + let open Lwt_syntax in + let open Int64_offset in + (* Save the current position of the file descriptor *) + let* current_position = Lwt_unix.lseek fd 0 Unix.SEEK_CUR in + (* Obtain the first offset of the file, altering the file descriptor position *) + let* offset = get_offset fd 0 in + (* Restore the former position of the file descriptor *) + let* _ofs = Lwt_unix.lseek fd current_position Unix.SEEK_SET in + return (offset = offset_length * nb_blocks) + +let read_block fd block_number nb_blocks = let open Lwt_syntax in - let* offset = Int32_offset.get_offset fd block_number in + let* offset = + let* use_64_bits_offsets = is_using_64_bit_offsets fd nb_blocks in + if use_64_bits_offsets then Int64_offset.get_offset fd block_number + else Int32_offset.get_offset fd block_number + in let* _ofs = Lwt_unix.lseek fd offset Unix.SEEK_SET in (* We move the cursor to the element's position *) let* block, _len = Block_repr_unix.read_next_block_exn fd in @@ -677,12 +725,12 @@ let get_cemented_block_by_level (cemented_store : t) ~read_metadata level = let open Lwt_result_syntax in match compute_location cemented_store level with | None -> return_none - | Some ((filename, block_number) as location) -> + | Some ((filename, block_number, nb_blocks) as location) -> let file_path = Naming.file_path filename in let*! fd = Lwt_unix.openfile file_path [Unix.O_RDONLY; O_CLOEXEC] 0o444 in let*! block = Lwt.finalize - (fun () -> read_block fd block_number) + (fun () -> read_block fd block_number nb_blocks) (fun () -> let*! _ = Lwt_utils_unix.safe_close fd in Lwt.return_unit) -- GitLab From abd348b96172156b8d48c267d6786dc7d02dc911 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Mon, 22 Jul 2024 15:45:33 +0100 Subject: [PATCH 3/7] Store: Add compatibility with 64-bit offsets for checking consistency of cemented files --- src/lib_store/unix/cemented_block_store.ml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/lib_store/unix/cemented_block_store.ml b/src/lib_store/unix/cemented_block_store.ml index 37cc24d3eed1..1014eb138f69 100644 --- a/src/lib_store/unix/cemented_block_store.ml +++ b/src/lib_store/unix/cemented_block_store.ml @@ -1134,7 +1134,14 @@ let check_indexes_consistency ?(post_step = fun () -> Lwt.return_unit) Int32.to_int (cemented_blocks_file_length cemented_blocks_file) in - let*! offsets = Int32_offset.load_offset_region fd nb_blocks in + let*! use_64_bits_offsets = + is_using_64_bit_offsets fd nb_blocks + in + let*! offsets = + if use_64_bits_offsets then + Int64_offset.load_offset_region fd nb_blocks + else Int32_offset.load_offset_region fd nb_blocks + in (* Cursor is now after the offset region *) let rec iter_blocks ?pred_block n = if n = nb_blocks then return_unit -- GitLab From 11034fc098fe0eaad5af85f258da96d096571190 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Mon, 22 Jul 2024 15:48:48 +0100 Subject: [PATCH 4/7] Store: Default to 64-bit offsets for writing cemented files --- src/lib_store/unix/cemented_block_store.ml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/lib_store/unix/cemented_block_store.ml b/src/lib_store/unix/cemented_block_store.ml index 1014eb138f69..12f94b6913e7 100644 --- a/src/lib_store/unix/cemented_block_store.ml +++ b/src/lib_store/unix/cemented_block_store.ml @@ -750,8 +750,6 @@ let get_cemented_block_by_hash ~read_metadata (cemented_store : t) hash = | Some level -> get_cemented_block_by_level ~read_metadata cemented_store level -(* TODO/FIXME: https://gitlab.com/tezos/tezos/-/issues/7035 - Cemented metadata cannot exceed 4Gib *) (* Hypothesis: - The block list is expected to be ordered by increasing level and no blocks are skipped. @@ -761,7 +759,7 @@ let cement_blocks ?(check_consistency = true) (cemented_store : t) ~write_metadata ({chunk_length; reading_sequence} : chunk_iterator) = let open Lwt_result_syntax in let nb_blocks = chunk_length in - let preamble_length = nb_blocks * Int32_offset.offset_length in + let preamble_length = nb_blocks * Int64_offset.offset_length in let* () = fail_when (nb_blocks = 0) (Cannot_cement_blocks `Empty) in let* first_block_level = let* _block_hash, _n, block_bytes = @@ -882,7 +880,7 @@ let cement_blocks ?(check_consistency = true) (cemented_store : t) in (* We start by blitting the corresponding offset in the preamble part *) let () = - Int32_offset.( + Int64_offset.( set_bytes offsets_buffer (i * offset_length) @@ -1060,8 +1058,20 @@ let raw_iter_cemented_file f ({file; _} as cemented_blocks_file) = file_path (fun channel -> let nb_blocks = cemented_blocks_file_length cemented_blocks_file in - let* first_block_offset = Lwt_io.BE.read_int channel in - let* () = Lwt_io.set_position channel (Int64.of_int first_block_offset) in + (* Check which offset format we have for the cemented file *) + let* offset = Lwt_io.BE.read_int64 channel in + let use_64_bits_offsets = + Int64.to_int offset + = Int64_offset.offset_length * Int32.to_int nb_blocks + in + let* () = Lwt_io.set_position channel 0L in + let* first_block_offset = + if use_64_bits_offsets then Lwt_io.BE.read_int64 channel + else + let+ ofs = Lwt_io.BE.read_int channel in + Int64.of_int ofs + in + let* () = Lwt_io.set_position channel first_block_offset in let rec loop n = if n = 0 then Lwt.return_unit else -- GitLab From d3d60bc4117269b3dfa829afbd10dd2dc6a1381b Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Tue, 16 Jul 2024 15:42:11 +0100 Subject: [PATCH 5/7] Store: Update documentation for 64-bit offsets --- src/lib_store/unix/cemented_block_store.ml | 8 ++++++-- src/lib_store/unix/cemented_block_store.mli | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/lib_store/unix/cemented_block_store.ml b/src/lib_store/unix/cemented_block_store.ml index 12f94b6913e7..fde2a80deb4a 100644 --- a/src/lib_store/unix/cemented_block_store.ml +++ b/src/lib_store/unix/cemented_block_store.ml @@ -2,6 +2,7 @@ (* *) (* Open Source License *) (* Copyright (c) 2020-2021 Nomadic Labs, *) +(* Copyright (c) 2024 TriliTech *) (* *) (* Permission is hereby granted, free of charge, to any person obtaining a *) (* copy of this software and associated documentation files (the "Software"),*) @@ -27,10 +28,13 @@ open Store_errors (* Cemented files overlay: - | x | x | + | x | x | is an absolute offset in the file. - are prefixed by 4 bytes of length + are prefixed by 4 bytes of length. + For legacy purposes, we allow compatibility with the old + cemented files, which have 32-bit offsets, but the default + behaviour is to create cemented files with 64-bit offsets. *) module type OffsetCore = sig diff --git a/src/lib_store/unix/cemented_block_store.mli b/src/lib_store/unix/cemented_block_store.mli index 85673e276f79..9c195ecc4008 100644 --- a/src/lib_store/unix/cemented_block_store.mli +++ b/src/lib_store/unix/cemented_block_store.mli @@ -77,7 +77,7 @@ v} | × | × | - where n is (j - i + 1), is a 4 bytes integer representing + where n is (j - i + 1), is a 8 bytes integer representing the absolute offset of a block where the k-th (with 0 <= k < n) offset stands for the absolute offset of the k-th block in the file and with , a {!Block_repr.t} value encoded using -- GitLab From b1535bd9cd2a6bcf043f48f21335a91a68a24785 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Mon, 22 Jul 2024 15:53:40 +0100 Subject: [PATCH 6/7] Store: Augment tests to work on both offsets format --- src/lib_store/unix/cemented_block_store.ml | 36 +++++++++--- src/lib_store/unix/cemented_block_store.mli | 12 ++++ .../unix/test/test_cemented_store.ml | 55 +++++++++++-------- 3 files changed, 73 insertions(+), 30 deletions(-) diff --git a/src/lib_store/unix/cemented_block_store.ml b/src/lib_store/unix/cemented_block_store.ml index fde2a80deb4a..28ba4997e272 100644 --- a/src/lib_store/unix/cemented_block_store.ml +++ b/src/lib_store/unix/cemented_block_store.ml @@ -759,11 +759,16 @@ let get_cemented_block_by_hash ~read_metadata (cemented_store : t) hash = level and no blocks are skipped. - If the first block has metadata, metadata are written and all blocks are expected to have metadata. *) -let cement_blocks ?(check_consistency = true) (cemented_store : t) - ~write_metadata ({chunk_length; reading_sequence} : chunk_iterator) = +let cement_blocks_helper ~use_32_bit_offset ?(check_consistency = true) + (cemented_store : t) ~write_metadata + ({chunk_length; reading_sequence} : chunk_iterator) = let open Lwt_result_syntax in let nb_blocks = chunk_length in - let preamble_length = nb_blocks * Int64_offset.offset_length in + let offset_length = + if use_32_bit_offset then Int32_offset.offset_length + else Int64_offset.offset_length + in + let preamble_length = nb_blocks * offset_length in let* () = fail_when (nb_blocks = 0) (Cannot_cement_blocks `Empty) in let* first_block_level = let* _block_hash, _n, block_bytes = @@ -884,11 +889,18 @@ let cement_blocks ?(check_consistency = true) (cemented_store : t) in (* We start by blitting the corresponding offset in the preamble part *) let () = - Int64_offset.( - set_bytes - offsets_buffer - (i * offset_length) - (of_int current_offset)) + if use_32_bit_offset then + Int32_offset.( + set_bytes + offsets_buffer + (i * offset_length) + (of_int current_offset)) + else + Int64_offset.( + set_bytes + offsets_buffer + (i * offset_length) + (of_int current_offset)) in (* We write the block in the file *) let*! () = @@ -960,6 +972,8 @@ let cement_blocks ?(check_consistency = true) (cemented_store : t) cemented_store.cemented_blocks_files <- Some new_array ; if write_metadata then metadata_finalizer () else return_unit +let cement_blocks = cement_blocks_helper ~use_32_bit_offset:false + let trigger_full_gc cemented_store cemented_blocks_files offset = let open Lwt_syntax in let nb_files = Array.length cemented_blocks_files in @@ -1211,3 +1225,9 @@ let check_indexes_consistency ?(post_step = fun () -> Lwt.return_unit) table_list in return_unit + +(************ For testing and internal purposes only **************) +module Unsafe = struct + (* This is a version of cementing blocks with 32-bit offsets, for testing purposes *) + let cement_blocks_32_bits = cement_blocks_helper ~use_32_bit_offset:true +end diff --git a/src/lib_store/unix/cemented_block_store.mli b/src/lib_store/unix/cemented_block_store.mli index 9c195ecc4008..f8181392eae5 100644 --- a/src/lib_store/unix/cemented_block_store.mli +++ b/src/lib_store/unix/cemented_block_store.mli @@ -309,3 +309,15 @@ val check_indexes_consistency : ?genesis_hash:Block_hash.t -> t -> unit tzresult Lwt.t + +module Unsafe : sig + (** This is a similar function to the [cement_blocks] one, but it uses 32-bit + offsets for blocks cementation, so we mark it as unsafe, for testing purposes only + as we do not want to use the 32-bit format anymore. *) + val cement_blocks_32_bits : + ?check_consistency:bool -> + t -> + write_metadata:bool -> + chunk_iterator -> + unit tzresult Lwt.t +end diff --git a/src/lib_store/unix/test/test_cemented_store.ml b/src/lib_store/unix/test/test_cemented_store.ml index 78a53dd80a17..3872a3e25349 100644 --- a/src/lib_store/unix/test/test_cemented_store.ml +++ b/src/lib_store/unix/test/test_cemented_store.ml @@ -32,6 +32,7 @@ *) open Test_utils +open Cemented_block_store let assert_presence_in_cemented_store ?(with_metadata = true) cemented_store blocks = @@ -63,51 +64,48 @@ let assert_presence_in_cemented_store ?(with_metadata = true) cemented_store return_unit)) blocks -let test_cement_pruned_blocks cemented_store = +let test_cement_pruned_blocks cemented_store ~cement_blocks = let open Lwt_result_syntax in let*! blocks, _head = make_raw_block_list ~kind:`Pruned (genesis_hash, -1l) 4095 in let* () = - Cemented_block_store.( - cement_blocks - cemented_store - ~write_metadata:false - (make_chunk_iterator blocks)) + cement_blocks + cemented_store + ~write_metadata:false + (make_chunk_iterator blocks) in assert_presence_in_cemented_store ~with_metadata:true cemented_store blocks -let test_cement_full_blocks cemented_store = +let test_cement_full_blocks cemented_store ~cement_blocks = let open Lwt_result_syntax in let*! blocks, _head = make_raw_block_list ~kind:`Full (genesis_hash, -1l) 4095 in let* () = - Cemented_block_store.( - cement_blocks - cemented_store - ~write_metadata:false - (make_chunk_iterator blocks)) + cement_blocks + cemented_store + ~write_metadata:false + (make_chunk_iterator blocks) in assert_presence_in_cemented_store ~with_metadata:false cemented_store blocks -let test_metadata_retrieval cemented_store = +let test_metadata_retrieval cemented_store ~cement_blocks = let open Lwt_result_syntax in let*! blocks, _head = make_raw_block_list ~kind:`Full (genesis_hash, -1l) 100 in let* () = - Cemented_block_store.( - cement_blocks - cemented_store - ~write_metadata:true - (make_chunk_iterator blocks)) + cement_blocks + cemented_store + ~write_metadata:true + (make_chunk_iterator blocks) in assert_presence_in_cemented_store ~with_metadata:true cemented_store blocks let wrap_cemented_store_test (name, f) = let open Lwt_result_syntax in - let cemented_store_init f _ () = + let cemented_store_init f _ ~cement_blocks () = let prefix_dir = "tezos_indexed_store_test_" in Lwt_utils_unix.with_tempdir prefix_dir (fun base_dir -> let run f = f base_dir in @@ -120,7 +118,7 @@ let wrap_cemented_store_test (name, f) = Cemented_block_store.init ~readonly:false chain_dir in Error_monad.protect (fun () -> - let* () = f cemented_store in + let* () = f cemented_store ~cement_blocks in Cemented_block_store.close cemented_store ; return_unit)) in @@ -133,11 +131,24 @@ let wrap_cemented_store_test (name, f) = Lwt.fail Alcotest.Test_error | Ok () -> Lwt.return_unit) in - Alcotest_lwt.test_case name `Quick (cemented_store_init f) + [ + Alcotest_lwt.test_case + (name ^ " (32-bit offsets)") + `Quick + (cemented_store_init + f + ~cement_blocks:(Unsafe.cement_blocks_32_bits ~check_consistency:true)); + Alcotest_lwt.test_case + (name ^ " (64-bit offsets)") + `Quick + (cemented_store_init + f + ~cement_blocks:(cement_blocks ~check_consistency:true)); + ] let tests = let test_cases = - List.map + List.concat_map wrap_cemented_store_test [ ("cementing pruned blocks", test_cement_pruned_blocks); -- GitLab From 7c555c58f235aba82ae9b6adff9b169d7d20eddc Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Mon, 22 Jul 2024 14:47:56 +0100 Subject: [PATCH 7/7] Store: Increase the version of the store --- src/lib_node_config/data_version.ml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/lib_node_config/data_version.ml b/src/lib_node_config/data_version.ml index 866d098b153c..6b56adafc1d9 100644 --- a/src/lib_node_config/data_version.ml +++ b/src/lib_node_config/data_version.ml @@ -102,16 +102,19 @@ end * - 2.0 : introduce context GC (upgrade to irmin.3.4) -- v15.0 * - 3.0 : change blocks' context hash semantics and introduce context split (upgrade to irmin.3.5) -- v16.0 - * - 3.1 : change encoding for block store status -- v21.0 *) + * - 3.1 : change encoding for block store status -- v21.0 + * - 3.2 : change cemented block store format -- v22.0 *) (* FIXME https://gitlab.com/tezos/tezos/-/issues/2861 We should enable the semantic versioning instead of applying hardcoded rules.*) let v_3_0 = Version.make ~major:3 ~minor:0 -let v_3_1 = Version.make ~major:3 ~minor:1 +let _v_3_1 = Version.make ~major:3 ~minor:1 -let current_version = v_3_1 +let v_3_2 = Version.make ~major:3 ~minor:2 + +let current_version = v_3_2 (* List of upgrade functions from each still supported previous version to the current [data_version] above. If this list grows too -- GitLab