From 7c1479eede01f9d751615f37cae25325c018d239 Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Wed, 28 Feb 2024 10:41:05 +0100 Subject: [PATCH 1/3] Store: limit cemented cycles size to 65_535 --- src/lib_store/unix/block_store.ml | 36 +++++++++++++++++++++- src/lib_store/unix/cemented_block_store.ml | 2 +- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/lib_store/unix/block_store.ml b/src/lib_store/unix/block_store.ml index 6d18f1953a9d..18e06298c762 100644 --- a/src/lib_store/unix/block_store.ml +++ b/src/lib_store/unix/block_store.ml @@ -881,6 +881,36 @@ let compute_new_caboose block_store history_mode ~new_savepoint module BlocksLPBL = Set.Make (Int32) +(* Limits the maximum number of elements that can be added into a + cycle. + This is mandatory when cementing metadata. Indeed, the current + version of camlzip support only 32bits zip files, that are files + smaller that ~4GB or containing less that 65_535 entries. When + cementing cycles, we might reach that limit. We set it to 2^16 - 1. *) +let default_cycle_size_limit = 65_535l + +(* May shrink the size of the given cycles to make sure that the size + of a cycle never exceeds the camlzip 32bits limitation. The shrink + consist in dividing the cycles in two even parts, recursively, + until the limit is not exceeded anymore. *) +let may_shrink_cycles cycles = + let rec loop acc cycles = + match cycles with + | [] -> List.rev acc + | ((cycle_start, cycle_end) as hd) :: tl -> + let diff = Int32.(sub cycle_end cycle_start) in + if diff >= cycle_size_limit then + let mid = Int32.(div diff 2l) in + let left_cycle_upper_bound = Int32.(add cycle_start mid) in + let left_cycle = (cycle_start, left_cycle_upper_bound) in + let right_cycle = + (Int32.(add left_cycle_upper_bound 1l), cycle_end) + in + loop acc (left_cycle :: right_cycle :: tl) + else loop (hd :: acc) tl + in + loop [] cycles + (* FIXME: update doc *) (* [update_floating_stores block_store ~history_mode ~ro_store ~rw_store ~new_store ~new_head ~new_head_lpbl @@ -1031,7 +1061,11 @@ let update_floating_stores block_store ~history_mode ~ro_store ~rw_store let sorted_lpbl = List.sort Compare.Int32.compare (BlocksLPBL.elements !blocks_lpbl) in - let* cycles_to_cement = loop [] initial_pred sorted_lpbl in + + let* cycles_to_cement = + let* cycles = loop [] initial_pred sorted_lpbl in + return (may_shrink_cycles cycles) + in let* new_savepoint = compute_new_savepoint block_store diff --git a/src/lib_store/unix/cemented_block_store.ml b/src/lib_store/unix/cemented_block_store.ml index d4ff48ef8c5d..955ddf3e3c06 100644 --- a/src/lib_store/unix/cemented_block_store.ml +++ b/src/lib_store/unix/cemented_block_store.ml @@ -573,7 +573,7 @@ let read_block fd block_number = let ofs = Bytes.get_int32_be offset_buffer 0 in (* We interpret the offset, written as an int32, as an unsigned int32. This is allowed by the encoded scheme and allows one - additional bit to encode the offset. In enables dealing with + additional bit to encode the offset. It enables dealing with files up to 4Gib. *) match Int32.unsigned_to_int ofs with | Some v -> v -- GitLab From 2e39a6b9788c5448bfbcb7b5da02ff2aee6866ae Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Fri, 1 Mar 2024 10:16:59 +0100 Subject: [PATCH 2/3] Store: add TODO/FIXME - Cemented metadata cannot exceed 4Gib --- src/lib_store/unix/cemented_block_store.ml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib_store/unix/cemented_block_store.ml b/src/lib_store/unix/cemented_block_store.ml index 955ddf3e3c06..95fb1aa26f2d 100644 --- a/src/lib_store/unix/cemented_block_store.ml +++ b/src/lib_store/unix/cemented_block_store.ml @@ -635,6 +635,8 @@ let get_cemented_block_by_hash ~read_metadata (cemented_store : t) hash = | Some level -> get_cemented_block_by_level ~read_metadata cemented_store level +(* TODO/FIXME: https://gitlab.com/tezos/tezos/-/issues/7035 + Cemented metadata cannot exceed 4Gib *) (* Hypothesis: - The block list is expected to be ordered by increasing level and no blocks are skipped. -- GitLab From 251c050e07835a5eae4af1c3be2d0cbd70f98c96 Mon Sep 17 00:00:00 2001 From: vbot Date: Mon, 4 Mar 2024 13:56:54 +0100 Subject: [PATCH 3/3] Store/Test: add cycle split test --- src/lib_store/unix/block_store.ml | 16 ++++---- src/lib_store/unix/block_store.mli | 7 ++++ src/lib_store/unix/test/test_block_store.ml | 43 +++++++++++++++++++-- 3 files changed, 56 insertions(+), 10 deletions(-) diff --git a/src/lib_store/unix/block_store.ml b/src/lib_store/unix/block_store.ml index 18e06298c762..a2668d75b1cb 100644 --- a/src/lib_store/unix/block_store.ml +++ b/src/lib_store/unix/block_store.ml @@ -893,7 +893,7 @@ let default_cycle_size_limit = 65_535l of a cycle never exceeds the camlzip 32bits limitation. The shrink consist in dividing the cycles in two even parts, recursively, until the limit is not exceeded anymore. *) -let may_shrink_cycles cycles = +let may_shrink_cycles cycles ~cycle_size_limit = let rec loop acc cycles = match cycles with | [] -> List.rev acc @@ -922,7 +922,7 @@ let may_shrink_cycles cycles = savepoint and caboose candidates. *) let update_floating_stores block_store ~history_mode ~ro_store ~rw_store ~new_store ~new_head ~new_head_lpbl ~lowest_bound_to_preserve_in_floating - ~cementing_highwatermark = + ~cementing_highwatermark ~cycle_size_limit = let open Lwt_result_syntax in let*! () = Store_events.(emit start_updating_floating_stores) () in let* lpbl_block = @@ -1064,7 +1064,7 @@ let update_floating_stores block_store ~history_mode ~ro_store ~rw_store let* cycles_to_cement = let* cycles = loop [] initial_pred sorted_lpbl in - return (may_shrink_cycles cycles) + return (may_shrink_cycles cycles ~cycle_size_limit) in let* new_savepoint = compute_new_savepoint @@ -1238,7 +1238,7 @@ let instanciate_temporary_floating_store block_store = let create_merging_thread block_store ~history_mode ~old_ro_store ~old_rw_store ~new_head ~new_head_lpbl ~lowest_bound_to_preserve_in_floating - ~cementing_highwatermark = + ~cementing_highwatermark ~cycle_size_limit = let open Lwt_result_syntax in let*! () = Store_events.(emit start_merging_thread) () in let*! new_ro_store = @@ -1258,6 +1258,7 @@ let create_merging_thread block_store ~history_mode ~old_ro_store ~old_rw_store ~new_head_lpbl ~lowest_bound_to_preserve_in_floating ~cementing_highwatermark + ~cycle_size_limit in let*! () = Store_events.(emit cementing_block_ranges) cycles_interval_to_cement @@ -1379,9 +1380,9 @@ let split_context block_store new_head_lpbl = let*! () = Store_events.(emit start_context_split new_head_lpbl) in split () -let merge_stores block_store ~(on_error : tztrace -> unit tzresult Lwt.t) - ~finalizer ~history_mode ~new_head ~new_head_metadata - ~cementing_highwatermark = +let merge_stores ?(cycle_size_limit = default_cycle_size_limit) block_store + ~(on_error : tztrace -> unit tzresult Lwt.t) ~finalizer ~history_mode + ~new_head ~new_head_metadata ~cementing_highwatermark = let open Lwt_result_syntax in let* () = fail_when block_store.readonly Cannot_write_in_readonly in (* Do not allow multiple merges: force waiting for a potential @@ -1441,6 +1442,7 @@ let merge_stores block_store ~(on_error : tztrace -> unit tzresult Lwt.t) let* new_ro_store, new_savepoint, new_caboose = create_merging_thread block_store + ~cycle_size_limit ~history_mode ~old_ro_store ~old_rw_store diff --git a/src/lib_store/unix/block_store.mli b/src/lib_store/unix/block_store.mli index 602d7f27a012..b12bb60381ce 100644 --- a/src/lib_store/unix/block_store.mli +++ b/src/lib_store/unix/block_store.mli @@ -271,6 +271,9 @@ val move_floating_store : in [block_store] to finish if any. *) val await_merging : block_store -> unit Lwt.t +(** Default cemented cycles maximum size. I.e.: [2^16 - 1] *) +val default_cycle_size_limit : int32 + (** (* TODO UPDATE MERGE DOC *) [merge_stores block_store ?finalizer ~nb_blocks_to_preserve @@ -294,11 +297,15 @@ val await_merging : block_store -> unit Lwt.t If a merge thread is already occurring, this function will first wait for the previous merge to be done. + The cemented cycles will have a max size of [cycle_size_limit] + blocks which default to [default_cycle_size_limit]. + {b Warning} For a given [block_store], the caller must wait for this function termination before calling it again or it may result in concurrent intertwining causing the cementing to be out of order. *) val merge_stores : + ?cycle_size_limit:int32 -> block_store -> on_error:(tztrace -> unit tzresult Lwt.t) -> finalizer:(int32 -> unit tzresult Lwt.t) -> diff --git a/src/lib_store/unix/test/test_block_store.ml b/src/lib_store/unix/test/test_block_store.ml index 9a899df2fada..cb708e2b4c2d 100644 --- a/src/lib_store/unix/test/test_block_store.ml +++ b/src/lib_store/unix/test/test_block_store.ml @@ -468,8 +468,9 @@ let test_merge_with_branches block_store = in assert_absence_in_block_store block_store (List.flatten blocks_to_gc) -let perform_n_cycles_merge ?(cycle_length = 10) block_store history_mode - nb_cycles = +let perform_n_cycles_merge ?(cycle_length = 10) + ?(cycle_size_limit = Block_store.default_cycle_size_limit) block_store + history_mode nb_cycles = let open Lwt_result_syntax in let*! cycles, head = make_n_initial_consecutive_cycles block_store ~cycle_length ~nb_cycles @@ -486,6 +487,7 @@ let perform_n_cycles_merge ?(cycle_length = 10) block_store history_mode let* () = Block_store.merge_stores block_store + ~cycle_size_limit ~on_error:(fun err -> Assert.fail_msg "merging failed: %a" pp_print_trace err) ~finalizer:(fun _ -> return_unit) @@ -695,7 +697,41 @@ let test_rolling_2_merge block_store = Assert.Int32.equal ~msg:"caboose" expected_savepoint (snd caboose) ; return_unit -let wrap_test ?(keep_dir = false) (name, g) = +let test_split_cycle_merge block_store = + let open Lwt_result_syntax in + let* cycles = + perform_n_cycles_merge + ~cycle_size_limit:3l + ~cycle_length:5 + block_store + Archive + 5 + in + (* All blocks w/ metadata should be present *) + let* () = + assert_presence_in_block_store + ~with_metadata:true + block_store + (List.concat cycles) + in + let cemented_cycles = + Cemented_block_store.cemented_blocks_files + (Block_store.cemented_block_store block_store) + |> function + | Some a -> Array.to_list a + | None -> [] + in + let () = + (* Ensure that cemented cycles are split as 2 or 3 block chunks *) + List.iter + (fun {Cemented_block_store.start_level; end_level; _} -> + let diff_nb = succ Int32.(sub end_level start_level |> to_int) in + assert (diff_nb >= 2 && diff_nb <= 3)) + cemented_cycles + in + return_unit + +let wrap_test ?(keep_dir = true) (name, g) = let open Lwt_result_syntax in let f dir_path = let genesis_block = @@ -759,6 +795,7 @@ let tests : string * unit Alcotest_lwt.test_case list = ("consecutive merge (Full + 2 cycles)", test_full_2_merge); ("consecutive merge (Rolling + 0 cycles)", test_rolling_0_merge); ("consecutive merge (Rolling + 2 cycles)", test_rolling_2_merge); + ("split cycle merge", test_split_cycle_merge); ] in ("block store", test_cases) -- GitLab