From 8291fb395ed4232e264c7bc76c02941c6409ca15 Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Fri, 6 Sep 2024 12:39:24 +0200 Subject: [PATCH 1/2] Node/Store: introduce progress status when inspecting metadata --- src/bin_node/node_storage_command.ml | 12 ++++++--- src/lib_store/unix/cemented_block_store.ml | 30 ++++++++++++++-------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/bin_node/node_storage_command.ml b/src/bin_node/node_storage_command.ml index 619a0bba9e4c..f432bbc1fb82 100644 --- a/src/bin_node/node_storage_command.ml +++ b/src/bin_node/node_storage_command.ml @@ -465,11 +465,15 @@ module Term = struct } res in - let report = - Data_encoding.Json.construct overall_stats_encoding overall_stats - in let* () = - Tezos_stdlib_unix.Lwt_utils_unix.Json.write_file report_path report + Animation.three_dots + ~progress_display_mode:Always + ~msg:"Generating report" + (fun () -> + let report = + Data_encoding.Json.construct overall_stats_encoding overall_stats + in + Tezos_stdlib_unix.Lwt_utils_unix.Json.write_file report_path report) in let*! () = Event.(emit stat_metadata_report_generated) report_path in let*! () = Store.close_store store in diff --git a/src/lib_store/unix/cemented_block_store.ml b/src/lib_store/unix/cemented_block_store.ml index cbf3380e5933..c33499c4ed73 100644 --- a/src/lib_store/unix/cemented_block_store.ml +++ b/src/lib_store/unix/cemented_block_store.ml @@ -1172,16 +1172,26 @@ let stat_metadata_cycles cemented_store = match cemented_store.cemented_blocks_files with | Some files -> let files = Array.to_list files in - List.map_es - (fun metadata_file -> - let* res = - map_over_metadata_file - cemented_store.cemented_blocks_dir - metadata_file - ~f:stat_metadata - in - return (Naming.(metadata_file.file |> file_path), res)) - files + let nb_cycles = List.length files in + let* res = + Animation.display_progress + ~pp_print_step:(fun fmt i -> + Format.fprintf fmt "Processing metadata %d/%d" i nb_cycles) + ~progress_display_mode:Always + (fun notify -> + List.map_es + (fun metadata_file -> + let*! () = notify () in + let* res = + map_over_metadata_file + cemented_store.cemented_blocks_dir + metadata_file + ~f:stat_metadata + in + return (Naming.(metadata_file.file |> file_path), res)) + files) + in + return res | None -> tzfail (Inconsistent_store_state "cannot find any metadata file") let get_and_upgrade_offsets fd nb_blocks = -- GitLab From 52f6ba4a28c4d3968f6bf36753c6ba1329bffd3c Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Fri, 6 Sep 2024 12:40:03 +0200 Subject: [PATCH 2/2] Store: introduce block_level field for metadata stats utilities --- src/bin_node/node_storage_command.ml | 62 +++++++++++++++++++++- src/lib_store/shared/store_types.ml | 2 + src/lib_store/shared/store_types.mli | 2 + src/lib_store/unix/cemented_block_store.ml | 21 ++++++-- 4 files changed, 82 insertions(+), 5 deletions(-) diff --git a/src/bin_node/node_storage_command.ml b/src/bin_node/node_storage_command.ml index f432bbc1fb82..497e0d8fe6d8 100644 --- a/src/bin_node/node_storage_command.ml +++ b/src/bin_node/node_storage_command.ml @@ -296,9 +296,46 @@ module Term = struct let () = Format.printf "%a@." Context_hash.pp head_context_hash in return_unit) + type block_stats = { + level : Int32.t; + block_metadata_size : Int64.t; + manager_operation_metadata_sizes : int list; + too_large_operation_metadata : bool; + } + + let block_stats_encoding = + let open Data_encoding in + conv + (fun { + level; + block_metadata_size; + manager_operation_metadata_sizes; + too_large_operation_metadata; + } -> + ( level, + block_metadata_size, + manager_operation_metadata_sizes, + too_large_operation_metadata )) + (fun ( level, + block_metadata_size, + manager_operation_metadata_sizes, + too_large_operation_metadata ) -> + { + level; + block_metadata_size; + manager_operation_metadata_sizes; + too_large_operation_metadata; + }) + (obj4 + (req "level" int32) + (req "block_metadata_size" int64) + (req "manager_operation_metadata_sizes" (list int31)) + (req "too_large_operation_metadata" bool)) + type cycle_stats = { name : string; block_count : int; + block_stats : block_stats list; block_metadata_size : Int64.t; operation_metadata_size : Int64.t; too_large_operation_metadata_count : Int64.t; @@ -310,30 +347,35 @@ module Term = struct (fun { name; block_count; + block_stats; block_metadata_size; operation_metadata_size; too_large_operation_metadata_count; } -> ( name, block_count, + block_stats, block_metadata_size, operation_metadata_size, too_large_operation_metadata_count )) (fun ( name, block_count, + block_stats, block_metadata_size, operation_metadata_size, too_large_operation_metadata_count ) -> { name; block_count; + block_stats; block_metadata_size; operation_metadata_size; too_large_operation_metadata_count; }) - (obj5 + (obj6 (req "name" string) (req "block_count" int31) + (req "block_stats" (list block_stats_encoding)) (req "block_metadata_size" int64) (req "operation_metadata_size" int64) (req "too_large_operation_metadata_count" int64)) @@ -411,15 +453,30 @@ module Term = struct (fun { name; block_count; + block_stats; block_metadata_size; operation_metadata_size; too_large_operation_metadata_count; } - stats -> + (stats : Tezos_store_shared.Store_types.metadata_stat) -> + let new_block_stats = + { + level = stats.block_level; + block_metadata_size = stats.operation_metadata_size; + manager_operation_metadata_sizes = + stats.manager_operation_metadata_sizes; + too_large_operation_metadata = + not + (Int64.equal + stats.too_large_operation_metadata_count + 0L); + } + in Tezos_store_shared.Store_types. { name; block_count; + block_stats = new_block_stats :: block_stats; block_metadata_size = Int64.add block_metadata_size stats.block_metadata_size; operation_metadata_size = @@ -434,6 +491,7 @@ module Term = struct { name = metadata_file; block_count = List.length cycle; + block_stats = []; block_metadata_size = 0L; operation_metadata_size = 0L; too_large_operation_metadata_count = 0L; diff --git a/src/lib_store/shared/store_types.ml b/src/lib_store/shared/store_types.ml index 4a20d92b663a..e1199289956c 100644 --- a/src/lib_store/shared/store_types.ml +++ b/src/lib_store/shared/store_types.ml @@ -353,8 +353,10 @@ module Protocol_levels = struct end type metadata_stat = { + block_level : Int32.t; block_metadata_size : Int64.t; operation_metadata_arity : int list; operation_metadata_size : Int64.t; + manager_operation_metadata_sizes : int list; too_large_operation_metadata_count : Int64.t; } diff --git a/src/lib_store/shared/store_types.mli b/src/lib_store/shared/store_types.mli index 7a1b8fbec05e..1af4e3fe2577 100644 --- a/src/lib_store/shared/store_types.mli +++ b/src/lib_store/shared/store_types.mli @@ -184,8 +184,10 @@ module Protocol_levels : sig end type metadata_stat = { + block_level : Int32.t; block_metadata_size : Int64.t; operation_metadata_arity : int list; operation_metadata_size : Int64.t; + manager_operation_metadata_sizes : int list; too_large_operation_metadata_count : Int64.t; } diff --git a/src/lib_store/unix/cemented_block_store.ml b/src/lib_store/unix/cemented_block_store.ml index c33499c4ed73..a19b94820822 100644 --- a/src/lib_store/unix/cemented_block_store.ml +++ b/src/lib_store/unix/cemented_block_store.ml @@ -1099,7 +1099,7 @@ let check_indexes_consistency ?(post_step = fun () -> Lwt.return_unit) (* For each given block metadata, returns the Store_types.metadata_stat of it. *) -let stat_metadata (metadata : Block_repr.metadata option) = +let stat_metadata ~block_level (metadata : Block_repr.metadata option) = let open Lwt_result_syntax in match metadata with | Some metadata -> @@ -1127,12 +1127,25 @@ let stat_metadata (metadata : Block_repr.metadata option) = (0L, 0L) metadata.operations_metadata in + let* manager_operation_metadata_sizes = + match metadata.operations_metadata with + | [_; _; _; manager] -> + return + @@ List.map + (function + | Block_validation.Metadata bytes -> Bytes.length bytes + | Block_validation.Too_large_metadata -> 0) + manager + | _ -> return_nil + in return Store_types. { + block_level; block_metadata_size; operation_metadata_arity; operation_metadata_size; + manager_operation_metadata_sizes; too_large_operation_metadata_count; } | None -> tzfail (Inconsistent_store_state "cannot read metadata") @@ -1142,7 +1155,9 @@ let stat_metadata (metadata : Block_repr.metadata option) = let map_over_metadata_file cemented_blocks_dir_path {start_level; end_level; file} ~(f : - Block_repr.metadata option -> Store_types.metadata_stat tzresult Lwt.t) = + block_level:Int32.t -> + Block_repr.metadata option -> + Store_types.metadata_stat tzresult Lwt.t) = let open Lwt_result_syntax in let metadata_file = Naming.( @@ -1157,7 +1172,7 @@ let map_over_metadata_file cemented_blocks_dir_path else let entry = Zip.find_entry fd (Int32.to_string level) in let metadata = Zip.read_entry fd entry in - let* app = f (Block_repr.decode_metadata metadata) in + let* app = f ~block_level:level (Block_repr.decode_metadata metadata) in loop Int32.(add level one) (app :: acc) in let* res = loop start_level [] in -- GitLab