From 1b98b3cb6b58020189fcd821b704d62661e5ed8f Mon Sep 17 00:00:00 2001 From: vbot Date: Thu, 11 Jan 2024 15:24:24 +0100 Subject: [PATCH 1/6] Store: Add a compressed representation of operations Co-authored-by: Albin Coquereau Co-authored-by: vbot --- src/lib_store/shared/block_repr.ml | 137 +++++++++++++++++--- src/lib_store/shared/block_repr.mli | 14 +- src/lib_store/unix/cemented_block_store.ml | 3 +- src/lib_store/unix/cemented_block_store.mli | 2 + src/lib_store/unix/snapshots.ml | 37 ++++-- src/lib_store/unix/store.ml | 58 +++++++-- src/lib_store/unix/store.mli | 4 + src/lib_store/unix/test/test_utils.ml | 9 +- 8 files changed, 220 insertions(+), 44 deletions(-) diff --git a/src/lib_store/shared/block_repr.ml b/src/lib_store/shared/block_repr.ml index 77e3954f6015..0550f7d0667d 100644 --- a/src/lib_store/shared/block_repr.ml +++ b/src/lib_store/shared/block_repr.ml @@ -25,9 +25,11 @@ open Store_errors +type encoded_operations = Raw of Operation.t list list | Compressed of bytes + type contents = { header : Block_header.t; - operations : Operation.t list list; + operations : encoded_operations; block_metadata_hash : Block_metadata_hash.t option; operations_metadata_hashes : Operation_metadata_hash.t list list option; } @@ -82,7 +84,7 @@ let create_genesis_block ~genesis context = let contents = { header; - operations = []; + operations = Raw []; block_metadata_hash = None; operations_metadata_hashes = None; } @@ -99,6 +101,70 @@ let create_genesis_block ~genesis context = in {hash = genesis.block; contents; metadata} +let raw_operations_encoding : encoded_operations Data_encoding.t = + let open Data_encoding in + conv + (function Raw oprerations -> oprerations | Compressed _ -> assert false) + (fun operations -> Raw operations) + (list (list (dynamic_size Operation.encoding))) + +let encoded_operations_encoding : encoded_operations Data_encoding.t = + let open Data_encoding in + let raw_op_encoding = list (list (dynamic_size Operation.encoding)) in + let magic_byte = Char.chr 0xff in + splitted + ~binary: + (conv + (function + | Raw operations -> + Data_encoding.Binary.to_bytes_exn raw_op_encoding operations + | Compressed bytes -> + Bytes.concat Bytes.empty [Bytes.make 1 magic_byte; bytes]) + (fun b -> + if Bytes.length b > 0 then + if Bytes.get b 0 = magic_byte then Compressed b + else Raw (Data_encoding.Binary.of_bytes_exn raw_op_encoding b) + else Raw []) + bytes) + ~json: + (union + @@ [ + case + ~title:"Raw operations" + (Tag 0) + (obj1 (req "raw" raw_op_encoding)) + (function Raw ops -> Some ops | _ -> None) + (fun ops -> Raw ops); + case + ~title:"Compressed operations" + (Tag 1) + (obj1 (req "compressed" bytes)) + (function Compressed b -> Some b | _ -> None) + (fun b -> Compressed b); + ]) + +let encoded_contents_encoding = + let open Data_encoding in + def "store.block_repr.encoded_contents" + @@ conv + (fun { + header; + operations; + block_metadata_hash; + operations_metadata_hashes; + } -> + (header, operations, block_metadata_hash, operations_metadata_hashes)) + (fun (header, operations, block_metadata_hash, operations_metadata_hashes) + -> + {header; operations; block_metadata_hash; operations_metadata_hashes}) + (obj4 + (req "header" (dynamic_size Block_header.encoding)) + (req "operations" encoded_operations_encoding) + (opt "block_metadata_hash" Block_metadata_hash.encoding) + (opt + "operations_metadata_hashes" + (list (list Operation_metadata_hash.encoding)))) + let contents_encoding = let open Data_encoding in def "store.block_repr.contents" @@ -115,7 +181,7 @@ let contents_encoding = {header; operations; block_metadata_hash; operations_metadata_hashes}) (obj4 (req "header" (dynamic_size Block_header.encoding)) - (req "operations" (list (list (dynamic_size Operation.encoding)))) + (req "operations" raw_operations_encoding) (opt "block_metadata_hash" Block_metadata_hash.encoding) (opt "operations_metadata_hashes" @@ -193,6 +259,19 @@ let legacy_metadata_encoding : legacy_metadata Data_encoding.t = (req "legacy_block_metadata" bytes) (req "legacy_operations_metadata" (list (list bytes)))) +let _encoded_block_encoding = + let open Data_encoding in + def "store.encoded_block_repr" + @@ conv + (fun {hash; contents; metadata} -> (hash, contents, metadata)) + (fun (hash, contents, metadata) -> {hash; contents; metadata}) + (dynamic_size + ~kind:`Uint30 + (obj3 + (req "hash" Block_hash.encoding) + (req "contents" encoded_contents_encoding) + (varopt "metadata" metadata_encoding))) + let encoding = let open Data_encoding in def "store.block_repr" @@ -243,10 +322,16 @@ let with_metadata [@@ocaml.inline] let contents_equal c1 c2 = + let encoded_operations_eq eo1 eo2 = + match (eo1, eo2) with + | Compressed b, Compressed b' -> Bytes.equal b b' + | Raw os, Raw os' -> List.equal (List.equal Operation.equal) os os' + | _ -> false + in with_contents c1 @@ fun h1 o1 b1 omh1 -> with_contents c2 @@ fun h2 o2 b2 omh2 -> Block_header.equal h1 h2 - && List.equal (List.equal Operation.equal) o1 o2 + && encoded_operations_eq o1 o2 && Option.equal Block_metadata_hash.equal b1 b2 && Option.equal (List.equal (List.equal Operation_metadata_hash.equal)) @@ -319,7 +404,7 @@ let block_metadata metadata = metadata.block_metadata let operations_metadata metadata = metadata.operations_metadata -let check_block_consistency ?genesis_hash ?pred_block block = +let check_block_consistency ?genesis_hash ?pred_block block protocol_levels = let open Lwt_result_syntax in let block_header = header block in let block_hash = hash block in @@ -353,21 +438,35 @@ let check_block_consistency ?genesis_hash ?pred_block block = computed_hash = predecessor block; }) in - let computed_operations_hash = - Operation_list_list_hash.compute - (List.map - Operation_list_hash.compute - (List.map (List.map Operation.hash) (operations block))) - in - let* () = - fail_unless - (Operation_list_list_hash.equal - computed_operations_hash - (operations_hash block)) - (Store_errors.Inconsistent_operations_hash - {expected = operations_hash block; got = computed_operations_hash}) + let check_raw_operations_hash operations = + let computed_operations_hash = + Operation_list_list_hash.compute + (List.map + Operation_list_hash.compute + (List.map (List.map Operation.hash) operations)) + in + let* () = + fail_unless + (Operation_list_list_hash.equal + computed_operations_hash + (operations_hash block)) + (Store_errors.Inconsistent_operations_hash + {expected = operations_hash block; got = computed_operations_hash}) + in + return_unit in - return_unit + match operations block with + | Raw operations -> check_raw_operations_hash operations + | Compressed _ -> ( + let protocol_level = proto_level block in + let protocol_info_opt : Store_types.Protocol_levels.protocol_info option = + Store_types.Protocol_levels.find protocol_level protocol_levels + in + match protocol_info_opt with + | None -> tzfail (Cannot_find_protocol protocol_level) + | Some _ -> + (* TBD *) + assert false) let convert_legacy_metadata (legacy_metadata : legacy_metadata) : metadata = let { diff --git a/src/lib_store/shared/block_repr.mli b/src/lib_store/shared/block_repr.mli index f6685fddbc17..084f43eb2c62 100644 --- a/src/lib_store/shared/block_repr.mli +++ b/src/lib_store/shared/block_repr.mli @@ -29,12 +29,14 @@ open Store_types (** {1 Type definitions and encodings} *) +type encoded_operations = Raw of Operation.t list list | Compressed of bytes + (** The type for the effective [contents] of a block is its header and the [operations] it contains. Their metadata hashes are also present. *) type contents = { header : Block_header.t; - operations : Operation.t list list; + operations : encoded_operations; block_metadata_hash : Block_metadata_hash.t option; operations_metadata_hashes : Operation_metadata_hash.t list list option; } @@ -102,6 +104,8 @@ val legacy_encoding : legacy_block Data_encoding.t contains metadata. *) val create_genesis_block : genesis:Genesis.t -> Context_hash.t -> t +val encoded_operations_encoding : encoded_operations Data_encoding.t + (** Encoding for {!type-contents}. *) val contents_encoding : contents Data_encoding.t @@ -134,7 +138,7 @@ val hash : t -> Block_hash.t (** [operations block] returns the list of list of operations contained in the [block]. *) -val operations : t -> Operation.t list list +val operations : t -> encoded_operations (** {2 Block header accessors} *) @@ -191,7 +195,11 @@ val operations_metadata : - Are the stored operations hashes consistent regarding the stored operations hashes? *) val check_block_consistency : - ?genesis_hash:Block_hash.t -> ?pred_block:t -> t -> unit tzresult Lwt.t + ?genesis_hash:Block_hash.t -> + ?pred_block:t -> + t -> + Protocol_levels.protocol_info Protocol_levels.t -> + unit tzresult Lwt.t (** [decode_metadata data] decodes metadata from [data] encoded either with the new encoding or the legacy one. *) diff --git a/src/lib_store/unix/cemented_block_store.ml b/src/lib_store/unix/cemented_block_store.ml index 0f377c2fca54..bb6a9541d870 100644 --- a/src/lib_store/unix/cemented_block_store.ml +++ b/src/lib_store/unix/cemented_block_store.ml @@ -964,7 +964,7 @@ let iter_cemented_file f ({file; _} as cemented_blocks_file) = (Printexc.to_string exn)) let check_indexes_consistency ?(post_step = fun () -> Lwt.return_unit) - ?genesis_hash cemented_store = + ?genesis_hash cemented_store ~protocol_levels = let open Lwt_result_syntax in match cemented_store.cemented_blocks_files with | None -> return_unit @@ -1042,6 +1042,7 @@ let check_indexes_consistency ?(post_step = fun () -> Lwt.return_unit) ?genesis_hash ?pred_block block + protocol_levels in let level = Block_repr.level block in let hash = Block_repr.hash block in diff --git a/src/lib_store/unix/cemented_block_store.mli b/src/lib_store/unix/cemented_block_store.mli index ad6d6b70ed29..ccf8f78496b3 100644 --- a/src/lib_store/unix/cemented_block_store.mli +++ b/src/lib_store/unix/cemented_block_store.mli @@ -297,4 +297,6 @@ val check_indexes_consistency : ?post_step:(unit -> unit Lwt.t) -> ?genesis_hash:Block_hash.t -> t -> + protocol_levels: + Store_types.Protocol_levels.protocol_info Store_types.Protocol_levels.t -> unit tzresult Lwt.t diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index 77c44f719827..b8ba613d1616 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -3126,6 +3126,7 @@ module type IMPORTER = sig val restore_floating_blocks : t -> Block_hash.t -> + protocol_levels:Protocol_levels.protocol_info Protocol_levels.t -> (unit tzresult Lwt.t * Block_repr.block Lwt_stream.t) tzresult Lwt.t val close : t -> unit Lwt.t @@ -3432,7 +3433,7 @@ module Raw_importer : IMPORTER = struct let* () = Lwt_utils_unix.copy_file ~src ~dst in return_ok_unit - let restore_floating_blocks t genesis_hash = + let restore_floating_blocks t genesis_hash ~protocol_levels = let open Lwt_result_syntax in let floating_blocks_file = Naming.(snapshot_floating_blocks_file t.snapshot_dir |> file_path) @@ -3453,7 +3454,11 @@ module Raw_importer : IMPORTER = struct else let*! block, len_read = Block_repr_unix.read_next_block_exn fd in let* () = - Block_repr.check_block_consistency ~genesis_hash ?pred_block block + Block_repr.check_block_consistency + ~genesis_hash + ?pred_block + block + protocol_levels in let*! () = bounded_push#push block in loop (nb_bytes_left - len_read) @@ -3809,7 +3814,7 @@ module Tar_importer : IMPORTER = struct in return_unit - let restore_floating_blocks t genesis_hash = + let restore_floating_blocks t genesis_hash ~protocol_levels = let open Lwt_result_syntax in let*! o = Onthefly.get_file @@ -3830,7 +3835,11 @@ module Tar_importer : IMPORTER = struct Block_repr_unix.read_next_block_exn floating_blocks_file_fd in let* () = - Block_repr.check_block_consistency ~genesis_hash ?pred_block block + Block_repr.check_block_consistency + ~genesis_hash + ?pred_block + block + protocol_levels in let*! () = bounded_push#push block in loop Int64.(sub nb_bytes_left (of_int len_read)) @@ -3888,7 +3897,7 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct let close = Importer.close let restore_cemented_blocks ?(check_consistency = true) ~dst_chain_dir - ~genesis_hash ~progress_display_mode snapshot_importer = + ~genesis_hash ~progress_display_mode snapshot_importer ~protocol_levels = let open Lwt_result_syntax in let*! () = Importer.restore_cemented_indexes snapshot_importer in let* cemented_files = Importer.load_cemented_files snapshot_importer in @@ -3954,14 +3963,18 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct Cemented_block_store.check_indexes_consistency ~post_step:notify ~genesis_hash - cemented_store) + cemented_store + ~protocol_levels) else return_unit in Cemented_block_store.close cemented_store ; return_unit - let read_floating_blocks snapshot_importer ~genesis_hash = - Importer.restore_floating_blocks snapshot_importer genesis_hash + let read_floating_blocks snapshot_importer ~genesis_hash ~protocol_levels = + Importer.restore_floating_blocks + snapshot_importer + genesis_hash + ~protocol_levels let restore_protocols snapshot_importer progress_display_mode = let open Lwt_result_syntax in @@ -4360,9 +4373,13 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct ~dst_chain_dir ~genesis_hash:genesis.block ~progress_display_mode + ~protocol_levels in let* reading_thread, floating_blocks_stream = - read_floating_blocks snapshot_importer ~genesis_hash:genesis.block + read_floating_blocks + snapshot_importer + ~genesis_hash:genesis.block + ~protocol_levels in let { Block_validation.validation_store; @@ -4375,7 +4392,7 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct let contents = { Block_repr.header = block_data.block_header; - operations = block_data.operations; + operations = Raw block_data.operations; block_metadata_hash = snd block_metadata; operations_metadata_hashes = (match ops_metadata with diff --git a/src/lib_store/unix/store.ml b/src/lib_store/unix/store.ml index af1d55190733..7a410fbc15c4 100644 --- a/src/lib_store/unix/store.ml +++ b/src/lib_store/unix/store.ml @@ -238,6 +238,8 @@ module Block = struct type t = block + type read_block = block + type metadata = Block_repr.metadata = { message : string option; max_operations_ttl : int; @@ -297,7 +299,24 @@ module Block = struct let genesis = genesis chain_store in Block_hash.equal hash genesis.Genesis.block - let read_block {block_store; _} ?(distance = 0) hash = + let uncompress_operations chain_state protocol_level encoded_operations : + Operation.t list list tzresult Lwt.t = + let open Lwt_result_syntax in + match encoded_operations with + | Block_repr.Raw ops -> return ops + | Compressed b -> + Shared.locked_use chain_state (fun chain_state -> + let*! proto_levels = + Stored_data.get chain_state.protocol_levels_data + in + match Protocol_levels.find protocol_level proto_levels with + | None -> assert false + | Some proto_hash -> + let find_plugin_uncompress _ = assert false in + let uncompress_f _ = find_plugin_uncompress proto_hash in + return (uncompress_f b)) + + let read_block {block_store; chain_state; _} ?(distance = 0) hash = let open Lwt_result_syntax in let* o = Block_store.read_block @@ -307,7 +326,20 @@ module Block = struct in match o with | None -> tzfail @@ Block_not_found {hash; distance} - | Some block -> return block + | Some block -> + let* uncompressed_operations = + uncompress_operations + chain_state + block.contents.header.Block_header.shell.proto_level + block.contents.operations + in + return + { + block with + contents = + Block_repr. + {block.contents with operations = Raw uncompressed_operations}; + } let read_block_metadata ?(distance = 0) chain_store hash = Block_store.read_block_metadata @@ -546,10 +578,11 @@ module Block = struct known_invalid Store_errors.(Cannot_store_block (hash, Invalid_block)) in + let may_compress_operations ops = Block_repr.Raw ops in let contents = { Block_repr.header = block_header; - operations; + operations = may_compress_operations operations; block_metadata_hash = snd block_metadata; operations_metadata_hashes = (match ops_metadata with @@ -623,7 +656,7 @@ module Block = struct contents = { header = block_header; - operations; + operations = Raw operations; block_metadata_hash = None; operations_metadata_hashes = None; }; @@ -802,7 +835,14 @@ module Block = struct let header blk = Block_repr.header blk - let operations blk = Block_repr.operations blk + let raw_operations_exn blk = + match Block_repr.operations blk with + | Raw ops -> ops + | Compressed _ -> + (* This cannot happen as the `read_block` only has the "Raw" option for operations *) + assert false + + let operations blk = raw_operations_exn blk let shell_header blk = Block_repr.shell_header blk @@ -864,7 +904,7 @@ module Block = struct let operations_path block i = if i < 0 || validation_passes block <= i then invalid_arg "operations_path" ; - let ops = operations block in + let ops = raw_operations_exn block in let hashes = List.(map (map Operation.hash)) ops in let path = compute_operation_path hashes in (List.nth ops i |> WithExceptions.Option.get ~loc:__LOC__, path i) @@ -872,13 +912,13 @@ module Block = struct let operations_hashes_path block i = if i < 0 || (header block).shell.validation_passes <= i then invalid_arg "operations_hashes_path" ; - let opss = operations block in + let opss = raw_operations_exn block in let hashes = List.(map (map Operation.hash)) opss in let path = compute_operation_path hashes in (List.nth hashes i |> WithExceptions.Option.get ~loc:__LOC__, path i) let all_operation_hashes block = - List.(map (map Operation.hash)) (operations block) + List.(map (map Operation.hash)) (raw_operations_exn block) end module Chain_traversal = struct @@ -1704,7 +1744,7 @@ module Chain = struct let contents = { Block_repr.header; - operations = []; + operations = Raw []; block_metadata_hash = None; operations_metadata_hashes = None; } diff --git a/src/lib_store/unix/store.mli b/src/lib_store/unix/store.mli index d8ee80d37fcc..3480a10a6f39 100644 --- a/src/lib_store/unix/store.mli +++ b/src/lib_store/unix/store.mli @@ -285,6 +285,10 @@ module Block : sig (** The type alias for a block. *) type block = t + (** All the blocks from [read_block] should have the [operations] field + as `Raw` *) + type read_block = block + (** The type for block's metadata. *) type metadata = Block_repr.metadata = { message : string option; diff --git a/src/lib_store/unix/test/test_utils.ml b/src/lib_store/unix/test/test_utils.ml index ec4a8c2ae0fe..c1a110bc4cec 100644 --- a/src/lib_store/unix/test/test_utils.ml +++ b/src/lib_store/unix/test/test_utils.ml @@ -436,7 +436,7 @@ let make_raw_block ?min_lpbl ?(max_operations_ttl = default_max_operations_ttl) contents = { header; - operations; + operations = Raw operations; block_metadata_hash = (if Random.bool () then Some Block_metadata_hash.zero else None); operations_metadata_hashes = @@ -510,7 +510,12 @@ let store_raw_block chain_store ?resulting_context (raw_block : Block_repr.t) = Store.Block.store_block chain_store ~block_header:(Block_repr.header raw_block) - ~operations:(Block_repr.operations raw_block) + ~operations: + (match Block_repr.operations raw_block with + | Raw ops -> ops + | Compressed _ -> + (* This will not happen because the [make_raw_block] only creates "Raw" operations *) + assert false) validation_result in match r with -- GitLab From 13b8bdf9932bba7e2a3dc93051a08c61e78543ba Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Mon, 8 Jan 2024 18:08:49 +0200 Subject: [PATCH 2/6] Proto: Add plugin to refactor attestations --- src/lib_shell_services/store_errors.ml | 19 +- src/lib_store/shared/block_repr.ml | 41 ++- src/lib_store/shared/block_repr.mli | 4 + src/lib_store/unix/block_repr_unix.ml | 83 +++--- src/lib_store/unix/store.ml | 74 +++-- src/lib_validation/protocol_plugin.ml | 14 + src/lib_validation/protocol_plugin.mli | 14 + src/proto_alpha/lib_plugin/plugin.ml | 1 + .../lib_plugin/plugin_registerer.ml | 8 + src/proto_alpha/lib_plugin/shell_plugin.ml | 279 ++++++++++++++++++ 10 files changed, 464 insertions(+), 73 deletions(-) create mode 100644 src/proto_alpha/lib_plugin/shell_plugin.ml diff --git a/src/lib_shell_services/store_errors.ml b/src/lib_shell_services/store_errors.ml index eefed9183f8d..afd2a78dd334 100644 --- a/src/lib_shell_services/store_errors.ml +++ b/src/lib_shell_services/store_errors.ml @@ -42,6 +42,7 @@ type error += expected : Operation_list_list_hash.t; got : Operation_list_list_hash.t; } + | Uncompress_without_plugin of {protocol_hash : Protocol_hash.t} let () = register_error_kind @@ -189,7 +190,23 @@ let () = (function | Inconsistent_operations_hash {expected; got} -> Some (expected, got) | _ -> None) - (fun (expected, got) -> Inconsistent_operations_hash {expected; got}) + (fun (expected, got) -> Inconsistent_operations_hash {expected; got}) ; + register_error_kind + `Permanent + ~id:"store.uncompress_without_plugin" + ~title:"No plugin to uncompress" + ~description:"No plugin was found. Could not uncompress operations." + ~pp:(fun ppf protocol_hash -> + Format.fprintf + ppf + "Could not find plugin to uncompress for protocol %a." + Protocol_hash.pp + protocol_hash) + Data_encoding.(obj1 (req "protocol_hash" Protocol_hash.encoding)) + (function + | Uncompress_without_plugin {protocol_hash} -> Some protocol_hash + | _ -> None) + (fun protocol_hash -> Uncompress_without_plugin {protocol_hash}) type cemented_store_inconsistency = | Missing_cycle of {low_cycle : string; high_cycle : string} diff --git a/src/lib_store/shared/block_repr.ml b/src/lib_store/shared/block_repr.ml index 0550f7d0667d..1c5325ec5d82 100644 --- a/src/lib_store/shared/block_repr.ml +++ b/src/lib_store/shared/block_repr.ml @@ -118,12 +118,22 @@ let encoded_operations_encoding : encoded_operations Data_encoding.t = (function | Raw operations -> Data_encoding.Binary.to_bytes_exn raw_op_encoding operations - | Compressed bytes -> - Bytes.concat Bytes.empty [Bytes.make 1 magic_byte; bytes]) + | Compressed bytes -> Bytes.cat (Bytes.make 1 magic_byte) bytes) (fun b -> - if Bytes.length b > 0 then - if Bytes.get b 0 = magic_byte then Compressed b - else Raw (Data_encoding.Binary.of_bytes_exn raw_op_encoding b) + let length = Bytes.length in + if length b > 0 then + if Bytes.get b 0 = magic_byte then + (* Get rid of the magic_byte *) + let data = Bytes.sub b 1 (length b - 1) in + Compressed data + else + let b = + let size = + Data_encoding.Binary.to_bytes_exn int31 (length b) + in + Bytes.cat size b + in + Raw (Data_encoding.Binary.of_bytes_exn raw_op_encoding b) else Raw []) bytes) ~json: @@ -259,7 +269,7 @@ let legacy_metadata_encoding : legacy_metadata Data_encoding.t = (req "legacy_block_metadata" bytes) (req "legacy_operations_metadata" (list (list bytes)))) -let _encoded_block_encoding = +let encoded_block_encoding = let open Data_encoding in def "store.encoded_block_repr" @@ conv @@ -457,16 +467,27 @@ let check_block_consistency ?genesis_hash ?pred_block block protocol_levels = in match operations block with | Raw operations -> check_raw_operations_hash operations - | Compressed _ -> ( + | Compressed b -> ( let protocol_level = proto_level block in let protocol_info_opt : Store_types.Protocol_levels.protocol_info option = Store_types.Protocol_levels.find protocol_level protocol_levels in match protocol_info_opt with | None -> tzfail (Cannot_find_protocol protocol_level) - | Some _ -> - (* TBD *) - assert false) + | Some {protocol; _} -> + let* uncompressed_operations = + match Protocol_plugin.find_shell protocol with + | Some (module Shell_plugin) -> + return + @@ Data_encoding.Binary.of_bytes_exn + Shell_plugin.refactoring_encoding + b + | None -> + (* There should never be a case where we uncompress Compressed + operations, but we do not have a refactoring_encoding *) + tzfail (Uncompress_without_plugin {protocol_hash = protocol}) + in + check_raw_operations_hash uncompressed_operations) let convert_legacy_metadata (legacy_metadata : legacy_metadata) : metadata = let { diff --git a/src/lib_store/shared/block_repr.mli b/src/lib_store/shared/block_repr.mli index 084f43eb2c62..a083a33c9059 100644 --- a/src/lib_store/shared/block_repr.mli +++ b/src/lib_store/shared/block_repr.mli @@ -115,6 +115,10 @@ val metadata_encoding : metadata Data_encoding.t (** Equality on {!block} *) val equal : t -> t -> bool +(** Encoding for {!t} (and {!block}) which uses the compressed mode for + consensus operations. *) +val encoded_block_encoding : t Data_encoding.t + (** Encoding for {!t} (and {!block}). {b Important} An encoded block is prefixed by 4 bytes which stands diff --git a/src/lib_store/unix/block_repr_unix.ml b/src/lib_store/unix/block_repr_unix.ml index 396d9b24aab7..adc1eba6a583 100644 --- a/src/lib_store/unix/block_repr_unix.ml +++ b/src/lib_store/unix/block_repr_unix.ml @@ -26,46 +26,51 @@ open Block_repr let decode_block_repr encoding block_bytes = - try Data_encoding.Binary.of_bytes_exn encoding block_bytes - with _ -> - (* If the decoding fails, try with the legacy block_repr encoding + (* First, try with the compressed consensus_operations version of the + block encoding *) + try Data_encoding.Binary.of_bytes_exn encoded_block_encoding block_bytes + with _ -> ( + (* If the decoding fails, try with the non-compressed operations encoding *) + try Data_encoding.Binary.of_bytes_exn encoding block_bytes + with _ -> + (* If the decoding fails, try with the legacy block_repr encoding *) - let legacy_block = - Data_encoding.Binary.of_bytes_exn legacy_encoding block_bytes - in - let legacy_metadata = legacy_block.legacy_metadata in - let metadata = - match legacy_metadata with - | Some metadata -> - let { - legacy_message; - legacy_max_operations_ttl; - legacy_last_allowed_fork_level; - legacy_block_metadata; - legacy_operations_metadata; - } = - metadata - in - let operations_metadata = - (List.map (List.map (fun x -> Block_validation.Metadata x))) - legacy_operations_metadata - in - Some - ({ - message = legacy_message; - max_operations_ttl = legacy_max_operations_ttl; - last_preserved_block_level = legacy_last_allowed_fork_level; - block_metadata = legacy_block_metadata; - operations_metadata; - } - : metadata) - | None -> None - in - { - hash = legacy_block.legacy_hash; - contents = legacy_block.legacy_contents; - metadata; - } + let legacy_block = + Data_encoding.Binary.of_bytes_exn legacy_encoding block_bytes + in + let legacy_metadata = legacy_block.legacy_metadata in + let metadata = + match legacy_metadata with + | Some metadata -> + let { + legacy_message; + legacy_max_operations_ttl; + legacy_last_allowed_fork_level; + legacy_block_metadata; + legacy_operations_metadata; + } = + metadata + in + let operations_metadata = + (List.map (List.map (fun x -> Block_validation.Metadata x))) + legacy_operations_metadata + in + Some + ({ + message = legacy_message; + max_operations_ttl = legacy_max_operations_ttl; + last_preserved_block_level = legacy_last_allowed_fork_level; + block_metadata = legacy_block_metadata; + operations_metadata; + } + : metadata) + | None -> None + in + { + hash = legacy_block.legacy_hash; + contents = legacy_block.legacy_contents; + metadata; + }) (* FIXME handle I/O errors *) let read_next_block_exn fd = diff --git a/src/lib_store/unix/store.ml b/src/lib_store/unix/store.ml index 7a410fbc15c4..c9e261793b87 100644 --- a/src/lib_store/unix/store.ml +++ b/src/lib_store/unix/store.ml @@ -310,11 +310,19 @@ module Block = struct Stored_data.get chain_state.protocol_levels_data in match Protocol_levels.find protocol_level proto_levels with - | None -> assert false - | Some proto_hash -> - let find_plugin_uncompress _ = assert false in - let uncompress_f _ = find_plugin_uncompress proto_hash in - return (uncompress_f b)) + | None -> tzfail (Cannot_find_protocol protocol_level) + | Some {protocol; _} -> ( + match Protocol_plugin.find_shell protocol with + | Some (module Shell_plugin) -> + return + @@ Data_encoding.Binary.of_bytes_exn + Shell_plugin.refactoring_encoding + b + | None -> + (* There should never be a case where we uncompress Compressed + operations, but we do not have a refactoring_encoding *) + tzfail + (Uncompress_without_plugin {protocol_hash = protocol}))) let read_block {block_store; chain_state; _} ?(distance = 0) hash = let open Lwt_result_syntax in @@ -475,6 +483,23 @@ module Block = struct operations_data_lengths = to_string ops_metadata; } )) + let protocol_hash chain_store block = + let open Lwt_result_syntax in + Shared.use chain_store.chain_state (fun chain_state -> + let*! protocol_levels = + Stored_data.get chain_state.protocol_levels_data + in + let open Protocol_levels in + let proto_level = Block_repr.proto_level block in + match find proto_level protocol_levels with + | Some {protocol; _} -> return protocol + | None -> tzfail (Cannot_find_protocol proto_level)) + + let protocol_hash_exn chain_store block = + let open Lwt_syntax in + let* r = protocol_hash chain_store block in + match r with Ok ph -> Lwt.return ph | Error _ -> Lwt.fail Not_found + let store_block chain_store ~block_header ~operations validation_result = let open Lwt_result_syntax in let { @@ -606,10 +631,30 @@ module Block = struct } in let block = {Block_repr.hash; contents; metadata} in + let*! (encoded_operations : Block_repr.encoded_operations) = + Lwt.catch + (fun () -> + let*! protocol_hash = protocol_hash_exn chain_store block in + Lwt.return + @@ + match Protocol_plugin.find_shell protocol_hash with + | Some (module Shell_plugin) -> + let b = + Data_encoding.Binary.to_bytes_exn + Shell_plugin.refactoring_encoding + operations + in + Block_repr.Compressed b + | None -> Block_repr.Raw operations) + (fun _ -> Lwt.return @@ Block_repr.Raw operations) + in let* () = Block_store.store_block chain_store.block_store - block + { + block with + contents = {contents with operations = encoded_operations}; + } resulting_context_hash in let*! () = @@ -779,23 +824,6 @@ module Block = struct | Forking _ -> return (status, Some (Block_repr.hash block)) | Not_running -> return (status, None) - let protocol_hash chain_store block = - let open Lwt_result_syntax in - Shared.use chain_store.chain_state (fun chain_state -> - let*! protocol_levels = - Stored_data.get chain_state.protocol_levels_data - in - let open Protocol_levels in - let proto_level = Block_repr.proto_level block in - match find proto_level protocol_levels with - | Some {protocol; _} -> return protocol - | None -> tzfail (Cannot_find_protocol proto_level)) - - let protocol_hash_exn chain_store block = - let open Lwt_syntax in - let* r = protocol_hash chain_store block in - match r with Ok ph -> Lwt.return ph | Error _ -> Lwt.fail Not_found - (** Operations on invalid blocks *) let read_invalid_block_opt {chain_state; _} hash = diff --git a/src/lib_validation/protocol_plugin.ml b/src/lib_validation/protocol_plugin.ml index cce8f4cdab9a..5499c1180c6d 100644 --- a/src/lib_validation/protocol_plugin.ml +++ b/src/lib_validation/protocol_plugin.ml @@ -137,6 +137,12 @@ module type METRICS = sig unit Lwt.t end +module type SHELL = sig + val hash : Protocol_hash.t + + val refactoring_encoding : Operation.t list list Data_encoding.t +end + module Undefined_metrics_plugin (Proto : sig val hash : Protocol_hash.t end) = @@ -152,6 +158,9 @@ let rpc_table : (module RPC) Protocol_hash.Table.t = let metrics_table : (module METRICS) Protocol_hash.Table.t = Protocol_hash.Table.create 5 +let shell_table : (module SHELL) Protocol_hash.Table.t = + Protocol_hash.Table.create 5 + let register_rpc (module Rpc : RPC) = assert (not (Protocol_hash.Table.mem rpc_table Rpc.Proto.hash)) ; Protocol_hash.Table.add rpc_table Rpc.Proto.hash (module Rpc) @@ -159,10 +168,15 @@ let register_rpc (module Rpc : RPC) = let register_metrics (module Metrics : METRICS) = Protocol_hash.Table.replace metrics_table Metrics.hash (module Metrics) +let register_shell (module Shell : SHELL) = + Protocol_hash.Table.replace shell_table Shell.hash (module Shell) + let find_rpc = Protocol_hash.Table.find rpc_table let find_metrics = Protocol_hash.Table.find metrics_table +let find_shell = Protocol_hash.Table.find shell_table + let safe_find_metrics hash = match find_metrics hash with | Some proto_metrics -> Lwt.return proto_metrics diff --git a/src/lib_validation/protocol_plugin.mli b/src/lib_validation/protocol_plugin.mli index f3461dd252e0..739465540b76 100644 --- a/src/lib_validation/protocol_plugin.mli +++ b/src/lib_validation/protocol_plugin.mli @@ -180,6 +180,14 @@ module type METRICS = sig unit Lwt.t end +(** This is a module that is used to refactor the list of operations + and thus to decrease the memory usage of consensus operations. *) +module type SHELL = sig + val hash : Protocol_hash.t + + val refactoring_encoding : Operation.t list list Data_encoding.t +end + (** Emtpy metrics module. All metrics are -1. *) module Undefined_metrics_plugin (P : sig val hash : Protocol_hash.t @@ -195,6 +203,9 @@ val register_rpc : (module RPC) -> unit (** Register a metrics plugin module *) val register_metrics : (module METRICS) -> unit +(** Register a SHELL plugin module *) +val register_shell : (module SHELL) -> unit + (** Retrieves the registered protocol with the provided hash and wraps it together with its validation plugin. @@ -216,5 +227,8 @@ val find_rpc : Protocol_hash.t -> (module RPC) option (** Looks for a metrics plugin module for a specific protocol *) val find_metrics : Protocol_hash.t -> (module METRICS) option +(** Looks for a shell plugin module for a specific protocol *) +val find_shell : Protocol_hash.t -> (module SHELL) option + (** Same as [find_metrics] but returns [Undefined_metrics_plugin] if not found *) val safe_find_metrics : Protocol_hash.t -> (module METRICS) Lwt.t diff --git a/src/proto_alpha/lib_plugin/plugin.ml b/src/proto_alpha/lib_plugin/plugin.ml index 8fb75ce08aad..26154d03b373 100644 --- a/src/proto_alpha/lib_plugin/plugin.ml +++ b/src/proto_alpha/lib_plugin/plugin.ml @@ -30,3 +30,4 @@ module View_helpers = View_helpers module RPC = RPC module Metrics = Metrics_plugin module Script_interpreter_logging = Script_interpreter_logging +module Shell = Shell_plugin diff --git a/src/proto_alpha/lib_plugin/plugin_registerer.ml b/src/proto_alpha/lib_plugin/plugin_registerer.ml index a34184865ed4..55f741bf1490 100644 --- a/src/proto_alpha/lib_plugin/plugin_registerer.ml +++ b/src/proto_alpha/lib_plugin/plugin_registerer.ml @@ -39,8 +39,16 @@ module Metrics = struct let hash = Registerer.Registered.hash end +module Shell = struct + include Plugin.Shell + + let hash = Registerer.Registered.hash +end + let () = Protocol_plugin.register_validation_plugin (module Validation) let () = Protocol_plugin.register_rpc (module RPC) let () = Protocol_plugin.register_metrics (module Metrics) + +let () = Protocol_plugin.register_shell (module Shell) diff --git a/src/proto_alpha/lib_plugin/shell_plugin.ml b/src/proto_alpha/lib_plugin/shell_plugin.ml new file mode 100644 index 000000000000..4397f5fad0ab --- /dev/null +++ b/src/proto_alpha/lib_plugin/shell_plugin.ml @@ -0,0 +1,279 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Trilitech *) +(* *) +(*****************************************************************************) + +(* + The purpose of the plugin is to reduce repeated information that appears + in the consensus operations to obtain a more compact form of storage. + + INPUT: operations : Operation.t trace trace + OUTPUT: refactored_operations : Operation.t trace trace + + operations : Operation.t trace trace + / \ + consensus_operations : Operation.t trace non_consensus_operations + | | + consensus_operations : packed_operation trace | + / \ / + attestations non_attestations / + | | / + refactored_attestation | / + \ | / + \ | / + refactored_operations : refactored_operations + | (encoding) + operation : Operation.t trace trace + + The transformation from attestations : packed_operation trace trace + to a refactored_attestation is done in the following way: + + attestations : + ----------------------------------------------------------- + | - shell | + | - protocol_data : -------------------------------------- | + | | - signature | | + | | - contents : ---------------------- | | + | | | - slot | | | + | | | - level | | | list + | | | - round | | | + | | | - block_payload_hash | | | + | | ---------------------- | | + | | - dal_content | | + | -------------------------------------- | + ----------------------------------------------------------- + + In the `attestations` list, the following fields are duplicated: + - shell + - level + - round + - block_payload_hash + Therefore, the only fields that differ remain: + - signature + - slot + - dal_content + These two will be grouped in a pair, and put into a list, which + will accompany the other 4 common fields in a new type of object + `refactored_attestation`: + + refactored_attestation : + ------------------------------------------------ + | - shell | + | - contents : ---------------------- | + | | - level | | + | | - round | | + | | - block_payload_hash | | + | ---------------------- | + | - unique_contents_list : --------------- | + | | - dal_conte | | + | | - slot | list | + | | - signature | | + | --------------- | + ------------------------------------------------ + + At the end, the refactored_operations is transformed into an Operation.t + trace trace object via the [refactoring_encoding]. +*) + +open Protocol +open Alpha_context + +(* TYPES *) + +type refactored_consensus_content = { + level : Raw_level.t; + round : Round.t; + block_payload_hash : Block_payload_hash.t; +} + +type refactored_attestation = { + shell : Tezos_base.Operation.shell_header; + contents : refactored_consensus_content; + unique_contents_list : (dal_content option * Slot.t * Signature.t) trace; +} + +type refactored_operations = { + refactored_attestation : refactored_attestation option; + consensus_operations : Tezos_base.Operation.t trace; + other_operations : Tezos_base.Operation.t trace trace; +} + +(* TRANSFORMATOR FUNCTIONS *) + +let to_packed_operation {Operation.shell; proto} = + { + shell; + protocol_data = + Data_encoding.Binary.of_bytes_exn + Protocol.operation_data_encoding_with_legacy_attestation_name + proto; + } + +let to_operation {shell; protocol_data} = + { + Operation.shell; + proto = + Data_encoding.Binary.to_bytes_exn + Protocol.operation_data_encoding_with_legacy_attestation_name + protocol_data; + } + +let add_to_refactored_attestation refactored_attestation shell contents + unique_content = + match refactored_attestation with + | None -> Some {unique_contents_list = [unique_content]; shell; contents} + | Some ({unique_contents_list; _} as refactored_attestation) -> + Some + { + refactored_attestation with + unique_contents_list = unique_content :: unique_contents_list; + } + +(* [refactor_attestations] takes a list of consensus operations and returns a + refactored_attestation object which contains all the attestations in a + compacted form, together with the rest of the consensus operations *) +let refactor_attestations consensus_operations = + let (accumulator : refactored_attestation option * packed_operation trace) = + (None, []) + in + let refactored_attestation, non_attestations = + List.fold_left + (fun (refactored_attestation, non_attestations) + {shell; protocol_data = Operation_data protocol_data} -> + match protocol_data.contents with + | Single + (Attestation + { + consensus_content = {slot; level; round; block_payload_hash}; + dal_content; + }) -> ( + let contents = {level; round; block_payload_hash} in + match protocol_data.signature with + | Some signature -> + let unique_content = (dal_content, slot, signature) in + ( add_to_refactored_attestation + refactored_attestation + shell + contents + unique_content, + non_attestations ) + | None -> (refactored_attestation, non_attestations)) + | _ -> + ( refactored_attestation, + {shell; protocol_data = Operation_data protocol_data} + :: non_attestations )) + accumulator + consensus_operations + in + match refactored_attestation with + | None -> (None, non_attestations) + | Some ({unique_contents_list; _} as refactored_attestation) -> + ( Some + { + refactored_attestation with + unique_contents_list = List.rev unique_contents_list; + }, + non_attestations ) + +let expand_refactored_attestation {shell; contents; unique_contents_list} = + let {level; round; block_payload_hash} = contents in + let packed_protocol_data_list = + List.map + (fun (dal_content, slot, signature) -> + let consensus_content = {slot; level; round; block_payload_hash} in + let contents = Single (Attestation {consensus_content; dal_content}) in + Operation_data {signature = Some signature; contents}) + unique_contents_list + in + List.map + (fun packed_protocol_data -> + let proto = + Data_encoding.Binary.to_bytes_exn + Protocol.operation_data_encoding_with_legacy_attestation_name + packed_protocol_data + in + {Operation.shell; proto}) + packed_protocol_data_list + +(* ENCODINGS *) + +let refactored_consensus_content_encoding = + let open Data_encoding in + conv + (fun {level; round; block_payload_hash} -> + (level, round, block_payload_hash)) + (fun (level, round, block_payload_hash) -> + {level; round; block_payload_hash}) + (obj3 + (req "level" Raw_level.encoding) + (req "round" Round.encoding) + (req "block_payload_hash" Block_payload_hash.encoding)) + +let dal_content_encoding = + let open Data_encoding in + conv + (fun {attestation} -> attestation) + (fun attestation -> {attestation}) + (obj1 (req "attestation" Dal.Attestation.encoding)) + +let refactored_attestation_type_encoding = + let open Data_encoding in + conv + (fun {shell; contents; unique_contents_list} -> + (shell, contents, unique_contents_list)) + (fun (shell, contents, unique_contents_list) -> + {shell; contents; unique_contents_list}) + (obj3 + (req "shell" Tezos_base.Operation.shell_header_encoding) + (req "contents" refactored_consensus_content_encoding) + (req + "unique_contents_list" + (list + (dynamic_size + (tup3 + (option dal_content_encoding) + Slot.encoding + Signature.encoding))))) + +let refactored_operations_encoding = + let open Data_encoding in + conv + (fun {refactored_attestation; consensus_operations; other_operations} -> + (refactored_attestation, consensus_operations, other_operations)) + (fun (refactored_attestation, consensus_operations, other_operations) -> + {refactored_attestation; consensus_operations; other_operations}) + (obj3 + (req + "refactored_attestation" + (option refactored_attestation_type_encoding)) + (req + "consensus_operations" + (list (dynamic_size Tezos_base.Operation.encoding))) + (req + "other_operations" + (list (list (dynamic_size Tezos_base.Operation.encoding))))) + +let refactoring_encoding : Tezos_base.Operation.t trace trace Data_encoding.t = + let open Data_encoding in + conv + (fun operations -> + match operations with + | [] -> assert false + | consensus_operations :: other_operations -> + let refactored_attestation, non_attestations = + refactor_attestations + @@ List.map to_packed_operation consensus_operations + in + let consensus_operations = List.map to_operation non_attestations in + {refactored_attestation; consensus_operations; other_operations}) + (fun {refactored_attestation; consensus_operations; other_operations} -> + match refactored_attestation with + | None -> consensus_operations :: other_operations + | Some refactored_attestation -> + let attestations = + expand_refactored_attestation refactored_attestation + in + (attestations @ consensus_operations) :: other_operations) + refactored_operations_encoding -- GitLab From 04bd404b3fa52a60bdf24ae4f115e51cbd0c8fa0 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Fri, 26 Jan 2024 15:16:12 +0000 Subject: [PATCH 3/6] Mocked/Store: Add compressed representation of operations --- src/lib_store/mocked/store.ml | 62 ++++++++++++++++++++++++++++++----- 1 file changed, 54 insertions(+), 8 deletions(-) diff --git a/src/lib_store/mocked/store.ml b/src/lib_store/mocked/store.ml index 16e58846a42c..0005b45b0031 100644 --- a/src/lib_store/mocked/store.ml +++ b/src/lib_store/mocked/store.ml @@ -244,7 +244,32 @@ module Block = struct let genesis = genesis chain_store in Block_hash.equal hash genesis.Genesis.block - let read_block {block_store; _} ?(distance = 0) hash = + let uncompress_operations chain_state protocol_level encoded_operations : + Operation.t list list tzresult Lwt.t = + let open Lwt_result_syntax in + match encoded_operations with + | Block_repr.Raw ops -> return ops + | Compressed b -> + Shared.locked_use chain_state (fun chain_state -> + let*! proto_levels = + Stored_data.get chain_state.protocol_levels_data + in + match Protocol_levels.find protocol_level proto_levels with + | None -> tzfail (Cannot_find_protocol protocol_level) + | Some {protocol; _} -> ( + match Protocol_plugin.find_shell protocol with + | Some (module Shell_plugin) -> + return + @@ Data_encoding.Binary.of_bytes_exn + Shell_plugin.refactoring_encoding + b + | None -> + (* There should never be a case where we uncompress Compressed + operations, but we do not have a refactoring_encoding *) + tzfail + (Uncompress_without_plugin {protocol_hash = protocol}))) + + let read_block {block_store; chain_state; _} ?(distance = 0) hash = let open Lwt_result_syntax in let* o = Block_store.read_block @@ -254,7 +279,20 @@ module Block = struct in match o with | None -> tzfail @@ Block_not_found {hash; distance} - | Some block -> return block + | Some block -> + let* uncompressed_operations = + uncompress_operations + chain_state + block.contents.header.Block_header.shell.proto_level + block.contents.operations + in + return + { + block with + contents = + Block_repr. + {block.contents with operations = Raw uncompressed_operations}; + } let read_block_metadata ?(distance = 0) chain_store hash = Block_store.read_block_metadata @@ -493,10 +531,11 @@ module Block = struct known_invalid Store_errors.(Cannot_store_block (hash, Invalid_block)) in + let may_compress_operations ops = Block_repr.Raw ops in let contents = { Block_repr.header = block_header; - operations; + operations = may_compress_operations operations; block_metadata_hash = snd block_metadata; operations_metadata_hashes = (match ops_metadata with @@ -570,7 +609,7 @@ module Block = struct contents = { header = block_header; - operations; + operations = Raw operations; block_metadata_hash = None; operations_metadata_hashes = None; }; @@ -704,7 +743,14 @@ module Block = struct let header blk = Block_repr.header blk - let operations blk = Block_repr.operations blk + let raw_operations_exn blk = + match Block_repr.operations blk with + | Raw ops -> ops + | Compressed _ -> + (* This cannot happen as the `read_block` only has the "Raw" option for operations *) + assert false + + let operations blk = raw_operations_exn blk let shell_header blk = Block_repr.shell_header blk @@ -766,7 +812,7 @@ module Block = struct let operations_path block i = if i < 0 || validation_passes block <= i then invalid_arg "operations_path" ; - let ops = operations block in + let ops = raw_operations_exn block in let hashes = List.(map (map Operation.hash)) ops in let path = compute_operation_path hashes in (List.nth ops i |> WithExceptions.Option.get ~loc:__LOC__, path i) @@ -774,13 +820,13 @@ module Block = struct let operations_hashes_path block i = if i < 0 || (header block).shell.validation_passes <= i then invalid_arg "operations_hashes_path" ; - let opss = operations block in + let opss = raw_operations_exn block in let hashes = List.(map (map Operation.hash)) opss in let path = compute_operation_path hashes in (List.nth hashes i |> WithExceptions.Option.get ~loc:__LOC__, path i) let all_operation_hashes block = - List.(map (map Operation.hash)) (operations block) + List.(map (map Operation.hash)) (raw_operations_exn block) end module Chain_traversal = struct -- GitLab From c8599647b0dcf8a1fd245480949d402438cb4249 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Wed, 31 Jan 2024 14:24:32 +0000 Subject: [PATCH 4/6] Proto: Nairobi backport (testing purposes) --- src/proto_017_PtNairob/lib_plugin/plugin.ml | 1 + .../lib_plugin/plugin_registerer.ml | 8 + .../lib_plugin/shell_plugin.ml | 258 ++++++++++++++++++ 3 files changed, 267 insertions(+) create mode 100644 src/proto_017_PtNairob/lib_plugin/shell_plugin.ml diff --git a/src/proto_017_PtNairob/lib_plugin/plugin.ml b/src/proto_017_PtNairob/lib_plugin/plugin.ml index 8fb75ce08aad..26154d03b373 100644 --- a/src/proto_017_PtNairob/lib_plugin/plugin.ml +++ b/src/proto_017_PtNairob/lib_plugin/plugin.ml @@ -30,3 +30,4 @@ module View_helpers = View_helpers module RPC = RPC module Metrics = Metrics_plugin module Script_interpreter_logging = Script_interpreter_logging +module Shell = Shell_plugin diff --git a/src/proto_017_PtNairob/lib_plugin/plugin_registerer.ml b/src/proto_017_PtNairob/lib_plugin/plugin_registerer.ml index a34184865ed4..55f741bf1490 100644 --- a/src/proto_017_PtNairob/lib_plugin/plugin_registerer.ml +++ b/src/proto_017_PtNairob/lib_plugin/plugin_registerer.ml @@ -39,8 +39,16 @@ module Metrics = struct let hash = Registerer.Registered.hash end +module Shell = struct + include Plugin.Shell + + let hash = Registerer.Registered.hash +end + let () = Protocol_plugin.register_validation_plugin (module Validation) let () = Protocol_plugin.register_rpc (module RPC) let () = Protocol_plugin.register_metrics (module Metrics) + +let () = Protocol_plugin.register_shell (module Shell) diff --git a/src/proto_017_PtNairob/lib_plugin/shell_plugin.ml b/src/proto_017_PtNairob/lib_plugin/shell_plugin.ml new file mode 100644 index 000000000000..cc8cbbfa4d0a --- /dev/null +++ b/src/proto_017_PtNairob/lib_plugin/shell_plugin.ml @@ -0,0 +1,258 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Trilitech *) +(* *) +(*****************************************************************************) + +(* + The purpose of the plugin is to reduce repeated information that appears + in the consensus operations to obtain a more compact form of storage. + + INPUT: operations : Operation.t trace trace + OUTPUT: refactored_operations : Operation.t trace trace + + operations : Operation.t trace trace + / \ + consensus_operations : Operation.t trace non_consensus_operations + | | + consensus_operations : packed_operation trace | + / \ / + attestations non_attestations / + | | / + refactored_attestation | / + \ | / + \ | / + refactored_operations : refactored_operations + | (encoding) + operation : Operation.t trace trace + + The transformation from attestations : packed_operation trace trace + to a refactored_attestation is done in the following way: + + attestations : + ----------------------------------------------------------- + | - shell | + | - protocol_data : -------------------------------------- | + | | - signature | | + | | - contents : ---------------------- | | + | | | - slot | | | + | | | - level | | | list + | | | - round | | | + | | | - block_payload_hash | | | + | | ---------------------- | | + | -------------------------------------- | + ----------------------------------------------------------- + + In the `attestations` list, the following fields are duplicated: + - shell + - level + - round + - block_payload_hash + Therefore, the only fields that differ remain: + - signature + - slot + These two will be grouped in a pair, and put into a list, which + will accompany the other 4 common fields in a new type of object + `refactored_attestation`: + + refactored_attestation : + ------------------------------------------------ + | - shell | + | - contents : ---------------------- | + | | - level | | + | | - round | | + | | - block_payload_hash | | + | ---------------------- | + | - unique_contents_list : --------------- | + | | - slot | list | + | | - signature | | + | --------------- | + ------------------------------------------------ + + At the end, the refactored_operations is transformed into an Operation.t + trace trace object via the [refactoring_encoding]. +*) + +open Protocol +open Alpha_context + +(* TYPES *) + +type refactored_consensus_content = { + level : Raw_level.t; + round : Round.t; + block_payload_hash : Block_payload_hash.t; +} + +type refactored_attestation = { + shell : Tezos_base.Operation.shell_header; + contents : refactored_consensus_content; + unique_contents_list : (Slot.t * Signature.t) trace; +} + +type refactored_operations = { + refactored_attestation : refactored_attestation option; + consensus_operations : Tezos_base.Operation.t trace; + other_operations : Tezos_base.Operation.t trace trace; +} + +(* TRANSFORMATOR FUNCTIONS *) + +let to_packed_operation {Operation.shell; proto} = + { + shell; + protocol_data = + Data_encoding.Binary.of_bytes_exn Protocol.operation_data_encoding proto; + } + +let to_operation {shell; protocol_data} = + { + Operation.shell; + proto = + Data_encoding.Binary.to_bytes_exn + Protocol.operation_data_encoding + protocol_data; + } + +let add_to_refactored_attestation refactored_attestation shell contents + unique_content = + match refactored_attestation with + | None -> Some {unique_contents_list = [unique_content]; shell; contents} + | Some ({unique_contents_list; _} as refactored_attestation) -> + Some + { + refactored_attestation with + unique_contents_list = unique_content :: unique_contents_list; + } + +(* [refactor_attestations] takes a list of consensus operations and returns a + refactored_attestation object which contains all the attestations in a + compacted form, together with the rest of the consensus operations *) +let refactor_attestations consensus_operations = + let (accumulator : refactored_attestation option * packed_operation trace) = + (None, []) + in + let refactored_attestation, non_attestations = + List.fold_left + (fun (refactored_attestation, non_attestations) + {shell; protocol_data = Operation_data protocol_data} -> + match protocol_data.contents with + | Single (Endorsement {slot; level; round; block_payload_hash}) -> ( + let contents = {level; round; block_payload_hash} in + match protocol_data.signature with + | Some signature -> + let unique_content = (slot, signature) in + ( add_to_refactored_attestation + refactored_attestation + shell + contents + unique_content, + non_attestations ) + | None -> (refactored_attestation, non_attestations)) + | _ -> + ( refactored_attestation, + {shell; protocol_data = Operation_data protocol_data} + :: non_attestations )) + accumulator + consensus_operations + in + match refactored_attestation with + | None -> (None, non_attestations) + | Some ({unique_contents_list; _} as refactored_attestation) -> + ( Some + { + refactored_attestation with + unique_contents_list = List.rev unique_contents_list; + }, + non_attestations ) + +let expand_refactored_attestation {shell; contents; unique_contents_list} = + let {level; round; block_payload_hash} = contents in + let packed_protocol_data_list = + List.map + (fun (slot, signature) -> + let contents = + Single (Endorsement {slot; level; round; block_payload_hash}) + in + Operation_data {signature = Some signature; contents}) + unique_contents_list + in + List.map + (fun packed_protocol_data -> + let proto = + Data_encoding.Binary.to_bytes_exn + Protocol.operation_data_encoding + packed_protocol_data + in + {Operation.shell; proto}) + packed_protocol_data_list + +(* ENCODINGS *) + +let refactored_consensus_content_encoding = + let open Data_encoding in + conv + (fun {level; round; block_payload_hash} -> + (level, round, block_payload_hash)) + (fun (level, round, block_payload_hash) -> + {level; round; block_payload_hash}) + (obj3 + (req "level" Raw_level.encoding) + (req "round" Round.encoding) + (req "block_payload_hash" Block_payload_hash.encoding)) + +let refactored_attestation_type_encoding = + let open Data_encoding in + conv + (fun {shell; contents; unique_contents_list} -> + (shell, contents, unique_contents_list)) + (fun (shell, contents, unique_contents_list) -> + {shell; contents; unique_contents_list}) + (obj3 + (req "shell" Tezos_base.Operation.shell_header_encoding) + (req "contents" refactored_consensus_content_encoding) + (req + "unique_contents_list" + (list (dynamic_size (tup2 Slot.encoding Signature.encoding))))) + +let refactored_operations_encoding = + let open Data_encoding in + conv + (fun {refactored_attestation; consensus_operations; other_operations} -> + (refactored_attestation, consensus_operations, other_operations)) + (fun (refactored_attestation, consensus_operations, other_operations) -> + {refactored_attestation; consensus_operations; other_operations}) + (obj3 + (req + "refactored_attestation" + (option refactored_attestation_type_encoding)) + (req + "consensus_operations" + (list (dynamic_size Tezos_base.Operation.encoding))) + (req + "other_operations" + (list (list (dynamic_size Tezos_base.Operation.encoding))))) + +let refactoring_encoding : Tezos_base.Operation.t trace trace Data_encoding.t = + let open Data_encoding in + conv + (fun operations -> + match operations with + | [] -> assert false + | consensus_operations :: other_operations -> + let refactored_attestation, non_attestations = + refactor_attestations + @@ List.map to_packed_operation consensus_operations + in + let consensus_operations = List.map to_operation non_attestations in + {refactored_attestation; consensus_operations; other_operations}) + (fun {refactored_attestation; consensus_operations; other_operations} -> + match refactored_attestation with + | None -> consensus_operations :: other_operations + | Some refactored_attestation -> + let attestations = + expand_refactored_attestation refactored_attestation + in + (attestations @ consensus_operations) :: other_operations) + refactored_operations_encoding -- GitLab From 2c76a1970990e584805c44f09dc4fbfc7c1cd534 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Wed, 31 Jan 2024 14:24:55 +0000 Subject: [PATCH 5/6] Store: Benchmarking tool --- src/lib_store/unix/store.ml | 106 ++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/src/lib_store/unix/store.ml b/src/lib_store/unix/store.ml index c9e261793b87..5d059633ca8b 100644 --- a/src/lib_store/unix/store.ml +++ b/src/lib_store/unix/store.ml @@ -2607,6 +2607,93 @@ let check_history_mode_consistency chain_dir history_mode = {previous_mode = stored_history_mode; next_mode = history_mode}) else (* Store is not yet initialized. *) return_unit +(* Loads a [distance] blocks into RAM (to avoid reading noise), and + then, flushes it to the disk.*) +let export_raw store ~head ~distance ~raw_blocks_path = + let open Lwt_result_syntax in + Format.printf "%s@." __LOC__ ; + let chain_store = main_chain_store store in + Format.printf "%s@." __LOC__ ; + let rec read_blocks acc block_hash n = + if n = 0 then return acc + else + let () = + Format.printf "Read block with hash : %a@." Block_hash.pp block_hash + in + let* block = Block.read_block chain_store block_hash in + let () = + Format.printf + "Finished reading block with hash : %a@." + Block_hash.pp + block_hash + in + read_blocks (block :: acc) (Block.predecessor block) (n - 1) + in + Format.printf "%s@." __LOC__ ; + let* blocks = read_blocks [] (Block.hash head) distance in + Format.printf "%s@." __LOC__ ; + let*! raw_blocks_fd = + Lwt_unix.openfile raw_blocks_path Unix.[O_CREAT; O_TRUNC; O_RDWR] 0o777 + in + Format.printf "%s@." __LOC__ ; + let start = Time.System.now () in + let* (last_offset : int) = + List.fold_left_es + (fun pos b -> + let bytes = Data_encoding.Binary.to_bytes_exn Block_repr.encoding b in + let len = Bytes.length bytes in + let*! () = Lwt_utils_unix.write_bytes raw_blocks_fd bytes in + return (pos + len)) + 0 + blocks + in + Format.printf "%s@." __LOC__ ; + let*! () = Lwt_unix.fsync raw_blocks_fd in + let stop = Time.System.now () in + let diff = Ptime.diff stop start in + Format.printf + "Wrote %d blocks (%d Kb) in %a@." + distance + (last_offset / 1024) + Ptime.Span.pp + diff ; + let*! () = Lwt_unix.close raw_blocks_fd in + return_unit + +(* Loads all blocks in the givent [raw_blocks_path] file *) +let import_raw ~raw_blocks_path = + let open Lwt_result_syntax in + let*! raw_blocks_fd = + Lwt_unix.openfile raw_blocks_path Unix.[O_RDONLY] 0o777 + in + let count = ref 0 in + let last_offset = ref 0 in + let start = Time.System.now () in + let rec loop offset = + let*! _b, ofs = + Block_repr_unix.pread_block_exn raw_blocks_fd ~file_offset:offset + in + incr count ; + let next = offset + ofs in + last_offset := next ; + loop next + in + let*! _ = + Lwt.catch + (fun () -> loop 0) + (function End_of_file -> return_unit | e -> tzfail e) + in + let stop = Time.System.now () in + let diff = Ptime.diff stop start in + Format.printf + "Read %d blocks (%d Kb) in %a@." + !count + (!last_offset / 1024) + Ptime.Span.pp + diff ; + let*! () = Lwt_unix.close raw_blocks_fd in + return_unit + let init ?patch_context ?commit_genesis ?history_mode ?(readonly = false) ?block_cache_limit ~store_dir ~context_dir ~allow_testchains genesis = let open Lwt_result_syntax in @@ -2691,6 +2778,25 @@ let init ?patch_context ?commit_genesis ?history_mode ?(readonly = false) in Store_metrics.set_invalid_blocks_collector invalid_blocks_collector ; let*! () = Store_events.(emit end_init_store) () in + let bench = + match Sys.getenv_opt "BENCH" with + | Some v -> int_of_string_opt v + | None -> None + in + let* () = + match bench with + | Some distance -> + Format.printf "WIP: break point to do some dirty benchs@." ; + let raw_blocks_path = "./raw_blocks" in + let*! head = Chain.current_head main_chain_store in + Format.printf "%s@." __LOC__ ; + let* () = export_raw store ~head ~distance ~raw_blocks_path in + Format.printf "%s@." __LOC__ ; + let* () = import_raw ~raw_blocks_path in + Format.printf "WIP: bench was run successfully@." ; + assert false + | None -> return_unit + in return store let close_store global_store = -- GitLab From 89345980ff75643af744652981ebe7125db5f6f8 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Wed, 31 Jan 2024 14:52:48 +0000 Subject: [PATCH 6/6] Fixup: Store: Add check for compressed encoding in floating block store --- src/lib_store/unix/floating_block_store.ml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/lib_store/unix/floating_block_store.ml b/src/lib_store/unix/floating_block_store.ml index 36ae46b368b4..82f52c4b3a6f 100644 --- a/src/lib_store/unix/floating_block_store.ml +++ b/src/lib_store/unix/floating_block_store.ml @@ -105,8 +105,13 @@ let locked_write_block floating_store ~offset ~block ~predecessors ~resulting_context_hash = let open Lwt_result_syntax in let* block_bytes = - match Data_encoding.Binary.to_bytes_opt Block_repr.encoding block with - | None -> tzfail (Cannot_encode_block block.Block_repr.hash) + match + Data_encoding.Binary.to_bytes_opt Block_repr.encoded_block_encoding block + with + | None -> ( + match Data_encoding.Binary.to_bytes_opt Block_repr.encoding block with + | None -> tzfail (Cannot_encode_block block.Block_repr.hash) + | Some bytes -> return bytes) | Some bytes -> return bytes in let block_length = Bytes.length block_bytes in -- GitLab