diff --git a/src/lib_scoru_wasm/test/test_encoding.ml b/src/lib_scoru_wasm/test/test_encoding.ml index b3e8268fb7238772d6e11bc82b9ff1cb25d1c798..7e56910e1eeb0692686eba77faefc9079d4c9cce 100644 --- a/src/lib_scoru_wasm/test/test_encoding.ml +++ b/src/lib_scoru_wasm/test/test_encoding.ml @@ -67,7 +67,9 @@ module Map = end) module Merklizer = - Tree_encoding_decoding.Make (Map) (Lazy_vector.LwtIntVector) (Tree) + Tree_encoding_decoding.Make (Map) (Lazy_vector.LwtIntVector) + (Chunked_byte_vector.Lwt) + (Tree) let empty_tree () = let open Lwt_syntax in @@ -208,6 +210,24 @@ let test_lazy_vector () = = Lazy_vector.LwtIntVector.to_string Fun.id decoded_vector) ; return_unit +let test_chunked_byte_vector () = + let open Merklizer in + let open Lwt_result_syntax in + let vector = + Chunked_byte_vector.Lwt.of_string + (String.make 10_000 'a' ^ String.make 10_000 'b') + in + let*! value = Chunked_byte_vector.Lwt.load_byte vector 5L in + assert (Char.chr value = 'a') ; + let*! value = Chunked_byte_vector.Lwt.load_byte vector 10_005L in + assert (Char.chr value = 'b') ; + let*! decoded_vector = encode_decode chunked_byte_vector vector in + let*! value = Chunked_byte_vector.Lwt.load_byte decoded_vector 5L in + assert (Char.chr value = 'a') ; + let*! value = Chunked_byte_vector.Lwt.load_byte decoded_vector 10_005L in + assert (Char.chr value = 'b') ; + return_unit + let tests = [ tztest "String" `Quick test_string; @@ -222,4 +242,5 @@ let tests = `Quick test_add_to_decoded_empty_map; tztest "Lazy vector" `Quick test_lazy_vector; + tztest "Chunked byte vector" `Quick test_chunked_byte_vector; ] diff --git a/src/lib_scoru_wasm/tree_encoding_decoding.ml b/src/lib_scoru_wasm/tree_encoding_decoding.ml index 0c0298e48ace73a71923c90881a93f07dc2ea893..73c350018466fa452daa02994a73e5ed48abd5b9 100644 --- a/src/lib_scoru_wasm/tree_encoding_decoding.ml +++ b/src/lib_scoru_wasm/tree_encoding_decoding.ml @@ -36,6 +36,8 @@ module type S = sig type 'a vector + type chunked_byte_vector + type ('tag, 'a) case module Decoding : Tree_decoding.S with type tree = tree @@ -68,6 +70,10 @@ module type S = sig val lazy_vector : vector_key t -> 'a t -> 'a vector t + val chunk : Chunked_byte_vector.Chunk.t t + + val chunked_byte_vector : chunked_byte_vector t + val case : 'tag -> 'b t -> ('a -> 'b option) -> ('b -> 'a) -> ('tag, 'a) case val tagged_union : 'tag t -> ('tag, 'a) case list -> 'a t @@ -76,12 +82,14 @@ end module Make (M : Lazy_map.S with type 'a effect = 'a Lwt.t) (V : Lazy_vector.S with type 'a effect = 'a Lwt.t) + (C : Chunked_byte_vector.S with type 'a effect = 'a Lwt.t) (T : Tree.S) : S with type tree = T.tree and type 'a map = 'a M.t and type vector_key = V.key - and type 'a vector = 'a V.t = struct + and type 'a vector = 'a V.t + and type chunked_byte_vector = C.t = struct module Encoding = Tree_encoding.Make (T) module Decoding = Tree_decoding.Make (T) module E = Encoding @@ -95,6 +103,8 @@ module Make type 'a map = 'a M.t + type chunked_byte_vector = C.t + type 'a encoding = 'a E.t type 'a decoding = 'a D.t @@ -167,6 +177,29 @@ module Make in {encode; decode} + let chunk = + let open Chunked_byte_vector.Chunk in + conv of_bytes to_bytes (raw []) + + let chunked_byte_vector = + let to_key k = [Int64.to_string k] in + let encode = + E.contramap + (fun vector -> (C.loaded_chunks vector, C.length vector)) + (E.tup2 + (E.lazy_mapping to_key chunk.encode) + (E.value ["length"] Data_encoding.int64)) + in + let decode = + D.map + (fun (get_chunk, len) -> C.create ~get_chunk len) + (let open D.Syntax in + let+ x = D.lazy_mapping to_key chunk.decode + and+ y = D.value ["length"] Data_encoding.int64 in + (x, y)) + in + {encode; decode} + type ('tag, 'a) case = | Case : { tag : 'tag; diff --git a/src/lib_scoru_wasm/tree_encoding_decoding.mli b/src/lib_scoru_wasm/tree_encoding_decoding.mli index f74e56aaeb46efbebc9ba51b27bb779f61c2b585..9dfb0ad2415af9ea86b9e261fb41bc2cd2ce9d81 100644 --- a/src/lib_scoru_wasm/tree_encoding_decoding.mli +++ b/src/lib_scoru_wasm/tree_encoding_decoding.mli @@ -43,6 +43,9 @@ module type S = sig (** The vector structure used. *) type 'a vector + (** The chunked byte vector structure used. *) + type chunked_byte_vector + (** Represents a partial encoder for a specific constructor of a sum-type. *) type ('tag, 'a) case @@ -105,6 +108,12 @@ module type S = sig [enc] for encoding values. *) val lazy_vector : vector_key t -> 'a t -> 'a vector t + (** [chunk] is an encoder for the chunks used by [chunked_by_vector]. *) + val chunk : Chunked_byte_vector.Chunk.t t + + (** [chunked_byte_vector] is an encoder for [chunked_byte_vector]. *) + val chunked_byte_vector : chunked_byte_vector t + (** [case tag enc f] returns a partial encoder that represents a case in a sum-type. The encoder hides the (existentially bound) type of the parameter to the specific case, provided a converter function [f] and @@ -124,9 +133,11 @@ end module Make (M : Lazy_map.S with type 'a effect = 'a Lwt.t) (V : Lazy_vector.S with type 'a effect = 'a Lwt.t) + (C : Chunked_byte_vector.S with type 'a effect = 'a Lwt.t) (T : Tree.S) : S with type tree = T.tree and type 'a map = 'a M.t and type vector_key = V.key and type 'a vector = 'a V.t + and type chunked_byte_vector = C.t diff --git a/src/lib_webassembly/util/chunked_byte_vector.ml b/src/lib_webassembly/util/chunked_byte_vector.ml index 91f9d2d3ff998df724790b16e75382e5f5aaa9ff..92065a83bfd38b40d38fdf9dccada71f09dad670 100644 --- a/src/lib_webassembly/util/chunked_byte_vector.ml +++ b/src/lib_webassembly/util/chunked_byte_vector.ml @@ -31,6 +31,10 @@ module Chunk = struct done ; chunk + let to_bytes chunk = + let len = Array1.size_in_bytes chunk in + Bytes.init len (fun i -> Char.chr @@ Array1.get chunk i) + let num_needed length = if Int64.compare length 0L > 0 then (* [pred length] is used to cover the edge cases where [length] is an exact @@ -82,6 +86,8 @@ module type S = sig val store_bytes : t -> int64 -> bytes -> unit effect + val loaded_chunks : t -> (int64 * Chunk.t) list + module Buffer : sig type vector := t @@ -188,6 +194,9 @@ module Make (Effect : Effect.S) : S with type 'a effect = 'a Effect.t = struct let+ () = store_bytes vector 0L bytes in vector + let loaded_chunks vector = + Vector.Vector.loaded_bindings (Vector.snapshot vector.chunks) + module Buffer = struct type nonrec t = {vector : t; offset : int64} diff --git a/src/lib_webassembly/util/chunked_byte_vector.mli b/src/lib_webassembly/util/chunked_byte_vector.mli index 14196bba9dd94806da26a99ec3ad7d0139a4948e..630c31a6e54449d1f5db96efc0b665d5c2e025a1 100644 --- a/src/lib_webassembly/util/chunked_byte_vector.mli +++ b/src/lib_webassembly/util/chunked_byte_vector.mli @@ -5,6 +5,9 @@ module Chunk : sig (** Create a chunk and copy the given bytes into it. *) val of_bytes : bytes -> t + (** Copy the contents of a chunk into a fresh [bytes]. *) + val to_bytes : t -> bytes + (** Size of a chunk in bytes - with 12 bits of address space the chunk is 4KiB *) val size : int64 @@ -61,6 +64,11 @@ module type S = sig [bytes]. *) val store_bytes : t -> int64 -> bytes -> unit effect + (** [loaded_chunks vector] returns the chunks of [vector] that have + been cached in-memory since [vector] has been created, either by + reading its contents, or by modifying it. *) + val loaded_chunks : t -> (int64 * Chunk.t) list + module Buffer : sig type vector := t