From dd23e80bf0bd2c0d9e4a140be4316f9f22ba4712 Mon Sep 17 00:00:00 2001 From: Hugo Heuzard Date: Thu, 30 Jun 2022 17:21:31 +0200 Subject: [PATCH 1/3] Shell: fix Distributed_db_message.Bounded_encoding.set_* --- manifest/main.ml | 1 + opam/tezos-shell.opam | 1 + src/lib_shell/distributed_db_message.ml | 259 ++++++++++++++---------- src/lib_shell/dune | 2 + 4 files changed, 155 insertions(+), 108 deletions(-) diff --git a/manifest/main.ml b/manifest/main.ml index 890be06c7b03..e29faed72cf1 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -2238,6 +2238,7 @@ let octez_shell = "Tezos: core of `tezos-node` (gossip, validation scheduling, mempool, \ ...)" ~documentation:[Dune.[S "package"; S "tezos-shell"]] + ~inline_tests:ppx_expect ~deps: [ lwt_watcher; diff --git a/opam/tezos-shell.opam b/opam/tezos-shell.opam index f5930bec9aab..c93f131a0067 100644 --- a/opam/tezos-shell.opam +++ b/opam/tezos-shell.opam @@ -9,6 +9,7 @@ dev-repo: "git+https://gitlab.com/tezos/tezos.git" license: "MIT" depends: [ "dune" { >= "3.0" } + "ppx_expect" "lwt-watcher" { = "0.2" } "lwt-canceler" { >= "0.3" & < "0.4" } "prometheus" { >= "1.2" } diff --git a/src/lib_shell/distributed_db_message.ml b/src/lib_shell/distributed_db_message.ml index 9d18827d47e0..71ad146a5369 100644 --- a/src/lib_shell/distributed_db_message.ml +++ b/src/lib_shell/distributed_db_message.ml @@ -27,126 +27,169 @@ module Bounded_encoding = struct open Data_encoding - let block_header_max_size = ref (8 * 1024 * 1024) - - let block_locator_max_length = ref 1000 - - (* FIXME: arbitrary *) - - let block_header_cache = - ref (Block_header.bounded_encoding ~max_size:!block_header_max_size ()) - - let block_locator_cache = - ref - (Block_locator.bounded_encoding - ~max_header_size:!block_header_max_size - ~max_length:!block_locator_max_length - ()) - - let update_block_header_encoding () = - block_header_cache := - Block_header.bounded_encoding ~max_size:!block_header_max_size () ; - block_locator_cache := - Block_locator.bounded_encoding - ~max_header_size:!block_header_max_size - ~max_length:!block_locator_max_length - () - - let set_block_header_max_size max = - block_header_max_size := max ; - update_block_header_encoding () - - let set_block_locator_max_length max = - block_locator_max_length := max ; - update_block_header_encoding () - - let block_header = delayed (fun () -> !block_header_cache) - - let block_locator = delayed (fun () -> !block_locator_cache) + module M : sig + type 'a t + + val create : 'a -> 'a t * ('a -> unit) + + val map : 'a t -> ('a -> 'b) -> 'b t + + val map2 : 'a t -> 'b t -> ('a -> 'b -> 'c) -> 'c t + + val encoding : 'a Data_encoding.t t -> 'a Data_encoding.t + end = struct + type 'a t = {get : unit -> 'a; mutable on_update : (unit -> unit) list} + + type deps = D : 'a t -> deps + + let register_on_update f (D x) = + let old = x.on_update in + x.on_update <- f :: old + + let make_with_deps cache deps f = + let res = + { + get = + (fun () -> + match !cache with + | Some x -> x + | None -> + let r = f () in + cache := Some r ; + r); + on_update = []; + } + in + let invalidate () = + cache := None ; + List.iter (fun f -> f ()) res.on_update + in + List.iter (register_on_update invalidate) deps ; + res + + let create x = + let cache = ref (Some x) in + let res = + make_with_deps cache [] (fun () -> + (* The cache is always set to some value, this function will never be called *) + assert false) + in + let set x = + cache := Some x ; + List.iter (fun f -> f ()) res.on_update + in + (res, set) + + let map x f = make_with_deps (ref None) [D x] (fun () -> f (x.get ())) + + let map2 x y f = + make_with_deps (ref None) [D x; D y] (fun () -> f (x.get ()) (y.get ())) + + let get x = x.get () + + let encoding x = delayed (fun () -> get x) + + let%expect_test _ = + let x, set_x = create 4 in + let y, _set_y = create 4 in + let z = + map2 x y (fun x y -> + print_endline "compute z" ; + x + y) + in + [%expect {||}] ; + print_int (get z) ; + [%expect {| + compute z + 8 |}] ; + print_int (get z) ; + [%expect {| 8 |}] ; + set_x 1 ; + print_int (get x) ; + [%expect {| 1 |}] ; + print_int (get z) ; + [%expect {| + compute z + 5 |}] + end + + let block_header_max_size, set_block_header_max_size = + M.create (8 * 1024 * 1024) + + let block_locator_max_length, set_block_locator_max_length = + M.create 1000 (* FIXME: arbitrary *) + + let block_header = + M.map block_header_max_size (fun max_size -> + Block_header.bounded_encoding ~max_size ()) + |> M.encoding + + let block_locator = + M.map2 + block_header_max_size + block_locator_max_length + (fun max_header_size max_length -> + Block_locator.bounded_encoding ~max_header_size ~max_length ()) + |> M.encoding (* FIXME: all constants below are arbitrary high bounds until we have the mechanism to update them properly *) - let operation_max_size = ref (Some (128 * 1024)) (* FIXME: arbitrary *) - - let operation_list_max_size = ref (Some (1024 * 1024)) (* FIXME: arbitrary *) - - let operation_list_max_length = ref None (* FIXME: arbitrary *) - - let operation_max_pass = ref (Some 8) (* FIXME: arbitrary *) - - let operation_cache = - ref (Operation.bounded_encoding ?max_size:!operation_max_size ()) - let operation_list_cache = - ref - (Operation.bounded_list_encoding - ?max_length:!operation_list_max_length - ?max_size:!operation_list_max_size - ?max_operation_size:!operation_max_size - ?max_pass:!operation_max_pass - ()) - - let update_operation_list_encoding () = - operation_list_cache := - Operation.bounded_list_encoding - ?max_length:!operation_list_max_length - ?max_size:!operation_list_max_size - ?max_operation_size:!operation_max_size - ?max_pass:!operation_max_pass - () - - let update_operation_hash_list_encoding () = - operation_list_cache := - Operation.bounded_list_encoding - ?max_length:!operation_list_max_length - ?max_pass:!operation_max_pass - () - - let update_operation_encoding () = - operation_cache := - Operation.bounded_encoding ?max_size:!operation_max_size () - - let set_operation_max_size max = - operation_max_size := max ; - update_operation_encoding () ; - update_operation_list_encoding () - - let set_operation_list_max_size max = - operation_list_max_size := max ; - update_operation_list_encoding () - - let set_operation_list_max_length max = - operation_list_max_length := max ; - update_operation_list_encoding () ; - update_operation_hash_list_encoding () - - let set_operation_max_pass max = - operation_max_pass := max ; - update_operation_list_encoding () ; - update_operation_hash_list_encoding () - - let operation = delayed (fun () -> !operation_cache) + let operation_max_size, set_operation_max_size = M.create (Some (128 * 1024)) + (* FIXME: arbitrary *) - let operation_list = delayed (fun () -> !operation_list_cache) + let operation_list_max_size, set_operation_list_max_size = + M.create (Some (1024 * 1024)) + (* FIXME: arbitrary *) - let protocol_max_size = ref (Some (2 * 1024 * 1024)) (* FIXME: arbitrary *) + let operation_list_max_length, set_operation_list_max_length = + M.create None (* FIXME: arbitrary *) - let protocol_cache = - ref (Protocol.bounded_encoding ?max_size:!protocol_max_size ()) + let operation_max_pass, set_operation_max_pass = M.create (Some 8) + (* FIXME: arbitrary *) - let set_protocol_max_size max = protocol_max_size := max + let operation = + M.map operation_max_size (fun max_size -> + Operation.bounded_encoding ?max_size ()) + |> M.encoding + + let map4 x y z t f = + M.map2 + (M.map2 x y (fun x y -> (x, y))) + (M.map2 z t (fun z t -> (z, t))) + (fun (x, y) (z, t) -> f x y z t) + + let operation_list = + map4 + operation_list_max_length + operation_list_max_size + operation_max_size + operation_max_pass + (fun max_length max_size max_operation_size max_pass -> + Operation.bounded_list_encoding + ?max_length + ?max_size + ?max_operation_size + ?max_pass + ()) + |> M.encoding + + let protocol_max_size, set_protocol_max_size = + M.create (Some (2 * 1024 * 1024)) + (* FIXME: arbitrary *) - let protocol = delayed (fun () -> !protocol_cache) + let protocol = + M.map protocol_max_size (fun max_size -> + Protocol.bounded_encoding ?max_size ()) + |> M.encoding (* Twice the current max size of a mempoool *) - let mempool_max_operations = ref (Some 4000) - - let mempool_cache = - ref (Mempool.bounded_encoding ?max_operations:!mempool_max_operations ()) - - let set_mempool_max_operations max = mempool_max_operations := max + let mempool_max_operations, set_mempool_max_operations = M.create (Some 4000) - let mempool = delayed (fun () -> !mempool_cache) + let mempool = + M.map mempool_max_operations (fun max_operations -> + Mempool.bounded_encoding ?max_operations ()) + |> M.encoding end type t = diff --git a/src/lib_shell/dune b/src/lib_shell/dune index 5b1413060369..94afb45eac0e 100644 --- a/src/lib_shell/dune +++ b/src/lib_shell/dune @@ -27,6 +27,8 @@ tezos-validation tezos-version lwt-exit) + (inline_tests (flags -verbose) (modes native)) + (preprocess (pps ppx_expect)) (flags (:standard) -open Tezos_base.TzPervasives -- GitLab From 141f27f5134c4d9cb7c0f1d0dabe1330225197bb Mon Sep 17 00:00:00 2001 From: Hugo Heuzard Date: Fri, 1 Jul 2022 13:59:26 +0200 Subject: [PATCH 2/3] Shell: distributed_db_message, group constant together --- src/lib_shell/distributed_db_message.ml | 43 +++++++++++-------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/src/lib_shell/distributed_db_message.ml b/src/lib_shell/distributed_db_message.ml index 71ad146a5369..c5848994af33 100644 --- a/src/lib_shell/distributed_db_message.ml +++ b/src/lib_shell/distributed_db_message.ml @@ -113,11 +113,27 @@ module Bounded_encoding = struct 5 |}] end + (* FIXME: all constants below are arbitrary high bounds until we + have the mechanism to update them properly *) + let block_header_max_size, set_block_header_max_size = M.create (8 * 1024 * 1024) - let block_locator_max_length, set_block_locator_max_length = - M.create 1000 (* FIXME: arbitrary *) + let block_locator_max_length, set_block_locator_max_length = M.create 1000 + + let operation_max_size, set_operation_max_size = M.create (Some (128 * 1024)) + + let operation_list_max_size, set_operation_list_max_size = + M.create (Some (1024 * 1024)) + + let operation_list_max_length, set_operation_list_max_length = M.create None + + let operation_max_pass, set_operation_max_pass = M.create (Some 8) + + let protocol_max_size, set_protocol_max_size = + M.create (Some (2 * 1024 * 1024)) + + let mempool_max_operations, set_mempool_max_operations = M.create (Some 4000) let block_header = M.map block_header_max_size (fun max_size -> @@ -132,22 +148,6 @@ module Bounded_encoding = struct Block_locator.bounded_encoding ~max_header_size ~max_length ()) |> M.encoding - (* FIXME: all constants below are arbitrary high bounds until we - have the mechanism to update them properly *) - - let operation_max_size, set_operation_max_size = M.create (Some (128 * 1024)) - (* FIXME: arbitrary *) - - let operation_list_max_size, set_operation_list_max_size = - M.create (Some (1024 * 1024)) - (* FIXME: arbitrary *) - - let operation_list_max_length, set_operation_list_max_length = - M.create None (* FIXME: arbitrary *) - - let operation_max_pass, set_operation_max_pass = M.create (Some 8) - (* FIXME: arbitrary *) - let operation = M.map operation_max_size (fun max_size -> Operation.bounded_encoding ?max_size ()) @@ -174,18 +174,11 @@ module Bounded_encoding = struct ()) |> M.encoding - let protocol_max_size, set_protocol_max_size = - M.create (Some (2 * 1024 * 1024)) - (* FIXME: arbitrary *) - let protocol = M.map protocol_max_size (fun max_size -> Protocol.bounded_encoding ?max_size ()) |> M.encoding - (* Twice the current max size of a mempoool *) - let mempool_max_operations, set_mempool_max_operations = M.create (Some 4000) - let mempool = M.map mempool_max_operations (fun max_operations -> Mempool.bounded_encoding ?max_operations ()) -- GitLab From 838deca31cc41a4ca6bbe53ba26b205b07f9c466 Mon Sep 17 00:00:00 2001 From: Hugo Heuzard Date: Mon, 11 Jul 2022 15:33:54 +0200 Subject: [PATCH 3/3] Shell: distributed_db_message, assign issue to FIXME --- src/lib_shell/distributed_db_message.ml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib_shell/distributed_db_message.ml b/src/lib_shell/distributed_db_message.ml index c5848994af33..765d7b8740dc 100644 --- a/src/lib_shell/distributed_db_message.ml +++ b/src/lib_shell/distributed_db_message.ml @@ -113,8 +113,9 @@ module Bounded_encoding = struct 5 |}] end - (* FIXME: all constants below are arbitrary high bounds until we - have the mechanism to update them properly *) + (* FIXME: https://gitlab.com/tezos/tezos/-/issues/3370 + all constants below are arbitrary high bounds until we have the + mechanism to update them properly. *) let block_header_max_size, set_block_header_max_size = M.create (8 * 1024 * 1024) -- GitLab