From 63480b11dc90d6812819f1d371b19f3feb1b244d Mon Sep 17 00:00:00 2001 From: Pierre Boutillier Date: Wed, 2 Nov 2022 13:42:29 +0100 Subject: [PATCH 1/5] Shell: Move config limits in a module of shell_services --- docs/doc_gen/node_helpers.ml | 8 +- src/bin_node/node_config_file.ml | 328 +++------------------- src/bin_node/node_config_file.mli | 10 +- src/bin_node/node_shared_arg.ml | 5 +- src/lib_shell/block_validator.ml | 13 +- src/lib_shell/block_validator.mli | 7 +- src/lib_shell/chain_validator.ml | 15 +- src/lib_shell/chain_validator.mli | 22 +- src/lib_shell/node.ml | 23 -- src/lib_shell/node.mli | 16 +- src/lib_shell/peer_validator.ml | 9 +- src/lib_shell/peer_validator.mli | 9 +- src/lib_shell/prevalidator.ml | 30 +- src/lib_shell/prevalidator.mli | 24 +- src/lib_shell/test/test_node.ml | 24 +- src/lib_shell/test/test_peer_validator.ml | 4 +- src/lib_shell/test/test_validator.ml | 8 +- src/lib_shell/validator.ml | 8 +- src/lib_shell/validator.mli | 8 +- src/lib_shell_services/shell_limits.ml | 316 +++++++++++++++++++++ src/lib_shell_services/shell_limits.mli | 103 +++++++ src/lib_validation/block_validation.ml | 4 - src/lib_validation/block_validation.mli | 3 - 23 files changed, 530 insertions(+), 467 deletions(-) create mode 100644 src/lib_shell_services/shell_limits.ml create mode 100644 src/lib_shell_services/shell_limits.mli diff --git a/docs/doc_gen/node_helpers.ml b/docs/doc_gen/node_helpers.ml index 534e4784dad4..136e3a2152a9 100644 --- a/docs/doc_gen/node_helpers.ml +++ b/docs/doc_gen/node_helpers.ml @@ -62,10 +62,10 @@ let with_node f = Node.create ~singleprocess:true node_config - Node.default_peer_validator_limits - Node.default_block_validator_limits - Node.default_prevalidator_limits - Node.default_chain_validator_limits + Tezos_shell_services.Shell_limits.default_peer_validator_limits + Tezos_shell_services.Shell_limits.default_block_validator_limits + Tezos_shell_services.Shell_limits.default_prevalidator_limits + Tezos_shell_services.Shell_limits.default_chain_validator_limits None in f node diff --git a/src/bin_node/node_config_file.ml b/src/bin_node/node_config_file.ml index 2666084a1ecb..ecb1ccc45472 100644 --- a/src/bin_node/node_config_file.ml +++ b/src/bin_node/node_config_file.ml @@ -395,7 +395,7 @@ type t = { rpc : rpc; log : Lwt_log_sink_unix.cfg; internal_events : Internal_event_config.t; - shell : shell; + shell : Shell_limits.limits; blockchain_network : blockchain_network; metrics_addr : string list; dal : dal; @@ -425,14 +425,6 @@ and rpc = { and tls = {cert : string; key : string} -and shell = { - block_validator_limits : Block_validator.limits; - prevalidator_limits : Prevalidator.limits; - peer_validator_limits : Peer_validator.limits; - chain_validator_limits : Chain_validator.limits; - history_mode : History_mode.t option; -} - let default_p2p_limits : P2p.limits = let greylist_timeout = Time.System.Span.of_seconds_exn 86400. (* one day *) in { @@ -488,15 +480,6 @@ let default_rpc = media_type = Media_type.Command_line.Any; } -let default_shell = - { - block_validator_limits = Node.default_block_validator_limits; - prevalidator_limits = Node.default_prevalidator_limits; - peer_validator_limits = Node.default_peer_validator_limits; - chain_validator_limits = Node.default_chain_validator_limits; - history_mode = None; - } - let default_disable_config_validation = false let default_dal : dal = {activated = false; srs_size = None} @@ -508,7 +491,7 @@ let default_config = rpc = default_rpc; log = Lwt_log_sink_unix.default_cfg; internal_events = Internal_event_config.default; - shell = default_shell; + shell = Shell_limits.default_limits; blockchain_network = blockchain_network_mainnet; disable_config_validation = default_disable_config_validation; metrics_addr = []; @@ -931,232 +914,6 @@ let rpc : rpc Data_encoding.t = Media_type.Command_line.encoding default_rpc.media_type)) -let timeout_encoding = Time.System.Span.encoding - -let block_validator_limits_encoding = - let open Data_encoding in - conv - (fun {Block_validator.protocol_timeout; operation_metadata_size_limit} -> - (protocol_timeout, operation_metadata_size_limit)) - (fun (protocol_timeout, operation_metadata_size_limit) -> - {protocol_timeout; operation_metadata_size_limit}) - (obj2 - (dft - "protocol_request_timeout" - timeout_encoding - default_shell.block_validator_limits.protocol_timeout) - (dft - "operation_metadata_size_limit" - (union - [ - case - ~title:"unlimited" - (Tag 0) - (constant "unlimited") - (function None -> Some () | _ -> None) - (fun () -> None); - case - ~title:"limited" - (Tag 1) - int31 - (function Some i -> Some i | None -> None) - (fun i -> Some i); - ]) - default_shell.block_validator_limits.operation_metadata_size_limit)) - -let prevalidator_limits_encoding = - let open Data_encoding in - conv - (fun { - Prevalidator.operation_timeout; - max_refused_operations; - operations_batch_size; - disable_precheck; - } -> - ( operation_timeout, - max_refused_operations, - operations_batch_size, - disable_precheck )) - (fun ( operation_timeout, - max_refused_operations, - operations_batch_size, - disable_precheck ) -> - { - operation_timeout; - max_refused_operations; - operations_batch_size; - disable_precheck; - }) - (obj4 - (dft - "operations_request_timeout" - timeout_encoding - default_shell.prevalidator_limits.operation_timeout) - (dft - "max_refused_operations" - uint16 - default_shell.prevalidator_limits.max_refused_operations) - (dft - "operations_batch_size" - int31 - default_shell.prevalidator_limits.operations_batch_size) - (dft - "disable_precheck" - bool - default_shell.prevalidator_limits.disable_precheck)) - -let peer_validator_limits_encoding = - let open Data_encoding in - let default_limits = default_shell.peer_validator_limits in - conv - (fun { - Peer_validator.block_header_timeout; - block_operations_timeout; - protocol_timeout; - new_head_request_timeout; - } -> - ( block_header_timeout, - block_operations_timeout, - protocol_timeout, - new_head_request_timeout )) - (fun ( block_header_timeout, - block_operations_timeout, - protocol_timeout, - new_head_request_timeout ) -> - { - block_header_timeout; - block_operations_timeout; - protocol_timeout; - new_head_request_timeout; - }) - (obj4 - (dft - "block_header_request_timeout" - timeout_encoding - default_limits.block_header_timeout) - (dft - "block_operations_request_timeout" - timeout_encoding - default_limits.block_operations_timeout) - (dft - "protocol_request_timeout" - timeout_encoding - default_limits.protocol_timeout) - (dft - "new_head_request_timeout" - timeout_encoding - default_limits.new_head_request_timeout)) - -let synchronisation_heuristic_encoding default_latency default_threshold = - let open Data_encoding in - conv - (fun {Chain_validator.latency; threshold} -> (latency, threshold)) - (fun (latency, threshold) -> {latency; threshold}) - (obj2 - (dft - "latency" - ~description: - "[latency] is the time interval (in seconds) used to determine if \ - a peer is synchronized with a chain. For instance, a peer whose \ - known head has a timestamp T is considered synchronized if T >= \ - now - latency. This parameter depends on the baking rate and the \ - latency of the network." - uint16 - default_latency) - (dft - "synchronisation_threshold" - ~description: - "The minimal number of peers this peer should be synchronized with \ - in order to be bootstrapped." - uint8 - default_threshold)) - -let chain_validator_limits_encoding = - let open Data_encoding in - conv - (fun {Chain_validator.synchronisation} -> synchronisation) - (fun synchronisation -> {synchronisation}) - (* Use a union to support both the deprecated - bootstrap_threshold and the new synchronisation_threshold - options when parsing. When printing, use the new - synchronisation_threshold option. *) - (union - [ - case - ~title:"synchronisation_heuristic_encoding" - Json_only - (synchronisation_heuristic_encoding - default_shell.chain_validator_limits.synchronisation.latency - default_shell.chain_validator_limits.synchronisation.threshold) - (fun x -> Some x) - (fun x -> x); - case - ~title:"legacy_bootstrap_threshold_encoding" - Json_only - (obj1 - (dft - "bootstrap_threshold" - ~description: - "[DEPRECATED] Set the number of peers with whom a chain \ - synchronisation must be completed to bootstrap the node." - uint8 - 4)) - (fun _ -> None) (* This is used for legacy *) - (fun x -> - Chain_validator. - { - threshold = x; - latency = - default_shell.chain_validator_limits.synchronisation.latency; - }); - ]) - -let shell = - let open Data_encoding in - conv - (fun { - peer_validator_limits; - block_validator_limits; - prevalidator_limits; - chain_validator_limits; - history_mode; - } -> - ( peer_validator_limits, - block_validator_limits, - prevalidator_limits, - chain_validator_limits, - history_mode )) - (fun ( peer_validator_limits, - block_validator_limits, - prevalidator_limits, - chain_validator_limits, - history_mode ) -> - { - peer_validator_limits; - block_validator_limits; - prevalidator_limits; - chain_validator_limits; - history_mode; - }) - (obj5 - (dft - "peer_validator" - peer_validator_limits_encoding - default_shell.peer_validator_limits) - (dft - "block_validator" - block_validator_limits_encoding - default_shell.block_validator_limits) - (dft - "prevalidator" - prevalidator_limits_encoding - default_shell.prevalidator_limits) - (dft - "chain_validator" - chain_validator_limits_encoding - default_shell.chain_validator_limits) - (opt "history_mode" History_mode.encoding)) - let encoding = let open Data_encoding in conv @@ -1239,8 +996,8 @@ let encoding = (dft "shell" ~description:"Configuration of network parameters" - shell - default_shell) + Shell_limits.limits_encoding + Shell_limits.default_limits) (dft "network" ~description:"Configuration of which network/blockchain to connect to" @@ -1353,7 +1110,7 @@ let update ?(disable_config_validation = false) ?data_dir ?min_connections ?(metrics_addr = []) ?operation_metadata_size_limit ?(private_mode = false) ?(disable_mempool = false) ?(disable_mempool_precheck = - default_shell.prevalidator_limits.disable_precheck) + Shell_limits.default_limits.prevalidator_limits.disable_precheck) ?(enable_testchain = false) ?(cors_origins = []) ?(cors_headers = []) ?rpc_tls ?log_output ?synchronisation_threshold ?history_mode ?network ?latency cfg = @@ -1427,43 +1184,44 @@ let update ?(disable_config_validation = false) ?data_dir ?min_connections and metrics_addr = unopt_list ~default:cfg.metrics_addr metrics_addr and log : Lwt_log_sink_unix.cfg = {cfg.log with output = Option.value ~default:cfg.log.output log_output} - and shell : shell = - { - peer_validator_limits = cfg.shell.peer_validator_limits; - block_validator_limits = - { - cfg.shell.block_validator_limits with - operation_metadata_size_limit = - Option.value - ~default: - cfg.shell.block_validator_limits.operation_metadata_size_limit - operation_metadata_size_limit; - }; - prevalidator_limits = - { - cfg.shell.prevalidator_limits with - disable_precheck = - cfg.shell.prevalidator_limits.disable_precheck - || disable_mempool_precheck; - }; - chain_validator_limits = - (let synchronisation : Chain_validator.synchronisation_limits = - { - latency = - Option.value - ~default: - cfg.shell.chain_validator_limits.synchronisation.latency - latency; - threshold = - Option.value - ~default: - cfg.shell.chain_validator_limits.synchronisation.threshold - synchronisation_threshold; - } - in - {synchronisation}); - history_mode = Option.either history_mode cfg.shell.history_mode; - } + and shell = + Shell_limits. + { + peer_validator_limits = cfg.shell.peer_validator_limits; + block_validator_limits = + { + cfg.shell.block_validator_limits with + operation_metadata_size_limit = + Option.value + ~default: + cfg.shell.block_validator_limits.operation_metadata_size_limit + operation_metadata_size_limit; + }; + prevalidator_limits = + { + cfg.shell.prevalidator_limits with + disable_precheck = + cfg.shell.prevalidator_limits.disable_precheck + || disable_mempool_precheck; + }; + chain_validator_limits = + (let synchronisation : synchronisation_limits = + { + latency = + Option.value + ~default: + cfg.shell.chain_validator_limits.synchronisation.latency + latency; + threshold = + Option.value + ~default: + cfg.shell.chain_validator_limits.synchronisation.threshold + synchronisation_threshold; + } + in + {synchronisation}); + history_mode = Option.either history_mode cfg.shell.history_mode; + } in (* If --network is specified it overrides the "network" entry of the configuration file, which itself defaults to mainnet. *) diff --git a/src/bin_node/node_config_file.mli b/src/bin_node/node_config_file.mli index 205c1b3cf2ab..4af620cfe228 100644 --- a/src/bin_node/node_config_file.mli +++ b/src/bin_node/node_config_file.mli @@ -65,7 +65,7 @@ type t = { rpc : rpc; log : Lwt_log_sink_unix.cfg; internal_events : Internal_event_config.t; - shell : shell; + shell : Shell_limits.limits; blockchain_network : blockchain_network; metrics_addr : string list; dal : dal; @@ -95,14 +95,6 @@ and rpc = { and tls = {cert : string; key : string} -and shell = { - block_validator_limits : Block_validator.limits; - prevalidator_limits : Prevalidator.limits; - peer_validator_limits : Peer_validator.limits; - chain_validator_limits : Chain_validator.limits; - history_mode : History_mode.t option; -} - val data_dir_env_name : string val default_data_dir : string diff --git a/src/bin_node/node_shared_arg.ml b/src/bin_node/node_shared_arg.ml index b2a50db659ee..b89711dd163d 100644 --- a/src/bin_node/node_shared_arg.ml +++ b/src/bin_node/node_shared_arg.ml @@ -413,7 +413,10 @@ module Term = struct in let doc = let default = - match Block_validation.default_operation_metadata_size_limit with + match + Shell_limits.default_limits.block_validator_limits + .operation_metadata_size_limit + with | None -> "$(i,unlimited)" | Some i -> Format.sprintf "$(i,%d) bytes" i in diff --git a/src/lib_shell/block_validator.ml b/src/lib_shell/block_validator.ml index 2d16325307d6..0da335efa97d 100644 --- a/src/lib_shell/block_validator.ml +++ b/src/lib_shell/block_validator.ml @@ -28,11 +28,6 @@ open Block_validator_worker_state open Block_validator_errors -type limits = { - protocol_timeout : Time.System.Span.t; - operation_metadata_size_limit : int option; -} - type validation_result = | Already_commited | Outdated_block @@ -63,12 +58,16 @@ module Types = struct type state = { protocol_validator : Protocol_validator.t; validation_process : Block_validator_process.t; - limits : limits; + limits : Shell_limits.block_validator_limits; start_testchain : bool; invalid_blocks_after_precheck : error trace Block_hash_ring.t; } - type parameters = limits * bool * Distributed_db.t * Block_validator_process.t + type parameters = + Shell_limits.block_validator_limits + * bool + * Distributed_db.t + * Block_validator_process.t end module Request = struct diff --git a/src/lib_shell/block_validator.mli b/src/lib_shell/block_validator.mli index 5207ac916496..ebf631dc08f5 100644 --- a/src/lib_shell/block_validator.mli +++ b/src/lib_shell/block_validator.mli @@ -28,11 +28,6 @@ type t -type limits = { - protocol_timeout : Time.System.Span.t; - operation_metadata_size_limit : int option; -} - (** [create limits ddb bvp start_testchain] creates a [Block_validator]. @@ -51,7 +46,7 @@ type limits = { This function is not supposed to fail. It is implemented this way because of the interface implemented by the [Worker] module. *) val create : - limits -> + Shell_limits.block_validator_limits -> Distributed_db.t -> Block_validator_process.t -> start_testchain:bool -> diff --git a/src/lib_shell/chain_validator.ml b/src/lib_shell/chain_validator.ml index e4844fb9b147..c629547d7274 100644 --- a/src/lib_shell/chain_validator.ml +++ b/src/lib_shell/chain_validator.ml @@ -62,10 +62,6 @@ module Request = struct | Disconnection peer_id -> PeerId peer_id end -type synchronisation_limits = {latency : int; threshold : int} - -type limits = {synchronisation : synchronisation_limits} - module Types = struct type parameters = { parent : Name.t option; @@ -77,9 +73,9 @@ module Types = struct global_valid_block_input : Store.Block.t Lwt_watcher.input; global_chains_input : (Chain_id.t * bool) Lwt_watcher.input; start_prevalidator : bool; - prevalidator_limits : Prevalidator.limits; - peer_validator_limits : Peer_validator.limits; - limits : limits; + prevalidator_limits : Shell_limits.prevalidator_limits; + peer_validator_limits : Shell_limits.peer_validator_limits; + limits : Shell_limits.chain_validator_limits; metrics : Shell_metrics.Chain_validator.t; } @@ -804,8 +800,9 @@ let on_launch w _ parameters = let metrics = Shell_metrics.Chain_validator.init Name.base let rec create ~start_testchain ~active_chains ?parent ~block_validator_process - start_prevalidator (peer_validator_limits : Peer_validator.limits) - (prevalidator_limits : Prevalidator.limits) block_validator + start_prevalidator + (peer_validator_limits : Shell_limits.peer_validator_limits) + (prevalidator_limits : Shell_limits.prevalidator_limits) block_validator global_valid_block_input global_chains_input db chain_store limits = let open Lwt_result_syntax in let spawn_child ~parent enable_prevalidator peer_validator_limits diff --git a/src/lib_shell/chain_validator.mli b/src/lib_shell/chain_validator.mli index 558562c50207..c01122f1b791 100644 --- a/src/lib_shell/chain_validator.mli +++ b/src/lib_shell/chain_validator.mli @@ -26,35 +26,19 @@ type t -(** Constants parameterizing the bootstrap heuristics. *) -type synchronisation_limits = { - latency : int; - (** [latency] is the time interval (seconds) used to determine - if a node is synchronized with a chain. For instance, a node that - knows head with timestamp T is synchronized if T >= now - - max_latency. This parameter depends on the baking rate and the - latency of the network. *) - threshold : int; - (** [threshold] determines the number of peers the synchronization - heuristic looks at to determine if the node is synchronized or - not. *) -} - -type limits = {synchronisation : synchronisation_limits} - val create : start_prevalidator:bool -> start_testchain:bool -> active_chains:t Chain_id.Table.t -> block_validator_process:Block_validator_process.t -> - Peer_validator.limits -> - Prevalidator.limits -> + Shell_limits.peer_validator_limits -> + Shell_limits.prevalidator_limits -> Block_validator.t -> Store.Block.t Lwt_watcher.input -> (Chain_id.t * bool) Lwt_watcher.input -> Distributed_db.t -> Store.chain_store -> - limits -> + Shell_limits.chain_validator_limits -> t tzresult Lwt.t val chain_id : t -> Chain_id.t diff --git a/src/lib_shell/node.ml b/src/lib_shell/node.ml index 50c485092f87..8533a56ccc11 100644 --- a/src/lib_shell/node.ml +++ b/src/lib_shell/node.ml @@ -137,29 +137,6 @@ type config = { enable_testchain : bool; } -let default_block_validator_limits = - let open Block_validator in - { - protocol_timeout = Time.System.Span.of_seconds_exn 120.; - operation_metadata_size_limit = - Block_validation.default_operation_metadata_size_limit; - } - -let default_prevalidator_limits = Prevalidator.default_limits - -let default_peer_validator_limits = - let open Peer_validator in - { - block_header_timeout = Time.System.Span.of_seconds_exn 300.; - block_operations_timeout = Time.System.Span.of_seconds_exn 300.; - protocol_timeout = Time.System.Span.of_seconds_exn 600.; - new_head_request_timeout = Time.System.Span.of_seconds_exn 90.; - } - -let default_chain_validator_limits = - let open Chain_validator in - {synchronisation = {latency = 150; threshold = 4}} - (* These protocols are linked with the node and do not have their actual hash on purpose. *) let test_protocol_hashes = diff --git a/src/lib_shell/node.mli b/src/lib_shell/node.mli index 3d991466b6e7..0411260ba977 100644 --- a/src/lib_shell/node.mli +++ b/src/lib_shell/node.mli @@ -49,23 +49,15 @@ type config = { (** If [false], testchain related messages will be ignored. *) } -val default_peer_validator_limits : Peer_validator.limits - -val default_prevalidator_limits : Prevalidator.limits - -val default_block_validator_limits : Block_validator.limits - -val default_chain_validator_limits : Chain_validator.limits - val create : ?sandboxed:bool -> ?sandbox_parameters:Data_encoding.json -> singleprocess:bool -> config -> - Peer_validator.limits -> - Block_validator.limits -> - Prevalidator.limits -> - Chain_validator.limits -> + Shell_limits.peer_validator_limits -> + Shell_limits.block_validator_limits -> + Shell_limits.prevalidator_limits -> + Shell_limits.chain_validator_limits -> History_mode.t option -> t tzresult Lwt.t diff --git a/src/lib_shell/peer_validator.ml b/src/lib_shell/peer_validator.ml index 97d8e21f5f33..2cc929cdd1ac 100644 --- a/src/lib_shell/peer_validator.ml +++ b/src/lib_shell/peer_validator.ml @@ -58,13 +58,6 @@ module Request = struct (locator.head_hash, Block_locator.estimated_length seed locator) end -type limits = { - new_head_request_timeout : Time.System.Span.t; - block_header_timeout : Time.System.Span.t; - block_operations_timeout : Time.System.Span.t; - protocol_timeout : Time.System.Span.t; -} - module Types = struct type parameters = { chain_db : Distributed_db.chain_db; @@ -72,7 +65,7 @@ module Types = struct (* callback to chain_validator *) notify_new_block : Store.Block.t -> unit; notify_termination : unit -> unit; - limits : limits; + limits : Shell_limits.peer_validator_limits; } type state = { diff --git a/src/lib_shell/peer_validator.mli b/src/lib_shell/peer_validator.mli index e53bad6fd6b2..56b06bab9a12 100644 --- a/src/lib_shell/peer_validator.mli +++ b/src/lib_shell/peer_validator.mli @@ -26,19 +26,12 @@ type t -type limits = { - new_head_request_timeout : Time.System.Span.t; - block_header_timeout : Time.System.Span.t; - block_operations_timeout : Time.System.Span.t; - protocol_timeout : Time.System.Span.t; -} - val peer_id : t -> P2p_peer.Id.t val create : ?notify_new_block:(Store.Block.t -> unit) -> ?notify_termination:(unit -> unit) -> - limits -> + Shell_limits.peer_validator_limits -> Block_validator.t -> Distributed_db.chain_db -> P2p_peer.Id.t -> diff --git a/src/lib_shell/prevalidator.ml b/src/lib_shell/prevalidator.ml index 193a427b698f..89b926372e74 100644 --- a/src/lib_shell/prevalidator.ml +++ b/src/lib_shell/prevalidator.ml @@ -27,21 +27,6 @@ open Prevalidator_worker_state module Events = Prevalidator_events -type limits = { - max_refused_operations : int; - operation_timeout : Time.System.Span.t; - operations_batch_size : int; - disable_precheck : bool; -} - -let default_limits = - { - operation_timeout = Time.System.Span.of_seconds_exn 10.; - max_refused_operations = 1000; - operations_batch_size = 50; - disable_precheck = false; - } - (* Minimal delay between two mempool advertisements *) let advertisement_delay = 0.1 @@ -146,7 +131,10 @@ module Tools = struct } end -type 'a parameters = {limits : limits; tools : 'a Tools.tools} +type 'a parameters = { + limits : Shell_limits.prevalidator_limits; + tools : 'a Tools.tools; +} (** The type needed for the implementation of [Make] below, but * which is independent from the protocol. *) @@ -1059,7 +1047,7 @@ module Make_s end module type ARG = sig - val limits : limits + val limits : Shell_limits.prevalidator_limits val chain_db : Distributed_db.chain_db @@ -1095,7 +1083,7 @@ module Make module Types = struct type state = types_state - type parameters = limits * Distributed_db.chain_db + type parameters = Shell_limits.prevalidator_limits * Distributed_db.chain_db end module Worker : @@ -1533,7 +1521,7 @@ module Make let classification_parameters = Classification. { - map_size_limit = limits.max_refused_operations; + map_size_limit = limits.Shell_limits.max_refused_operations; on_discarded_operation = Distributed_db.Operation.clear_or_cancel chain_db; } @@ -1816,7 +1804,9 @@ module Internal_for_tests = struct let mk_types_state_shell ~(predecessor : Store.Block.t) ~(tools : 'a tools) ~(worker : worker_tools) : (_, 'a) types_state_shell = - let parameters = {limits = default_limits; tools} in + let parameters = + {limits = Shell_limits.default_prevalidator_limits; tools} + in let c_parameters : Classification.parameters = {map_size_limit = 32; on_discarded_operation = Fun.const ()} in diff --git a/src/lib_shell/prevalidator.mli b/src/lib_shell/prevalidator.mli index 8870c0ad1373..62a1bde1a49a 100644 --- a/src/lib_shell/prevalidator.mli +++ b/src/lib_shell/prevalidator.mli @@ -52,31 +52,9 @@ used for separate chains (e.g., mainchain vs testchain). *) type t -(** This record contains the differents limits and settings that can be updated - from a node configuration for a prevalidator *) -type limits = { - max_refused_operations : int; - (** The maximum number of operations tracked by the mempool for each of - the [refused], [branch delayed], [branch refused] and [outdated] - operation classifications. Default is [1000] *) - operation_timeout : Time.System.Span.t; - (** The maximum time allowed to fetch the contents of an operation - advertised by a remote peer. Default is [10] seconds *) - operations_batch_size : int; - (** Maximum number of pending operations processed (or classified) at the - end of each request to the prevalidator worker. Default is [50] *) - disable_precheck : bool; - (** If [disable_precheck] is [true] (default is [false]) operations are - executed by the protocol before being propagated. This flag is - intended to be used for testing and debugging. *) -} - -(** Sane default values for {!limits} *) -val default_limits : limits - (** Creates/tear-down a new prevalidator context. *) val create : - limits -> + Shell_limits.prevalidator_limits -> (module Shell_plugin.FILTER) -> Distributed_db.chain_db -> t tzresult Lwt.t diff --git a/src/lib_shell/test/test_node.ml b/src/lib_shell/test/test_node.ml index a1c894d940c5..17510a0773aa 100644 --- a/src/lib_shell/test/test_node.ml +++ b/src/lib_shell/test/test_node.ml @@ -137,13 +137,13 @@ let node_sandbox_initialization_events sandbox_parameters config _switch () = (* Tezos_shell.Node.config *) config (* Tezos_shell.Node.peer_validator_limits *) - Node.default_peer_validator_limits + Shell_limits.default_peer_validator_limits (* Tezos_shell.Node.block_validator_limits *) - Node.default_block_validator_limits + Shell_limits.default_block_validator_limits (* Tezos_shell.Node.prevalidator_limits *) - Node.default_prevalidator_limits + Shell_limits.default_prevalidator_limits (* Tezos_shell.Node.chain_validator_limits *) - Node.default_chain_validator_limits + Shell_limits.default_chain_validator_limits (* Tezos_shell_services.History_mode.t option *) None in @@ -171,13 +171,13 @@ let node_initialization_events _sandbox_parameters config _switch () = (* Tezos_shell.Node.config *) {config with p2p = default_p2p} (* Tezos_shell.Node.peer_validator_limits *) - Node.default_peer_validator_limits + Shell_limits.default_peer_validator_limits (* Tezos_shell.Node.block_validator_limits *) - Node.default_block_validator_limits + Shell_limits.default_block_validator_limits (* Tezos_shell.Node.prevalidator_limits *) - Node.default_prevalidator_limits + Shell_limits.default_prevalidator_limits (* Tezos_shell.Node.chain_validator_limits *) - Node.default_chain_validator_limits + Shell_limits.default_chain_validator_limits (* Tezos_shell_services.History_mode.t option *) None in @@ -211,13 +211,13 @@ let node_store_known_protocol_events _sandbox_parameters config _switch () = (* Tezos_shell.Node.config *) {config with p2p = default_p2p} (* Tezos_shell.Node.peer_validator_limits *) - Node.default_peer_validator_limits + Shell_limits.default_peer_validator_limits (* Tezos_shell.Node.block_validator_limits *) - Node.default_block_validator_limits + Shell_limits.default_block_validator_limits (* Tezos_shell.Node.prevalidator_limits *) - Node.default_prevalidator_limits + Shell_limits.default_prevalidator_limits (* Tezos_shell.Node.chain_validator_limits *) - Node.default_chain_validator_limits + Shell_limits.default_chain_validator_limits (* Tezos_shell_services.History_mode.t option *) None in diff --git a/src/lib_shell/test/test_peer_validator.ml b/src/lib_shell/test/test_peer_validator.ml index dadff2002107..1d96dc874dbd 100644 --- a/src/lib_shell/test/test_peer_validator.ml +++ b/src/lib_shell/test/test_peer_validator.ml @@ -81,14 +81,14 @@ let wrap in let*! block_validator = Block_validator.create - Node.default_block_validator_limits + Shell_limits.default_block_validator_limits db block_validator_processs ~start_testchain:false in let*! pv = Peer_validator.create - Node.default_peer_validator_limits + Shell_limits.default_peer_validator_limits block_validator chain_db Crypto_box.Public_key_hash.zero diff --git a/src/lib_shell/test/test_validator.ml b/src/lib_shell/test/test_validator.ml index d264da378273..291fdc1f6ec0 100644 --- a/src/lib_shell/test/test_validator.ml +++ b/src/lib_shell/test/test_validator.ml @@ -66,11 +66,11 @@ let init_validator Validator.create store db - Node.default_peer_validator_limits - Node.default_block_validator_limits + Shell_limits.default_peer_validator_limits + Shell_limits.default_block_validator_limits block_validator - Node.default_prevalidator_limits - Node.default_chain_validator_limits + Shell_limits.default_prevalidator_limits + Shell_limits.default_chain_validator_limits ~start_testchain:false in Lwt.return_ok (block_validator, validator, Store.main_chain_store store) diff --git a/src/lib_shell/validator.ml b/src/lib_shell/validator.ml index 4d75f1bfc75d..753558b4333c 100644 --- a/src/lib_shell/validator.ml +++ b/src/lib_shell/validator.ml @@ -28,10 +28,10 @@ type t = { state : Store.t; db : Distributed_db.t; block_validator : Block_validator.t; - chain_validator_limits : Chain_validator.limits; - peer_validator_limits : Peer_validator.limits; - block_validator_limits : Block_validator.limits; - prevalidator_limits : Prevalidator.limits; + chain_validator_limits : Shell_limits.chain_validator_limits; + peer_validator_limits : Shell_limits.peer_validator_limits; + block_validator_limits : Shell_limits.block_validator_limits; + prevalidator_limits : Shell_limits.prevalidator_limits; start_testchain : bool; valid_block_input : Store.Block.t Lwt_watcher.input; chains_input : (Chain_id.t * bool) Lwt_watcher.input; diff --git a/src/lib_shell/validator.mli b/src/lib_shell/validator.mli index 0823298aa307..e1a308e0e59a 100644 --- a/src/lib_shell/validator.mli +++ b/src/lib_shell/validator.mli @@ -31,11 +31,11 @@ type t val create : Store.t -> Distributed_db.t -> - Peer_validator.limits -> - Block_validator.limits -> + Shell_limits.peer_validator_limits -> + Shell_limits.block_validator_limits -> Block_validator_process.t -> - Prevalidator.limits -> - Chain_validator.limits -> + Shell_limits.prevalidator_limits -> + Shell_limits.chain_validator_limits -> start_testchain:bool -> t tzresult Lwt.t diff --git a/src/lib_shell_services/shell_limits.ml b/src/lib_shell_services/shell_limits.ml new file mode 100644 index 000000000000..04c561aa96fe --- /dev/null +++ b/src/lib_shell_services/shell_limits.ml @@ -0,0 +1,316 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2022 Nomadic Labs, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +let timeout_encoding = Time.System.Span.encoding + +type block_validator_limits = { + protocol_timeout : Time.System.Span.t; + operation_metadata_size_limit : int option; +} + +(* [default_operation_metadata_size_limit] is used to filter and + potentially discard a given metadata if its size exceed the cap. *) +let default_operation_metadata_size_limit = Some 10_000_000 + +let default_block_validator_limits = + { + protocol_timeout = Time.System.Span.of_seconds_exn 120.; + operation_metadata_size_limit = default_operation_metadata_size_limit; + } + +let block_validator_limits_encoding = + let open Data_encoding in + conv + (fun {protocol_timeout; operation_metadata_size_limit} -> + (protocol_timeout, operation_metadata_size_limit)) + (fun (protocol_timeout, operation_metadata_size_limit) -> + {protocol_timeout; operation_metadata_size_limit}) + (obj2 + (dft + "protocol_request_timeout" + timeout_encoding + default_block_validator_limits.protocol_timeout) + (dft + "operation_metadata_size_limit" + (union + [ + case + ~title:"unlimited" + (Tag 0) + (constant "unlimited") + (function None -> Some () | _ -> None) + (fun () -> None); + case + ~title:"limited" + (Tag 1) + int31 + (function Some i -> Some i | None -> None) + (fun i -> Some i); + ]) + default_block_validator_limits.operation_metadata_size_limit)) + +type prevalidator_limits = { + max_refused_operations : int; + operation_timeout : Time.System.Span.t; + operations_batch_size : int; + disable_precheck : bool; +} + +let default_prevalidator_limits = + { + operation_timeout = Time.System.Span.of_seconds_exn 10.; + max_refused_operations = 1000; + operations_batch_size = 50; + disable_precheck = false; + } + +let prevalidator_limits_encoding = + let open Data_encoding in + conv + (fun { + operation_timeout; + max_refused_operations; + operations_batch_size; + disable_precheck; + } -> + ( operation_timeout, + max_refused_operations, + operations_batch_size, + disable_precheck )) + (fun ( operation_timeout, + max_refused_operations, + operations_batch_size, + disable_precheck ) -> + { + operation_timeout; + max_refused_operations; + operations_batch_size; + disable_precheck; + }) + (obj4 + (dft + "operations_request_timeout" + timeout_encoding + default_prevalidator_limits.operation_timeout) + (dft + "max_refused_operations" + uint16 + default_prevalidator_limits.max_refused_operations) + (dft + "operations_batch_size" + int31 + default_prevalidator_limits.operations_batch_size) + (dft + "disable_precheck" + bool + default_prevalidator_limits.disable_precheck)) + +type peer_validator_limits = { + new_head_request_timeout : Time.System.Span.t; + block_header_timeout : Time.System.Span.t; + block_operations_timeout : Time.System.Span.t; + protocol_timeout : Time.System.Span.t; +} + +let default_peer_validator_limits = + { + block_header_timeout = Time.System.Span.of_seconds_exn 300.; + block_operations_timeout = Time.System.Span.of_seconds_exn 300.; + protocol_timeout = Time.System.Span.of_seconds_exn 600.; + new_head_request_timeout = Time.System.Span.of_seconds_exn 90.; + } + +let peer_validator_limits_encoding = + let open Data_encoding in + conv + (fun { + block_header_timeout; + block_operations_timeout; + protocol_timeout; + new_head_request_timeout; + } -> + ( block_header_timeout, + block_operations_timeout, + protocol_timeout, + new_head_request_timeout )) + (fun ( block_header_timeout, + block_operations_timeout, + protocol_timeout, + new_head_request_timeout ) -> + { + block_header_timeout; + block_operations_timeout; + protocol_timeout; + new_head_request_timeout; + }) + (obj4 + (dft + "block_header_request_timeout" + timeout_encoding + default_peer_validator_limits.block_header_timeout) + (dft + "block_operations_request_timeout" + timeout_encoding + default_peer_validator_limits.block_operations_timeout) + (dft + "protocol_request_timeout" + timeout_encoding + default_peer_validator_limits.protocol_timeout) + (dft + "new_head_request_timeout" + timeout_encoding + default_peer_validator_limits.new_head_request_timeout)) + +type synchronisation_limits = {latency : int; threshold : int} + +let synchronisation_heuristic_encoding default_latency default_threshold = + let open Data_encoding in + conv + (fun {latency; threshold} -> (latency, threshold)) + (fun (latency, threshold) -> {latency; threshold}) + (obj2 + (dft + "latency" + ~description: + "[latency] is the time interval (in seconds) used to determine if \ + a peer is synchronized with a chain. For instance, a peer whose \ + known head has a timestamp T is considered synchronized if T >= \ + now - latency. This parameter depends on the baking rate and the \ + latency of the network." + uint16 + default_latency) + (dft + "synchronisation_threshold" + ~description: + "The minimal number of peers this peer should be synchronized with \ + in order to be bootstrapped." + uint8 + default_threshold)) + +type chain_validator_limits = {synchronisation : synchronisation_limits} + +let default_chain_validator_limits = + {synchronisation = {latency = 150; threshold = 4}} + +let chain_validator_limits_encoding = + let open Data_encoding in + conv + (fun {synchronisation} -> synchronisation) + (fun synchronisation -> {synchronisation}) + (* Use a union to support both the deprecated + bootstrap_threshold and the new synchronisation_threshold + options when parsing. When printing, use the new + synchronisation_threshold option. *) + (union + [ + case + ~title:"synchronisation_heuristic_encoding" + Json_only + (synchronisation_heuristic_encoding + default_chain_validator_limits.synchronisation.latency + default_chain_validator_limits.synchronisation.threshold) + (fun x -> Some x) + (fun x -> x); + case + ~title:"legacy_bootstrap_threshold_encoding" + Json_only + (obj1 + (dft + "bootstrap_threshold" + ~description: + "[DEPRECATED] Set the number of peers with whom a chain \ + synchronisation must be completed to bootstrap the node." + uint8 + 4)) + (fun _ -> None) (* This is used for legacy *) + (fun x -> + { + threshold = x; + latency = default_chain_validator_limits.synchronisation.latency; + }); + ]) + +type limits = { + block_validator_limits : block_validator_limits; + prevalidator_limits : prevalidator_limits; + peer_validator_limits : peer_validator_limits; + chain_validator_limits : chain_validator_limits; + history_mode : History_mode.t option; +} + +let default_limits = + { + block_validator_limits = default_block_validator_limits; + prevalidator_limits = default_prevalidator_limits; + peer_validator_limits = default_peer_validator_limits; + chain_validator_limits = default_chain_validator_limits; + history_mode = None; + } + +let limits_encoding = + let open Data_encoding in + conv + (fun { + peer_validator_limits; + block_validator_limits; + prevalidator_limits; + chain_validator_limits; + history_mode; + } -> + ( peer_validator_limits, + block_validator_limits, + prevalidator_limits, + chain_validator_limits, + history_mode )) + (fun ( peer_validator_limits, + block_validator_limits, + prevalidator_limits, + chain_validator_limits, + history_mode ) -> + { + peer_validator_limits; + block_validator_limits; + prevalidator_limits; + chain_validator_limits; + history_mode; + }) + (obj5 + (dft + "peer_validator" + peer_validator_limits_encoding + default_peer_validator_limits) + (dft + "block_validator" + block_validator_limits_encoding + default_block_validator_limits) + (dft + "prevalidator" + prevalidator_limits_encoding + default_prevalidator_limits) + (dft + "chain_validator" + chain_validator_limits_encoding + default_chain_validator_limits) + (opt "history_mode" History_mode.encoding)) diff --git a/src/lib_shell_services/shell_limits.mli b/src/lib_shell_services/shell_limits.mli new file mode 100644 index 000000000000..69e89a23d746 --- /dev/null +++ b/src/lib_shell_services/shell_limits.mli @@ -0,0 +1,103 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2022 Nomadic Labs, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(** The configurable constants used by shell components as maximum, + with their encodings and default values. *) + +type block_validator_limits = { + protocol_timeout : Time.System.Span.t; + operation_metadata_size_limit : int option; +} + +val default_block_validator_limits : block_validator_limits + +val block_validator_limits_encoding : block_validator_limits Data_encoding.t + +(** This record contains the differents limits and settings that can be updated + from a node configuration for a prevalidator *) +type prevalidator_limits = { + max_refused_operations : int; + (** The maximum number of operations tracked by the mempool for each of + the [refused], [branch delayed], [branch refused] and [outdated] + operation classifications. Default is [1000] *) + operation_timeout : Time.System.Span.t; + (** The maximum time allowed to fetch the contents of an operation + advertised by a remote peer. Default is [10] seconds *) + operations_batch_size : int; + (** Maximum number of pending operations processed (or classified) at the + end of each request to the prevalidator worker. Default is [50] *) + disable_precheck : bool; + (** If [disable_precheck] is [true] (default is [false]) operations are + executed by the protocol before being propagated. This flag is + intended to be used for testing and debugging. *) +} + +(** Sane default values for {!limits} *) +val default_prevalidator_limits : prevalidator_limits + +val prevalidator_limits_encoding : prevalidator_limits Data_encoding.t + +type peer_validator_limits = { + new_head_request_timeout : Time.System.Span.t; + block_header_timeout : Time.System.Span.t; + block_operations_timeout : Time.System.Span.t; + protocol_timeout : Time.System.Span.t; +} + +val default_peer_validator_limits : peer_validator_limits + +val peer_validator_limits_encoding : peer_validator_limits Data_encoding.t + +(** Constants parameterizing the bootstrap heuristics. *) +type synchronisation_limits = { + latency : int; + (** [latency] is the time interval (seconds) used to determine + if a node is synchronized with a chain. For instance, a node that + knows head with timestamp T is synchronized if T >= now - + max_latency. This parameter depends on the baking rate and the + latency of the network. *) + threshold : int; + (** [threshold] determines the number of peers the synchronization + heuristic looks at to determine if the node is synchronized or + not. *) +} + +type chain_validator_limits = {synchronisation : synchronisation_limits} + +val default_chain_validator_limits : chain_validator_limits + +val chain_validator_limits_encoding : chain_validator_limits Data_encoding.t + +type limits = { + block_validator_limits : block_validator_limits; + prevalidator_limits : prevalidator_limits; + peer_validator_limits : peer_validator_limits; + chain_validator_limits : chain_validator_limits; + history_mode : History_mode.t option; +} + +val default_limits : limits + +val limits_encoding : limits Data_encoding.t diff --git a/src/lib_validation/block_validation.ml b/src/lib_validation/block_validation.ml index 48edea85f0f6..b3b0abf5f779 100644 --- a/src/lib_validation/block_validation.ml +++ b/src/lib_validation/block_validation.ml @@ -85,10 +85,6 @@ let validation_store_encoding = type operation_metadata = Metadata of Bytes.t | Too_large_metadata -(* [default_operation_metadata_size_limit] is used to filter and - potentially discard a given metadata if its size exceed the cap. *) -let default_operation_metadata_size_limit = Some 10_000_000 - let operation_metadata_encoding = let open Data_encoding in def diff --git a/src/lib_validation/block_validation.mli b/src/lib_validation/block_validation.mli index 6d53db17f11f..b5ecdd53c071 100644 --- a/src/lib_validation/block_validation.mli +++ b/src/lib_validation/block_validation.mli @@ -117,9 +117,6 @@ type apply_environment = { (** size limit for operation metadata that should be written on disk *) } -(** Default size limit for operation metadata *) -val default_operation_metadata_size_limit : int option - (** [apply env header ops] gets the protocol [P] of the context of the predecessor block and calls successively: 1. [P.begin_application] -- GitLab From 0d6593e21705e4ab8c5542725049e67f64d96e90 Mon Sep 17 00:00:00 2001 From: Pierre Boutillier Date: Wed, 2 Nov 2022 14:49:09 +0100 Subject: [PATCH 2/5] Shell: Move P2P limits in a module of p2p_services --- src/bin_node/node_config_file.ml | 260 +----------------------- src/bin_node/node_config_file.mli | 2 +- src/lib_p2p/p2p.ml | 38 +--- src/lib_p2p/p2p.mli | 50 +---- src/lib_p2p_services/p2p_limits.ml | 296 ++++++++++++++++++++++++++++ src/lib_p2p_services/p2p_limits.mli | 79 ++++++++ src/lib_shell/node.ml | 2 +- src/lib_shell/node.mli | 2 +- src/lib_shell/test/test_node.ml | 35 +--- 9 files changed, 402 insertions(+), 362 deletions(-) create mode 100644 src/lib_p2p_services/p2p_limits.ml create mode 100644 src/lib_p2p_services/p2p_limits.mli diff --git a/src/bin_node/node_config_file.ml b/src/bin_node/node_config_file.ml index ecb1ccc45472..a98af1698e9b 100644 --- a/src/bin_node/node_config_file.ml +++ b/src/bin_node/node_config_file.ml @@ -408,7 +408,7 @@ and p2p = { advertised_net_port : int option; discovery_addr : string option; private_mode : bool; - limits : P2p.limits; + limits : Tezos_p2p_services.P2p_limits.t; disable_mempool : bool; enable_testchain : bool; reconnection_config : P2p_point_state.Info.reconnection_config; @@ -425,37 +425,6 @@ and rpc = { and tls = {cert : string; key : string} -let default_p2p_limits : P2p.limits = - let greylist_timeout = Time.System.Span.of_seconds_exn 86400. (* one day *) in - { - connection_timeout = Time.System.Span.of_seconds_exn 10.; - authentication_timeout = Time.System.Span.of_seconds_exn 5.; - greylist_timeout; - maintenance_idle_time = - Time.System.Span.of_seconds_exn 120. (* two minutes *); - min_connections = 10; - expected_connections = 50; - max_connections = 100; - backlog = 20; - max_incoming_connections = 20; - max_download_speed = None; - max_upload_speed = None; - read_buffer_size = 1 lsl 14; - read_queue_size = None; - write_queue_size = None; - incoming_app_message_queue_size = None; - incoming_message_queue_size = None; - outgoing_message_queue_size = None; - max_known_points = Some (400, 300); - max_known_peer_ids = Some (400, 300); - peer_greylist_size = 1023 (* historical value *); - ip_greylist_size_in_kilobytes = - 2 * 1024 (* two megabytes has shown good properties in simulation *); - ip_greylist_cleanup_delay = greylist_timeout; - swap_linger = Time.System.Span.of_seconds_exn 30.; - binary_chunks_size = None; - } - let default_p2p = { expected_pow = 26.; @@ -464,7 +433,7 @@ let default_p2p = advertised_net_port = None; discovery_addr = None; private_mode = false; - limits = default_p2p_limits; + limits = Tezos_p2p_services.P2p_limits.default; disable_mempool = false; enable_testchain = false; reconnection_config = P2p_point_state.Info.default_reconnection_config; @@ -498,223 +467,6 @@ let default_config = dal = default_dal; } -let limit : P2p.limits Data_encoding.t = - let open Data_encoding in - conv - (fun { - P2p.connection_timeout; - authentication_timeout; - greylist_timeout; - maintenance_idle_time; - min_connections; - expected_connections; - max_connections; - backlog; - max_incoming_connections; - max_download_speed; - max_upload_speed; - read_buffer_size; - read_queue_size; - write_queue_size; - incoming_app_message_queue_size; - incoming_message_queue_size; - outgoing_message_queue_size; - max_known_points; - max_known_peer_ids; - peer_greylist_size; - ip_greylist_size_in_kilobytes; - ip_greylist_cleanup_delay; - swap_linger; - binary_chunks_size; - } -> - ( ( ( connection_timeout, - authentication_timeout, - min_connections, - expected_connections, - max_connections, - backlog, - max_incoming_connections, - max_download_speed, - max_upload_speed, - swap_linger ), - ( binary_chunks_size, - read_buffer_size, - read_queue_size, - write_queue_size, - incoming_app_message_queue_size, - incoming_message_queue_size, - outgoing_message_queue_size, - max_known_points ) ), - ( max_known_peer_ids, - peer_greylist_size, - ip_greylist_size_in_kilobytes, - ip_greylist_cleanup_delay, - greylist_timeout, - maintenance_idle_time ) )) - (fun ( ( ( connection_timeout, - authentication_timeout, - min_connections, - expected_connections, - max_connections, - backlog, - max_incoming_connections, - max_download_speed, - max_upload_speed, - swap_linger ), - ( binary_chunks_size, - read_buffer_size, - read_queue_size, - write_queue_size, - incoming_app_message_queue_size, - incoming_message_queue_size, - outgoing_message_queue_size, - max_known_points ) ), - ( max_known_peer_ids, - peer_greylist_size, - ip_greylist_size_in_kilobytes, - ip_greylist_cleanup_delay, - greylist_timeout, - maintenance_idle_time ) ) -> - { - connection_timeout; - authentication_timeout; - greylist_timeout; - maintenance_idle_time; - min_connections; - expected_connections; - max_connections; - backlog; - max_incoming_connections; - max_download_speed; - max_upload_speed; - read_buffer_size; - read_queue_size; - write_queue_size; - incoming_app_message_queue_size; - incoming_message_queue_size; - outgoing_message_queue_size; - max_known_points; - max_known_peer_ids; - peer_greylist_size; - ip_greylist_size_in_kilobytes; - ip_greylist_cleanup_delay; - swap_linger; - binary_chunks_size; - }) - (merge_objs - (merge_objs - (obj10 - (dft - "connection-timeout" - ~description: - "Delay acceptable when initiating a connection to a new \ - peer, in seconds." - Time.System.Span.encoding - default_p2p_limits.authentication_timeout) - (dft - "authentication-timeout" - ~description: - "Delay granted to a peer to perform authentication, in \ - seconds." - Time.System.Span.encoding - default_p2p_limits.authentication_timeout) - (dft - "min-connections" - ~description: - "Strict minimum number of connections (triggers an urgent \ - maintenance)." - uint16 - default_p2p_limits.min_connections) - (dft - "expected-connections" - ~description: - "Targeted number of connections to reach when bootstrapping \ - / maintaining." - uint16 - default_p2p_limits.expected_connections) - (dft - "max-connections" - ~description: - "Maximum number of connections (exceeding peers are \ - disconnected)." - uint16 - default_p2p_limits.max_connections) - (dft - "backlog" - ~description: - "Number above which pending incoming connections are \ - immediately rejected." - uint8 - default_p2p_limits.backlog) - (dft - "max-incoming-connections" - ~description: - "Number above which pending incoming connections are \ - immediately rejected." - uint8 - default_p2p_limits.max_incoming_connections) - (opt - "max-download-speed" - ~description:"Max download speeds in KiB/s." - int31) - (opt - "max-upload-speed" - ~description:"Max upload speeds in KiB/s." - int31) - (dft - "swap-linger" - Time.System.Span.encoding - default_p2p_limits.swap_linger)) - (obj8 - (opt "binary-chunks-size" uint8) - (dft - "read-buffer-size" - ~description:"Size of the buffer passed to read(2)." - int31 - default_p2p_limits.read_buffer_size) - (opt "read-queue-size" int31) - (opt "write-queue-size" int31) - (opt "incoming-app-message-queue-size" int31) - (opt "incoming-message-queue-size" int31) - (opt "outgoing-message-queue-size" int31) - (opt - "max_known_points" - ~description: - "The max and target size for the known address table." - (tup2 uint16 uint16)))) - (obj6 - (opt - "max_known_peer_ids" - ~description:"The max and target size for the known peers table." - (tup2 uint16 uint16)) - (dft - "peer_greylist_size" - ~description:"The number of peer_ids kept in the peer_id greylist." - uint16 - default_p2p_limits.peer_greylist_size) - (dft - "ip_greylist_size_in_kilobytes" - ~description:"The size of the IP address greylist (in kilobytes)." - uint16 - default_p2p_limits.ip_greylist_size_in_kilobytes) - (dft - "ip_greylist_cleanup_delay" - ~description:"The time an IP address is kept in the greylist." - Time.System.Span.encoding - default_p2p_limits.ip_greylist_cleanup_delay) - (dft - "greylist-timeout" - ~description:"GC delay for the greylists tables, in seconds." - Time.System.Span.encoding - default_p2p_limits.greylist_timeout) - (dft - "maintenance-idle-time" - ~description: - "How long to wait at most, in seconds, before running a \ - maintenance loop." - Time.System.Span.encoding - default_p2p_limits.maintenance_idle_time))) - let p2p = let open Data_encoding in conv @@ -806,7 +558,11 @@ let p2p = the identity and the address of the private node secret." bool false) - (dft "limits" ~description:"Network limits" limit default_p2p_limits) + (dft + "limits" + ~description:"Network limits" + Tezos_p2p_services.P2p_limits.encoding + Tezos_p2p_services.P2p_limits.default) (dft "disable_mempool" ~description: @@ -1127,7 +883,7 @@ let update ?(disable_config_validation = false) ?data_dir ?min_connections let* () = Node_data_version.ensure_data_dir ~mode:Exists data_dir in let peer_table_size = Option.map (fun i -> (i, i / 4 * 3)) peer_table_size in let unopt_list ~default = function [] -> default | l -> l in - let limits : P2p.limits = + let limits : Tezos_p2p_services.P2p_limits.t = { cfg.p2p.limits with min_connections = diff --git a/src/bin_node/node_config_file.mli b/src/bin_node/node_config_file.mli index 4af620cfe228..74525cfa432b 100644 --- a/src/bin_node/node_config_file.mli +++ b/src/bin_node/node_config_file.mli @@ -78,7 +78,7 @@ and p2p = { advertised_net_port : int option; discovery_addr : string option; private_mode : bool; - limits : P2p.limits; + limits : Tezos_p2p_services.P2p_limits.t; disable_mempool : bool; enable_testchain : bool; reconnection_config : P2p_point_state.Info.reconnection_config; diff --git a/src/lib_p2p/p2p.ml b/src/lib_p2p/p2p.ml index a625f2d64553..0daafcb854de 100644 --- a/src/lib_p2p/p2p.ml +++ b/src/lib_p2p/p2p.ml @@ -41,34 +41,8 @@ type config = { reconnection_config : P2p_point_state.Info.reconnection_config; } -type limits = { - connection_timeout : Time.System.Span.t; - authentication_timeout : Time.System.Span.t; - greylist_timeout : Time.System.Span.t; - maintenance_idle_time : Time.System.Span.t; - min_connections : int; - expected_connections : int; - max_connections : int; - backlog : int; - max_incoming_connections : int; - max_download_speed : int option; - max_upload_speed : int option; - read_buffer_size : int; - read_queue_size : int option; - write_queue_size : int option; - incoming_app_message_queue_size : int option; - incoming_message_queue_size : int option; - outgoing_message_queue_size : int option; - max_known_peer_ids : (int * int) option; - max_known_points : (int * int) option; - peer_greylist_size : int; - ip_greylist_size_in_kilobytes : int; - ip_greylist_cleanup_delay : Time.System.Span.t; - swap_linger : Time.System.Span.t; - binary_chunks_size : int option; -} - let create_scheduler limits = + let open P2p_limits in let max_upload_speed = Option.map (( * ) 1024) limits.max_upload_speed in let max_download_speed = Option.map (( * ) 1024) limits.max_download_speed in P2p_io_scheduler.create @@ -80,6 +54,7 @@ let create_scheduler limits = () let create_connection_pool config limits meta_cfg log triggers = + let open P2p_limits in let pool_cfg = { P2p_pool.identity = config.identity; @@ -97,6 +72,7 @@ let create_connection_pool config limits meta_cfg log triggers = let create_connect_handler config limits pool msg_cfg conn_meta_cfg io_sched triggers log answerer = + let open P2p_limits in let connect_handler_cfg = { P2p_connect_handler.identity = config.identity; @@ -142,6 +118,7 @@ let may_create_discovery_worker _limits config pool = | _, _, _ -> None let create_maintenance_worker limits pool connect_handler config triggers log = + let open P2p_limits in let maintenance_config = { P2p_maintenance.maintenance_idle_time = limits.maintenance_idle_time; @@ -171,7 +148,7 @@ let may_create_welcome_worker config limits connect_handler = config.listening_port |> Option.map_es (fun port -> P2p_welcome.create - ~backlog:limits.backlog + ~backlog:limits.P2p_limits.backlog connect_handler ?addr:config.listening_addr port) @@ -182,7 +159,7 @@ type ('msg, 'peer_meta, 'conn_meta) connection = module Real = struct type ('msg, 'peer_meta, 'conn_meta) net = { config : config; - limits : limits; + limits : P2p_limits.t; io_sched : P2p_io_scheduler.t; pool : ('msg, 'peer_meta, 'conn_meta) P2p_pool.t; connect_handler : ('msg, 'peer_meta, 'conn_meta) P2p_connect_handler.t; @@ -212,7 +189,7 @@ module Real = struct in let proto_conf = { - P2p_protocol.swap_linger = limits.swap_linger; + P2p_protocol.swap_linger = limits.P2p_limits.swap_linger; pool; log; connect; @@ -481,6 +458,7 @@ let connect_handler net = net.connect_handler let check_limits = let open Result_syntax in + let open P2p_limits in let fail_1 v orig = if not (Ptime.Span.compare v Ptime.Span.zero <= 0) then return_unit else diff --git a/src/lib_p2p/p2p.mli b/src/lib_p2p/p2p.mli index a744db93297c..0eab16e8932e 100644 --- a/src/lib_p2p/p2p.mli +++ b/src/lib_p2p/p2p.mli @@ -109,54 +109,6 @@ type config = { (** The reconnection delat configuration. *) } -(** Network capacities *) -type limits = { - connection_timeout : Time.System.Span.t; - (** Maximum time allowed to the establishment of a connection. *) - authentication_timeout : Time.System.Span.t; - (** Delay granted to a peer to perform authentication. *) - greylist_timeout : Time.System.Span.t; - (** GC delay for the greylists tables. *) - maintenance_idle_time : Time.System.Span.t; - (** How long to wait at most before running a maintenance loop. *) - min_connections : int; - (** Strict minimum number of connections (triggers an urgent maintenance) *) - expected_connections : int; - (** Targeted number of connections to reach when bootstrapping / maintaining *) - max_connections : int; - (** Maximum number of connections (exceeding peers are disconnected) *) - backlog : int; (** Argument of [Lwt_unix.accept].*) - max_incoming_connections : int; - (** Maximum not-yet-authenticated incoming connections. *) - max_download_speed : int option; - (** Hard-limit in the number of bytes received per second. *) - max_upload_speed : int option; - (** Hard-limit in the number of bytes sent per second. *) - read_buffer_size : int; - (** Size in bytes of the buffer passed to [Lwt_unix.read]. *) - read_queue_size : int option; - write_queue_size : int option; - incoming_app_message_queue_size : int option; - incoming_message_queue_size : int option; - outgoing_message_queue_size : int option; - (** Various bounds for internal queues. *) - max_known_peer_ids : (int * int) option; - max_known_points : (int * int) option; - (** Optional limitation of internal hashtables (max, target) *) - peer_greylist_size : int; - (** The number of peer_ids kept in the peer_id greylist. *) - ip_greylist_size_in_kilobytes : int; - (** The size of the IP address greylist in kilobytes. *) - ip_greylist_cleanup_delay : Time.System.Span.t; - (** The time an IP address is kept in the greylist. *) - swap_linger : Time.System.Span.t; - (** Peer swapping does not occur more than once during a timespan of - [swap_linger]. *) - binary_chunks_size : int option; - (** Size (in bytes) of binary blocks that are sent to other - peers. Default value is 64 kB. Max value is 64kB. *) -} - (** Type of a P2P layer instance *) type ('msg, 'peer_meta, 'conn_meta) t @@ -183,7 +135,7 @@ val faked_network : (** Main network initialization function *) val create : config:config -> - limits:limits -> + limits:Tezos_p2p_services.P2p_limits.t -> 'peer_meta P2p_params.peer_meta_config -> 'conn_meta P2p_params.conn_meta_config -> 'msg P2p_params.message_config -> diff --git a/src/lib_p2p_services/p2p_limits.ml b/src/lib_p2p_services/p2p_limits.ml new file mode 100644 index 000000000000..f918c1a41d53 --- /dev/null +++ b/src/lib_p2p_services/p2p_limits.ml @@ -0,0 +1,296 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2022 Nomadic Labs, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +type t = { + connection_timeout : Time.System.Span.t; + authentication_timeout : Time.System.Span.t; + greylist_timeout : Time.System.Span.t; + maintenance_idle_time : Time.System.Span.t; + min_connections : int; + expected_connections : int; + max_connections : int; + backlog : int; + max_incoming_connections : int; + max_download_speed : int option; + max_upload_speed : int option; + read_buffer_size : int; + read_queue_size : int option; + write_queue_size : int option; + incoming_app_message_queue_size : int option; + incoming_message_queue_size : int option; + outgoing_message_queue_size : int option; + max_known_peer_ids : (int * int) option; + max_known_points : (int * int) option; + peer_greylist_size : int; + ip_greylist_size_in_kilobytes : int; + ip_greylist_cleanup_delay : Time.System.Span.t; + swap_linger : Time.System.Span.t; + binary_chunks_size : int option; +} + +let default = + let greylist_timeout = Time.System.Span.of_seconds_exn 86400. (* one day *) in + { + connection_timeout = Time.System.Span.of_seconds_exn 10.; + authentication_timeout = Time.System.Span.of_seconds_exn 5.; + greylist_timeout; + maintenance_idle_time = + Time.System.Span.of_seconds_exn 120. (* two minutes *); + min_connections = 10; + expected_connections = 50; + max_connections = 100; + backlog = 20; + max_incoming_connections = 20; + max_download_speed = None; + max_upload_speed = None; + read_buffer_size = 1 lsl 14; + read_queue_size = None; + write_queue_size = None; + incoming_app_message_queue_size = None; + incoming_message_queue_size = None; + outgoing_message_queue_size = None; + max_known_points = Some (400, 300); + max_known_peer_ids = Some (400, 300); + peer_greylist_size = 1023 (* historical value *); + ip_greylist_size_in_kilobytes = + 2 * 1024 (* two megabytes has shown good properties in simulation *); + ip_greylist_cleanup_delay = greylist_timeout; + swap_linger = Time.System.Span.of_seconds_exn 30.; + binary_chunks_size = None; + } + +let encoding : t Data_encoding.t = + let open Data_encoding in + conv + (fun { + connection_timeout; + authentication_timeout; + greylist_timeout; + maintenance_idle_time; + min_connections; + expected_connections; + max_connections; + backlog; + max_incoming_connections; + max_download_speed; + max_upload_speed; + read_buffer_size; + read_queue_size; + write_queue_size; + incoming_app_message_queue_size; + incoming_message_queue_size; + outgoing_message_queue_size; + max_known_points; + max_known_peer_ids; + peer_greylist_size; + ip_greylist_size_in_kilobytes; + ip_greylist_cleanup_delay; + swap_linger; + binary_chunks_size; + } -> + ( ( ( connection_timeout, + authentication_timeout, + min_connections, + expected_connections, + max_connections, + backlog, + max_incoming_connections, + max_download_speed, + max_upload_speed, + swap_linger ), + ( binary_chunks_size, + read_buffer_size, + read_queue_size, + write_queue_size, + incoming_app_message_queue_size, + incoming_message_queue_size, + outgoing_message_queue_size, + max_known_points ) ), + ( max_known_peer_ids, + peer_greylist_size, + ip_greylist_size_in_kilobytes, + ip_greylist_cleanup_delay, + greylist_timeout, + maintenance_idle_time ) )) + (fun ( ( ( connection_timeout, + authentication_timeout, + min_connections, + expected_connections, + max_connections, + backlog, + max_incoming_connections, + max_download_speed, + max_upload_speed, + swap_linger ), + ( binary_chunks_size, + read_buffer_size, + read_queue_size, + write_queue_size, + incoming_app_message_queue_size, + incoming_message_queue_size, + outgoing_message_queue_size, + max_known_points ) ), + ( max_known_peer_ids, + peer_greylist_size, + ip_greylist_size_in_kilobytes, + ip_greylist_cleanup_delay, + greylist_timeout, + maintenance_idle_time ) ) -> + { + connection_timeout; + authentication_timeout; + greylist_timeout; + maintenance_idle_time; + min_connections; + expected_connections; + max_connections; + backlog; + max_incoming_connections; + max_download_speed; + max_upload_speed; + read_buffer_size; + read_queue_size; + write_queue_size; + incoming_app_message_queue_size; + incoming_message_queue_size; + outgoing_message_queue_size; + max_known_points; + max_known_peer_ids; + peer_greylist_size; + ip_greylist_size_in_kilobytes; + ip_greylist_cleanup_delay; + swap_linger; + binary_chunks_size; + }) + (merge_objs + (merge_objs + (obj10 + (dft + "connection-timeout" + ~description: + "Delay acceptable when initiating a connection to a new \ + peer, in seconds." + Time.System.Span.encoding + default.authentication_timeout) + (dft + "authentication-timeout" + ~description: + "Delay granted to a peer to perform authentication, in \ + seconds." + Time.System.Span.encoding + default.authentication_timeout) + (dft + "min-connections" + ~description: + "Strict minimum number of connections (triggers an urgent \ + maintenance)." + uint16 + default.min_connections) + (dft + "expected-connections" + ~description: + "Targeted number of connections to reach when bootstrapping \ + / maintaining." + uint16 + default.expected_connections) + (dft + "max-connections" + ~description: + "Maximum number of connections (exceeding peers are \ + disconnected)." + uint16 + default.max_connections) + (dft + "backlog" + ~description: + "Number above which pending incoming connections are \ + immediately rejected." + uint8 + default.backlog) + (dft + "max-incoming-connections" + ~description: + "Number above which pending incoming connections are \ + immediately rejected." + uint8 + default.max_incoming_connections) + (opt + "max-download-speed" + ~description:"Max download speeds in KiB/s." + int31) + (opt + "max-upload-speed" + ~description:"Max upload speeds in KiB/s." + int31) + (dft "swap-linger" Time.System.Span.encoding default.swap_linger)) + (obj8 + (opt "binary-chunks-size" uint8) + (dft + "read-buffer-size" + ~description:"Size of the buffer passed to read(2)." + int31 + default.read_buffer_size) + (opt "read-queue-size" int31) + (opt "write-queue-size" int31) + (opt "incoming-app-message-queue-size" int31) + (opt "incoming-message-queue-size" int31) + (opt "outgoing-message-queue-size" int31) + (opt + "max_known_points" + ~description: + "The max and target size for the known address table." + (tup2 uint16 uint16)))) + (obj6 + (opt + "max_known_peer_ids" + ~description:"The max and target size for the known peers table." + (tup2 uint16 uint16)) + (dft + "peer_greylist_size" + ~description:"The number of peer_ids kept in the peer_id greylist." + uint16 + default.peer_greylist_size) + (dft + "ip_greylist_size_in_kilobytes" + ~description:"The size of the IP address greylist (in kilobytes)." + uint16 + default.ip_greylist_size_in_kilobytes) + (dft + "ip_greylist_cleanup_delay" + ~description:"The time an IP address is kept in the greylist." + Time.System.Span.encoding + default.ip_greylist_cleanup_delay) + (dft + "greylist-timeout" + ~description:"GC delay for the greylists tables, in seconds." + Time.System.Span.encoding + default.greylist_timeout) + (dft + "maintenance-idle-time" + ~description: + "How long to wait at most, in seconds, before running a \ + maintenance loop." + Time.System.Span.encoding + default.maintenance_idle_time))) diff --git a/src/lib_p2p_services/p2p_limits.mli b/src/lib_p2p_services/p2p_limits.mli new file mode 100644 index 000000000000..0121c7b6d81d --- /dev/null +++ b/src/lib_p2p_services/p2p_limits.mli @@ -0,0 +1,79 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2022 Nomadic Labs, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(** The configurable constants used by the p2p layer as maximum, + with their encoding and default values. *) + +(** Network capacities *) +type t = { + connection_timeout : Time.System.Span.t; + (** Maximum time allowed to the establishment of a connection. *) + authentication_timeout : Time.System.Span.t; + (** Delay granted to a peer to perform authentication. *) + greylist_timeout : Time.System.Span.t; + (** GC delay for the greylists tables. *) + maintenance_idle_time : Time.System.Span.t; + (** How long to wait at most before running a maintenance loop. *) + min_connections : int; + (** Strict minimum number of connections (triggers an urgent maintenance) *) + expected_connections : int; + (** Targeted number of connections to reach when bootstrapping / maintaining *) + max_connections : int; + (** Maximum number of connections (exceeding peers are disconnected) *) + backlog : int; (** Argument of [Lwt_unix.accept].*) + max_incoming_connections : int; + (** Maximum not-yet-authenticated incoming connections. *) + max_download_speed : int option; + (** Hard-limit in the number of bytes received per second. *) + max_upload_speed : int option; + (** Hard-limit in the number of bytes sent per second. *) + read_buffer_size : int; + (** Size in bytes of the buffer passed to [Lwt_unix.read]. *) + read_queue_size : int option; + write_queue_size : int option; + incoming_app_message_queue_size : int option; + incoming_message_queue_size : int option; + outgoing_message_queue_size : int option; + (** Various bounds for internal queues. *) + max_known_peer_ids : (int * int) option; + max_known_points : (int * int) option; + (** Optional limitation of internal hashtables (max, target) *) + peer_greylist_size : int; + (** The number of peer_ids kept in the peer_id greylist. *) + ip_greylist_size_in_kilobytes : int; + (** The size of the IP address greylist in kilobytes. *) + ip_greylist_cleanup_delay : Time.System.Span.t; + (** The time an IP address is kept in the greylist. *) + swap_linger : Time.System.Span.t; + (** Peer swapping does not occur more than once during a timespan of + [swap_linger]. *) + binary_chunks_size : int option; + (** Size (in bytes) of binary blocks that are sent to other + peers. Default value is 64 kB. Max value is 64kB. *) +} + +val default : t + +val encoding : t Data_encoding.t diff --git a/src/lib_shell/node.ml b/src/lib_shell/node.ml index 8533a56ccc11..464e5ff37ff3 100644 --- a/src/lib_shell/node.ml +++ b/src/lib_shell/node.ml @@ -131,7 +131,7 @@ type config = { (Tezos_protocol_environment.Context.t -> Tezos_protocol_environment.Context.t tzresult Lwt.t) option; - p2p : (P2p.config * P2p.limits) option; + p2p : (P2p.config * P2p_limits.t) option; target : (Block_hash.t * int32) option; disable_mempool : bool; enable_testchain : bool; diff --git a/src/lib_shell/node.mli b/src/lib_shell/node.mli index 0411260ba977..22f48d7b1142 100644 --- a/src/lib_shell/node.mli +++ b/src/lib_shell/node.mli @@ -41,7 +41,7 @@ type config = { (Tezos_protocol_environment.Context.t -> Tezos_protocol_environment.Context.t tzresult Lwt.t) option; - p2p : (P2p.config * P2p.limits) option; + p2p : (P2p.config * P2p_limits.t) option; target : (Block_hash.t * int32) option; disable_mempool : bool; (** If [true], all non-empty mempools will be ignored. *) diff --git a/src/lib_shell/test/test_node.ml b/src/lib_shell/test/test_node.ml index 17510a0773aa..cd2d3b172769 100644 --- a/src/lib_shell/test/test_node.ml +++ b/src/lib_shell/test/test_node.ml @@ -78,34 +78,13 @@ let default_p2p : P2p.config = reconnection_config = P2p_point_state.Info.default_reconnection_config; } -let default_p2p_limits : P2p.limits = - { - connection_timeout = Time.System.Span.of_seconds_exn 10.; - authentication_timeout = Time.System.Span.of_seconds_exn 5.; - greylist_timeout = Time.System.Span.of_seconds_exn 86400. (* one day *); - maintenance_idle_time = - Time.System.Span.of_seconds_exn 120. (* two minutes *); - min_connections = 10; - expected_connections = 50; - max_connections = 100; - backlog = 20; - max_incoming_connections = 20; - max_download_speed = None; - max_upload_speed = None; - read_buffer_size = 1 lsl 14; - read_queue_size = None; - write_queue_size = None; - incoming_app_message_queue_size = None; - incoming_message_queue_size = None; - outgoing_message_queue_size = None; - max_known_points = Some (400, 300); - max_known_peer_ids = Some (400, 300); - swap_linger = Time.System.Span.of_seconds_exn 30.; - binary_chunks_size = None; - peer_greylist_size = 1023; - ip_greylist_size_in_kilobytes = 256; - ip_greylist_cleanup_delay = Ptime.Span.of_int_s 3600; - } +let default_p2p_limits = + P2p_limits. + { + default with + ip_greylist_size_in_kilobytes = 256 (* default/8 *); + ip_greylist_cleanup_delay = Ptime.Span.of_int_s 3600 (* default/24 *); + } let default_p2p = Some (default_p2p, default_p2p_limits) -- GitLab From ea32891da11c0b2d4297c5bc6f5188b36c2930a9 Mon Sep 17 00:00:00 2001 From: Pierre Boutillier Date: Wed, 2 Nov 2022 15:34:08 +0100 Subject: [PATCH 3/5] Shell/P2P: Move Point_reconnection_config in p2p_services --- src/bin_node/node_config_file.ml | 10 +-- src/bin_node/node_config_file.mli | 2 +- src/lib_p2p/p2p.ml | 2 +- src/lib_p2p/p2p.mli | 2 +- src/lib_p2p/p2p_connect_handler.ml | 6 +- src/lib_p2p/p2p_connect_handler.mli | 2 +- src/lib_p2p/p2p_point_state.ml | 78 +++--------------- src/lib_p2p/p2p_point_state.mli | 13 +-- src/lib_p2p/test/node.ml | 2 +- .../point_reconnection_config.ml | 81 +++++++++++++++++++ .../point_reconnection_config.mli | 38 +++++++++ src/lib_shell/test/test_node.ml | 2 +- 12 files changed, 144 insertions(+), 94 deletions(-) create mode 100644 src/lib_p2p_services/point_reconnection_config.ml create mode 100644 src/lib_p2p_services/point_reconnection_config.mli diff --git a/src/bin_node/node_config_file.ml b/src/bin_node/node_config_file.ml index a98af1698e9b..7522fbae41f9 100644 --- a/src/bin_node/node_config_file.ml +++ b/src/bin_node/node_config_file.ml @@ -411,7 +411,7 @@ and p2p = { limits : Tezos_p2p_services.P2p_limits.t; disable_mempool : bool; enable_testchain : bool; - reconnection_config : P2p_point_state.Info.reconnection_config; + reconnection_config : Tezos_p2p_services.Point_reconnection_config.t; } and rpc = { @@ -436,7 +436,7 @@ let default_p2p = limits = Tezos_p2p_services.P2p_limits.default; disable_mempool = false; enable_testchain = false; - reconnection_config = P2p_point_state.Info.default_reconnection_config; + reconnection_config = Tezos_p2p_services.Point_reconnection_config.default; } let default_rpc = @@ -582,14 +582,14 @@ let p2p = blocks." bool false) - (let open P2p_point_state.Info in + (let open Tezos_p2p_services.Point_reconnection_config in dft "greylisting_config" ~description: "The reconnection policy regulates the frequency with which the \ node tries to reconnect to an old known peer." - reconnection_config_encoding - default_reconnection_config)) + encoding + default)) let rpc : rpc Data_encoding.t = let open Data_encoding in diff --git a/src/bin_node/node_config_file.mli b/src/bin_node/node_config_file.mli index 74525cfa432b..957f865114ed 100644 --- a/src/bin_node/node_config_file.mli +++ b/src/bin_node/node_config_file.mli @@ -81,7 +81,7 @@ and p2p = { limits : Tezos_p2p_services.P2p_limits.t; disable_mempool : bool; enable_testchain : bool; - reconnection_config : P2p_point_state.Info.reconnection_config; + reconnection_config : Tezos_p2p_services.Point_reconnection_config.t; } and rpc = { diff --git a/src/lib_p2p/p2p.ml b/src/lib_p2p/p2p.ml index 0daafcb854de..7cb9a1405951 100644 --- a/src/lib_p2p/p2p.ml +++ b/src/lib_p2p/p2p.ml @@ -38,7 +38,7 @@ type config = { identity : P2p_identity.t; proof_of_work_target : Crypto_box.pow_target; trust_discovered_peers : bool; - reconnection_config : P2p_point_state.Info.reconnection_config; + reconnection_config : Point_reconnection_config.t; } let create_scheduler limits = diff --git a/src/lib_p2p/p2p.mli b/src/lib_p2p/p2p.mli index 0eab16e8932e..4d6d6fa1a5c6 100644 --- a/src/lib_p2p/p2p.mli +++ b/src/lib_p2p/p2p.mli @@ -105,7 +105,7 @@ type config = { (** Expected level of proof of work of peers' identity. *) trust_discovered_peers : bool; (** If [true], peers discovered on the local network will be trusted. *) - reconnection_config : P2p_point_state.Info.reconnection_config; + reconnection_config : Point_reconnection_config.t; (** The reconnection delat configuration. *) } diff --git a/src/lib_p2p/p2p_connect_handler.ml b/src/lib_p2p/p2p_connect_handler.ml index 320674d1962d..ebdee7e3a5aa 100644 --- a/src/lib_p2p/p2p_connect_handler.ml +++ b/src/lib_p2p/p2p_connect_handler.ml @@ -38,7 +38,7 @@ type config = { identity : P2p_identity.t; connection_timeout : Time.System.Span.t; authentication_timeout : Time.System.Span.t; - reconnection_config : P2p_point_state.Info.reconnection_config; + reconnection_config : Point_reconnection_config.t; proof_of_work_target : Crypto_box.pow_target; listening_port : P2p_addr.port option; advertised_port : P2p_addr.port option; @@ -768,9 +768,7 @@ module Internal_for_tests = struct let identity = P2p_identity.generate_with_pow_target_0 () in let connection_timeout = Time.System.Span.of_seconds_exn 10. in let authentication_timeout = Time.System.Span.of_seconds_exn 5. in - let reconnection_config = - P2p_point_state.Info.default_reconnection_config - in + let reconnection_config = Point_reconnection_config.default in let proof_of_work_target = Crypto_box.make_pow_target 0. in let listening_port = Some 9732 in let advertised_port = None in diff --git a/src/lib_p2p/p2p_connect_handler.mli b/src/lib_p2p/p2p_connect_handler.mli index c9d6e749dd0f..c74ebd894ee9 100644 --- a/src/lib_p2p/p2p_connect_handler.mli +++ b/src/lib_p2p/p2p_connect_handler.mli @@ -72,7 +72,7 @@ type config = { (** Maximum time allowed to the establishment of a connection. *) authentication_timeout : Time.System.Span.t; (** Maximum time allowed to the establishment of a connection. *) - reconnection_config : P2p_point_state.Info.reconnection_config; + reconnection_config : Point_reconnection_config.t; (** Delay granted to a peer to perform authentication. *) proof_of_work_target : Crypto_box.pow_target; (** The proof of work target we require from peers. *) diff --git a/src/lib_p2p/p2p_point_state.ml b/src/lib_p2p/p2p_point_state.ml index 57c1124c7f3e..eecc430561f5 100644 --- a/src/lib_p2p/p2p_point_state.ml +++ b/src/lib_p2p/p2p_point_state.ml @@ -43,13 +43,6 @@ let pp ppf = function | Disconnected -> Format.fprintf ppf "disconnected" module Info = struct - type reconnection_config = { - factor : float; - initial_delay : Time.System.Span.t; - disconnection_delay : Time.System.Span.t; - increase_cap : Time.System.Span.t; - } - type reconnection_info = { delay : Time.System.Span.t; end_time : Time.System.t; @@ -77,56 +70,6 @@ module Info = struct let log_size = 100 - let default_reconnection_config = - { - factor = 1.2; - initial_delay = Ptime.Span.of_int_s 1; - disconnection_delay = Ptime.Span.of_int_s 60; - increase_cap = Ptime.Span.of_int_s 172800 (* 2 days *); - } - - let reconnection_config_encoding = - let open Data_encoding in - conv - (fun {factor; initial_delay; disconnection_delay; increase_cap} -> - (factor, initial_delay, disconnection_delay, increase_cap)) - (fun (factor, initial_delay, disconnection_delay, increase_cap) -> - {factor; initial_delay; disconnection_delay; increase_cap}) - (obj4 - (dft - "factor" - ~description: - "The factor by which the reconnection delay is increased when a \ - peer that was previously disconnected is disconnected again. \ - This value should be set to 1 for a linear back-off and to >1 \ - for an exponential back-off." - float - default_reconnection_config.factor) - (dft - "initial-delay" - ~description: - "The span of time a peer is disconnected for when it is first \ - disconnected." - Time.System.Span.encoding - default_reconnection_config.initial_delay) - (dft - "disconnection-delay" - ~description: - "The span of time a peer is disconnected for when it is \ - disconnected as the result of an error." - Time.System.Span.encoding - default_reconnection_config.disconnection_delay) - (dft - "increase-cap" - ~description: - "The maximum amount by which the reconnection is extended. This \ - limits the rate of the exponential back-off, which eventually \ - becomes linear when it reaches this limit. This limit is set to \ - avoid reaching the End-of-Time when repeatedly reconnection a \ - peer." - Time.System.Span.encoding - default_reconnection_config.increase_cap)) - let create ?(trusted = false) ?expected_peer_id addr port = { point = (addr, port); @@ -251,27 +194,28 @@ let set_running ~timestamp point_info peer_id data = let maxed_time_add t s = match Ptime.add_span t s with Some t -> t | None -> Ptime.max -let set_reconnection_delay reconnection_config timestamp point_info = +let set_reconnection_delay (reconnection_config : Point_reconnection_config.t) + timestamp point_info = let disconnection_delay = match point_info.Info.reconnection_info with - | None -> reconnection_config.Info.initial_delay + | None -> reconnection_config.initial_delay | Some gr -> gr.delay in let end_time = maxed_time_add timestamp disconnection_delay in let delay = let new_delay = Time.System.Span.multiply_exn - reconnection_config.Info.factor + reconnection_config.factor disconnection_delay in - if Ptime.Span.compare reconnection_config.Info.increase_cap new_delay > 0 - then new_delay - else reconnection_config.Info.increase_cap + if Ptime.Span.compare reconnection_config.increase_cap new_delay > 0 then + new_delay + else reconnection_config.increase_cap in point_info.Info.reconnection_info <- Some {delay; end_time} -let set_disconnected ~timestamp ?(requested = false) reconnection_config - point_info = +let set_disconnected ~timestamp ?(requested = false) + (reconnection_config : Point_reconnection_config.t) point_info = let event : Pool_event.kind = match point_info.Info.state with | Requested _ -> @@ -283,9 +227,9 @@ let set_disconnected ~timestamp ?(requested = false) reconnection_config point_info.last_rejected_connection <- Some (current_peer_id, timestamp) ; Request_rejected (Some current_peer_id) | Running {current_peer_id; _} -> - let delay = reconnection_config.Info.initial_delay in + let delay = reconnection_config.initial_delay in let end_time = - maxed_time_add timestamp reconnection_config.Info.disconnection_delay + maxed_time_add timestamp reconnection_config.disconnection_delay in point_info.reconnection_info <- Some {delay; end_time} ; point_info.last_disconnection <- Some (current_peer_id, timestamp) ; diff --git a/src/lib_p2p/p2p_point_state.mli b/src/lib_p2p/p2p_point_state.mli index 8a64fab9ae22..2af40da1e925 100644 --- a/src/lib_p2p/p2p_point_state.mli +++ b/src/lib_p2p/p2p_point_state.mli @@ -46,17 +46,6 @@ module Info : sig val compare : 'conn point_info -> 'conn point_info -> int - type reconnection_config = { - factor : float; - initial_delay : Time.System.Span.t; - disconnection_delay : Time.System.Span.t; - increase_cap : Time.System.Span.t; - } - - val default_reconnection_config : reconnection_config - - val reconnection_config_encoding : reconnection_config Data_encoding.encoding - (** [create ~trusted addr port] is a freshly minted point_info. If [trusted] is true, this point is considered trusted and will be treated as such. If [expected_peer_id] is specified, we check @@ -161,7 +150,7 @@ val set_private : 'conn Info.t -> bool -> unit val set_disconnected : timestamp:Time.System.t -> ?requested:bool -> - Info.reconnection_config -> + Point_reconnection_config.t -> 'conn Info.t -> unit diff --git a/src/lib_p2p/test/node.ml b/src/lib_p2p/test/node.ml index d69615a7956b..45c7da7eb66c 100644 --- a/src/lib_p2p/test/node.ml +++ b/src/lib_p2p/test/node.ml @@ -164,7 +164,7 @@ let detach_node ?(prefix = "") ?timeout ?(min_connections : int option) listening_port = Some port; advertised_port = Some port; private_mode; - reconnection_config = P2p_point_state.Info.default_reconnection_config; + reconnection_config = Point_reconnection_config.default; min_connections = unopt min_connections; max_connections = unopt max_connections; max_incoming_connections = unopt max_incoming_connections; diff --git a/src/lib_p2p_services/point_reconnection_config.ml b/src/lib_p2p_services/point_reconnection_config.ml new file mode 100644 index 000000000000..09ee901dab67 --- /dev/null +++ b/src/lib_p2p_services/point_reconnection_config.ml @@ -0,0 +1,81 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2022 Nomadic Labs, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +type t = { + factor : float; + initial_delay : Time.System.Span.t; + disconnection_delay : Time.System.Span.t; + increase_cap : Time.System.Span.t; +} + +let default = + { + factor = 1.2; + initial_delay = Ptime.Span.of_int_s 1; + disconnection_delay = Ptime.Span.of_int_s 60; + increase_cap = Ptime.Span.of_int_s 172800 (* 2 days *); + } + +let encoding = + let open Data_encoding in + conv + (fun {factor; initial_delay; disconnection_delay; increase_cap} -> + (factor, initial_delay, disconnection_delay, increase_cap)) + (fun (factor, initial_delay, disconnection_delay, increase_cap) -> + {factor; initial_delay; disconnection_delay; increase_cap}) + (obj4 + (dft + "factor" + ~description: + "The factor by which the reconnection delay is increased when a \ + peer that was previously disconnected is disconnected again. This \ + value should be set to 1 for a linear back-off and to >1 for an \ + exponential back-off." + float + default.factor) + (dft + "initial-delay" + ~description: + "The span of time a peer is disconnected for when it is first \ + disconnected." + Time.System.Span.encoding + default.initial_delay) + (dft + "disconnection-delay" + ~description: + "The span of time a peer is disconnected for when it is \ + disconnected as the result of an error." + Time.System.Span.encoding + default.disconnection_delay) + (dft + "increase-cap" + ~description: + "The maximum amount by which the reconnection is extended. This \ + limits the rate of the exponential back-off, which eventually \ + becomes linear when it reaches this limit. This limit is set to \ + avoid reaching the End-of-Time when repeatedly reconnection a \ + peer." + Time.System.Span.encoding + default.increase_cap)) diff --git a/src/lib_p2p_services/point_reconnection_config.mli b/src/lib_p2p_services/point_reconnection_config.mli new file mode 100644 index 000000000000..4b173ed92271 --- /dev/null +++ b/src/lib_p2p_services/point_reconnection_config.mli @@ -0,0 +1,38 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2022 Nomadic Labs, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(** Parameters of the heuristic determining waiting time between + reconnection to a neighbour that misbehaved *) + +type t = { + factor : float; + initial_delay : Time.System.Span.t; + disconnection_delay : Time.System.Span.t; + increase_cap : Time.System.Span.t; +} + +val default : t + +val encoding : t Data_encoding.encoding diff --git a/src/lib_shell/test/test_node.ml b/src/lib_shell/test/test_node.ml index cd2d3b172769..5c5d32025073 100644 --- a/src/lib_shell/test/test_node.ml +++ b/src/lib_shell/test/test_node.ml @@ -75,7 +75,7 @@ let default_p2p : P2p.config = identity = P2p_identity.generate_with_pow_target_0 (); proof_of_work_target = Crypto_box.default_pow_target; trust_discovered_peers = false; - reconnection_config = P2p_point_state.Info.default_reconnection_config; + reconnection_config = Point_reconnection_config.default; } let default_p2p_limits = -- GitLab From 450db3950f4f581883e5976f51e824ed39e362d0 Mon Sep 17 00:00:00 2001 From: Nicolas BACQUEY Date: Wed, 12 Oct 2022 18:17:07 +0200 Subject: [PATCH 4/5] Bin_node: create separate lib for node config This commit takes all files pertaining to a node's configuration and puts them in a separate library. This allows other parts of the code to read configuration files. --- .gitlab/ci/jobs/packaging/opam_package.yml | 15 ++++++++--- docs/developer/event_logging_framework.rst | 2 +- docs/developer/howto-freeze-protocols.rst | 2 +- docs/user/multinetwork.rst | 2 +- dune-project | 1 + manifest/main.ml | 17 +++++++++++++ opam/octez-node-config.opam | 25 +++++++++++++++++++ opam/octez-node.opam | 1 + scripts/gen-genesis/gen_genesis.ml | 2 +- scripts/user_activated_upgrade.sh | 8 +++--- src/bin_node/dune | 2 ++ src/lib_node_config/dune | 25 +++++++++++++++++++ .../node_config_command.ml | 0 .../node_config_command.mli | 0 .../node_config_file.ml | 0 .../node_config_file.mli | 2 +- .../node_config_validation.ml | 0 .../node_config_validation.mli | 0 .../node_data_version.ml | 0 .../node_data_version.mli | 0 .../node_shared_arg.ml | 0 .../node_shared_arg.mli | 0 22 files changed, 91 insertions(+), 13 deletions(-) create mode 100644 opam/octez-node-config.opam create mode 100644 src/lib_node_config/dune rename src/{bin_node => lib_node_config}/node_config_command.ml (100%) rename src/{bin_node => lib_node_config}/node_config_command.mli (100%) rename src/{bin_node => lib_node_config}/node_config_file.ml (100%) rename src/{bin_node => lib_node_config}/node_config_file.mli (99%) rename src/{bin_node => lib_node_config}/node_config_validation.ml (100%) rename src/{bin_node => lib_node_config}/node_config_validation.mli (100%) rename src/{bin_node => lib_node_config}/node_data_version.ml (100%) rename src/{bin_node => lib_node_config}/node_data_version.mli (100%) rename src/{bin_node => lib_node_config}/node_shared_arg.ml (100%) rename src/{bin_node => lib_node_config}/node_shared_arg.mli (100%) diff --git a/.gitlab/ci/jobs/packaging/opam_package.yml b/.gitlab/ci/jobs/packaging/opam_package.yml index 3b20995115bf..d693fe6d900d 100644 --- a/.gitlab/ci/jobs/packaging/opam_package.yml +++ b/.gitlab/ci/jobs/packaging/opam_package.yml @@ -304,6 +304,13 @@ opam:octez-node: variables: package: octez-node +opam:octez-node-config: + extends: + - .opam_template + - .rules_template__trigger_opam_batch_3 + variables: + package: octez-node-config + opam:octez-protocol-compiler: extends: - .opam_template @@ -790,7 +797,7 @@ opam:tezos-embedded-protocol-006-PsCARTHA: opam:tezos-embedded-protocol-007-PsDELPH1: extends: - .opam_template - - .rules_template__trigger_opam_batch_3 + - .rules_template__trigger_opam_batch_4 variables: package: tezos-embedded-protocol-007-PsDELPH1 @@ -916,7 +923,7 @@ opam:tezos-expect-helper: opam:tezos-hacl: extends: - .opam_template - - .rules_template__trigger_opam_batch_6 + - .rules_template__trigger_opam_batch_7 variables: package: tezos-hacl @@ -1182,7 +1189,7 @@ opam:tezos-protocol-demo-noops: opam:tezos-protocol-environment: extends: - .opam_template - - .rules_template__trigger_opam_batch_5 + - .rules_template__trigger_opam_batch_6 variables: package: tezos-protocol-environment @@ -1427,7 +1434,7 @@ opam:tezos-shell: opam:tezos-shell-benchmarks: extends: - .opam_template - - .rules_template__trigger_opam_batch_4 + - .rules_template__trigger_opam_batch_5 variables: package: tezos-shell-benchmarks diff --git a/docs/developer/event_logging_framework.rst b/docs/developer/event_logging_framework.rst index 4bbf23c4f37e..bf078f5431a9 100644 --- a/docs/developer/event_logging_framework.rst +++ b/docs/developer/event_logging_framework.rst @@ -26,7 +26,7 @@ Simple events are record-like structures which are constructed directly programmatically). The API is less generic than the “Heavy” Events'. See for instance -:src:`src/bin_node/node_config_file.ml` (search for the string `declare_`): +:src:`src/lib_node_config/node_config_file.ml` (search for the string `declare_`): - one declares typed-record events with ``Internal_event.Simple.declare_``, diff --git a/docs/developer/howto-freeze-protocols.rst b/docs/developer/howto-freeze-protocols.rst index 533a4acfc8aa..a2c9b3555bf1 100644 --- a/docs/developer/howto-freeze-protocols.rst +++ b/docs/developer/howto-freeze-protocols.rst @@ -56,7 +56,7 @@ Remove Testnet From ``bin_node`` -------------------------------- Mentions of the protocol's testnets should be removed from the node executable. -In particular the file :src:`src/bin_node/node_config_file.ml` should be +In particular the file :src:`src/lib_node_config/node_config_file.ml` should be amended. The protocol plugin registration module should be removed from diff --git a/docs/user/multinetwork.rst b/docs/user/multinetwork.rst index 42df655c9927..8d2fdfd437e0 100644 --- a/docs/user/multinetwork.rst +++ b/docs/user/multinetwork.rst @@ -228,7 +228,7 @@ you can configure a custom network. Development ----------- -The list of built-in networks is in ``src/bin_node/node_config_file.ml``. +The list of built-in networks is in ``src/lib_node_config/node_config_file.ml``. Edit the ``builtin_blockchain_networks_with_tags`` variable in this file to add or remove built-in networks. diff --git a/dune-project b/dune-project index 9b4d42dce0f8..c7f9d4508e02 100644 --- a/dune-project +++ b/dune-project @@ -13,6 +13,7 @@ (package (name octez-codec)) (package (name octez-dal-node)) (package (name octez-node)) +(package (name octez-node-config)) (package (name octez-protocol-compiler)) (package (name octez-proxy-server)) (package (name octez-sc-rollup-client-alpha)) diff --git a/manifest/main.ml b/manifest/main.ml index d50f25638e22..29097fa47fa9 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -3190,6 +3190,22 @@ let octez_dal_node_lib = octez_client_base_unix |> open_; ] +let octez_node_config = + public_lib + "octez-node-config" + ~path:"src/lib_node_config" + ~synopsis:"Octez: `octez-node-config` library" + ~deps: + [ + octez_base |> open_ ~m:"TzPervasives" |> open_; + octez_stdlib_unix |> open_; + octez_shell_services |> open_; + octez_rpc_http |> open_; + octez_rpc_http_server |> open_; + octez_context |> open_; + octez_validation |> open_; + ] + let octez_scoru_wasm_tests_helpers = private_lib "test_scoru_wasm_test_helpers" @@ -5474,6 +5490,7 @@ let _octez_node = octez_base |> open_ ~m:"TzPervasives" |> open_; octez_base_unix; octez_version; + octez_node_config |> open_; octez_stdlib_unix |> open_; octez_shell_services |> open_; octez_rpc_http |> open_; diff --git a/opam/octez-node-config.opam b/opam/octez-node-config.opam new file mode 100644 index 000000000000..4372a79c4761 --- /dev/null +++ b/opam/octez-node-config.opam @@ -0,0 +1,25 @@ +# This file was automatically generated, do not edit. +# Edit file manifest/main.ml instead. +opam-version: "2.0" +maintainer: "contact@tezos.com" +authors: ["Tezos devteam"] +homepage: "https://www.tezos.com/" +bug-reports: "https://gitlab.com/tezos/tezos/issues" +dev-repo: "git+https://gitlab.com/tezos/tezos.git" +license: "MIT" +depends: [ + "dune" { >= "3.0" } + "tezos-base" + "tezos-stdlib-unix" + "tezos-shell-services" + "tezos-rpc-http" + "tezos-rpc-http-server" + "tezos-context" + "tezos-validation" +] +build: [ + ["rm" "-r" "vendors"] + ["dune" "build" "-p" name "-j" jobs] + ["dune" "runtest" "-p" name "-j" jobs] {with-test} +] +synopsis: "Octez: `octez-node-config` library" diff --git a/opam/octez-node.opam b/opam/octez-node.opam index 66d8b7ed9664..6f022aa17061 100644 --- a/opam/octez-node.opam +++ b/opam/octez-node.opam @@ -11,6 +11,7 @@ depends: [ "dune" { >= "3.0" } "tezos-base" "tezos-version" + "octez-node-config" "tezos-stdlib-unix" "tezos-shell-services" "tezos-rpc-http" diff --git a/scripts/gen-genesis/gen_genesis.ml b/scripts/gen-genesis/gen_genesis.ml index 2ad343c172a6..7dc8bd2bc765 100644 --- a/scripts/gen-genesis/gen_genesis.ml +++ b/scripts/gen-genesis/gen_genesis.ml @@ -56,7 +56,7 @@ let sandboxed_chain_name = "SANDBOXED_TEZOS" let () = echo "Here is the configuration that you can add to \ - src/bin_node/node_config_file.ml:" ; + src/lib_node_config/node_config_file.ml:" ; echo "" ; echo "let blockchain_network_%s =" (String.lowercase_ascii network_name) ; echo " make_blockchain_network" ; diff --git a/scripts/user_activated_upgrade.sh b/scripts/user_activated_upgrade.sh index 24a1fb6f02ff..d92005808070 100755 --- a/scripts/user_activated_upgrade.sh +++ b/scripts/user_activated_upgrade.sh @@ -66,8 +66,8 @@ else { {found=0; print } else { if (!found){print}} -}}' src/bin_node/node_config_file.ml > tmp_file - mv tmp_file src/bin_node/node_config_file.ml +}}' src/lib_node_config/node_config_file.ml > tmp_file + mv tmp_file src/lib_node_config/node_config_file.ml echo "The sandbox will now switch to $full_hash at level $level." else # we are in sandbox @@ -84,8 +84,8 @@ else { {found=0; print } else { if (!found){print}} -}}' src/bin_node/node_config_file.ml > tmp_file - mv tmp_file src/bin_node/node_config_file.ml +}}' src/lib_node_config/node_config_file.ml > tmp_file + mv tmp_file src/lib_node_config/node_config_file.ml sed -i.old "s/\$bin_dir\/..\/proto_alpha\/parameters\/sandbox-parameters.json/\$bin_dir\/..\/proto_${pred}_${pred_short_hash}\/parameters\/sandbox-parameters.json/" src/bin_client/octez-init-sandboxed-client.sh sed -i.old "s/activate_alpha()/activate_${pred}_${pred_short_hash}()/" src/bin_client/octez-init-sandboxed-client.sh diff --git a/src/bin_node/dune b/src/bin_node/dune index 5e0b74b9266a..badf4a3f87aa 100644 --- a/src/bin_node/dune +++ b/src/bin_node/dune @@ -10,6 +10,7 @@ tezos-base tezos-base.unix tezos-version + octez-node-config tezos-stdlib-unix tezos-shell-services tezos-rpc-http @@ -125,6 +126,7 @@ (:standard) -open Tezos_base.TzPervasives -open Tezos_base + -open Octez_node_config -open Tezos_stdlib_unix -open Tezos_shell_services -open Tezos_rpc_http diff --git a/src/lib_node_config/dune b/src/lib_node_config/dune new file mode 100644 index 000000000000..1a152fe80a78 --- /dev/null +++ b/src/lib_node_config/dune @@ -0,0 +1,25 @@ +; This file was automatically generated, do not edit. +; Edit file manifest/main.ml instead. + +(library + (name octez_node_config) + (public_name octez-node-config) + (instrumentation (backend bisect_ppx)) + (libraries + tezos-base + tezos-stdlib-unix + tezos-shell-services + tezos-rpc-http + tezos-rpc-http-server + tezos-context + tezos-validation) + (flags + (:standard) + -open Tezos_base.TzPervasives + -open Tezos_base + -open Tezos_stdlib_unix + -open Tezos_shell_services + -open Tezos_rpc_http + -open Tezos_rpc_http_server + -open Tezos_context + -open Tezos_validation)) diff --git a/src/bin_node/node_config_command.ml b/src/lib_node_config/node_config_command.ml similarity index 100% rename from src/bin_node/node_config_command.ml rename to src/lib_node_config/node_config_command.ml diff --git a/src/bin_node/node_config_command.mli b/src/lib_node_config/node_config_command.mli similarity index 100% rename from src/bin_node/node_config_command.mli rename to src/lib_node_config/node_config_command.mli diff --git a/src/bin_node/node_config_file.ml b/src/lib_node_config/node_config_file.ml similarity index 100% rename from src/bin_node/node_config_file.ml rename to src/lib_node_config/node_config_file.ml diff --git a/src/bin_node/node_config_file.mli b/src/lib_node_config/node_config_file.mli similarity index 99% rename from src/bin_node/node_config_file.mli rename to src/lib_node_config/node_config_file.mli index 957f865114ed..85a61ddbf6fe 100644 --- a/src/bin_node/node_config_file.mli +++ b/src/lib_node_config/node_config_file.mli @@ -64,7 +64,7 @@ type t = { p2p : p2p; rpc : rpc; log : Lwt_log_sink_unix.cfg; - internal_events : Internal_event_config.t; + internal_events : Tezos_base.Internal_event_config.t; shell : Shell_limits.limits; blockchain_network : blockchain_network; metrics_addr : string list; diff --git a/src/bin_node/node_config_validation.ml b/src/lib_node_config/node_config_validation.ml similarity index 100% rename from src/bin_node/node_config_validation.ml rename to src/lib_node_config/node_config_validation.ml diff --git a/src/bin_node/node_config_validation.mli b/src/lib_node_config/node_config_validation.mli similarity index 100% rename from src/bin_node/node_config_validation.mli rename to src/lib_node_config/node_config_validation.mli diff --git a/src/bin_node/node_data_version.ml b/src/lib_node_config/node_data_version.ml similarity index 100% rename from src/bin_node/node_data_version.ml rename to src/lib_node_config/node_data_version.ml diff --git a/src/bin_node/node_data_version.mli b/src/lib_node_config/node_data_version.mli similarity index 100% rename from src/bin_node/node_data_version.mli rename to src/lib_node_config/node_data_version.mli diff --git a/src/bin_node/node_shared_arg.ml b/src/lib_node_config/node_shared_arg.ml similarity index 100% rename from src/bin_node/node_shared_arg.ml rename to src/lib_node_config/node_shared_arg.ml diff --git a/src/bin_node/node_shared_arg.mli b/src/lib_node_config/node_shared_arg.mli similarity index 100% rename from src/bin_node/node_shared_arg.mli rename to src/lib_node_config/node_shared_arg.mli -- GitLab From abf51e4d8dc5cc47f595efe46a3907075e0d90f8 Mon Sep 17 00:00:00 2001 From: Pierre Boutillier Date: Thu, 3 Nov 2022 11:38:43 +0100 Subject: [PATCH 5/5] Test: Signer use a Port.fresh and wait for readiness --- tezt/lib_tezos/signer.ml | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/tezt/lib_tezos/signer.ml b/tezt/lib_tezos/signer.ml index 1c93fba63c89..1d366f3bd924 100644 --- a/tezt/lib_tezos/signer.ml +++ b/tezt/lib_tezos/signer.ml @@ -38,7 +38,8 @@ module Parameters = struct let base_default_name = "signer" - let default_uri = Uri.make ~scheme:"http" ~host:"localhost" ~port:17732 () + let default_uri () = + Uri.make ~scheme:"http" ~host:"localhost" ~port:(Port.fresh ()) () let default_colors = Log.Color. @@ -61,8 +62,8 @@ let set_ready signer = | Running status -> status.session_state.ready <- true) ; trigger_ready signer (Some ()) -let handle_raw_stdout signer line = - if line =~ rex "^.*accepting HTTP requests on port$" then set_ready signer +let handle_readiness signer (event : event) = + if event.name = "signer_listening.v0" then set_ready signer let base_dir_arg client = ["--base-dir"; client.base_dir] @@ -89,12 +90,15 @@ let spawn_import_secret_key signer (key : Account.key) = let import_secret_key signer (key : Account.key) = spawn_import_secret_key signer key |> Process.check -let create ?name ?color ?event_pipe ?base_dir ?(uri = Parameters.default_uri) - ?runner ?(keys = [Constant.bootstrap1]) () = +let create ?name ?color ?event_pipe ?base_dir ?uri ?runner + ?(keys = [Constant.bootstrap1]) () = let name = match name with None -> fresh_name () | Some name -> name in let base_dir = match base_dir with None -> Temp.dir name | Some dir -> dir in + let uri = + match uri with None -> Parameters.default_uri () | Some uri -> uri + in let signer = create ~path:signer_path @@ -104,7 +108,7 @@ let create ?name ?color ?event_pipe ?base_dir ?(uri = Parameters.default_uri) ?runner {runner; base_dir; uri; keys; pending_ready = []} in - on_stdout signer (handle_raw_stdout signer) ; + on_event signer (handle_readiness signer) ; let* () = Lwt_list.iter_s (import_secret_key signer) keys in return signer @@ -116,8 +120,10 @@ let run signer = let host = Option.value ~default:"localhost" (Uri.host signer.persistent_state.uri) in - let port = - Option.value ~default:7732 (Uri.port signer.persistent_state.uri) + let port_args = + match Uri.port signer.persistent_state.uri with + | None -> [] + | Some port -> ["--port"; Int.to_string port] in let arguments = [ @@ -128,9 +134,8 @@ let run signer = "signer"; "--address"; host; - "--port"; - Int.to_string port; ] + @ port_args in let arguments = if !passfile = "" then arguments @@ -165,6 +170,7 @@ let init ?name ?color ?event_pipe ?base_dir ?uri ?runner ?keys () = create ?name ?color ?event_pipe ?base_dir ?uri ?runner ?keys () in let* () = run signer in + let* () = wait_for_ready signer in return signer let restart signer = -- GitLab