diff --git a/CHANGES.rst b/CHANGES.rst index e6c54742bafbee0a162016126622e6cb7943e0b8..e80f6fa24ac2cc2f4edc388c792b253172fa12a8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -28,6 +28,46 @@ General Node ---- +- Changed the bounding specification of valid operations in the mempool: + + + Before, the number of valid **manager operations** in the mempool + was at most ``max_prechecked_manager_operations`` (default 5_000), + with no other constraints. (Operations to keep were selected + according to a "weight" that consists in the ratio of fee over + "resources"; the latter is the maximum between the following + ratios: operation gas over maximal allowed gas, and operation size + over maximal allowed size. The baker uses the same notion of + "weight" to select operations.) + + + Now, the number of valid **operations of any kind** is at most + ``max_operations`` (default 10_000), and also the **sum of the + sizes in bytes** of all valid operations is at most + ``max_total_bytes`` (default 10_000_000). See + [src/lib_shell/prevalidator_bounding.mli] for the reasoning behind + the default values. (Operations are selected according to the + protocol's ``compare_operations`` function, which currently orders + operations according to their validation pass (consensus is + highest and manager is lowest); note that two manager operations + are ordered using their fee over gas ratio.) + + The values of ``max_operations`` and ``max_total_bytes`` can be + retrieved with ``GET /chains//mempool/filter`` and configured + with ``POST /chains//mempool/filter`` (just as + ``max_prechecked_manager_operations`` used to be). As a result, the + JSON format of the outputs of these two RPCs and the input of the + second one have slightly changed; see their updated descriptions. + (MR :gl:`!6787`) + +- Errors ``prefilter.fees_too_low_for_mempool`` and + ``plugin.removed_fees_too_low_for_mempool`` have been replaced with + ``node.mempool.rejected_by_full_mempool`` and + ``node.mempool.removed_from_full_mempool`` with different + descriptions and messages. In particular, the + ``rejected_by_full_mempool`` error no longer indicates the minimal + fee needed by the rejected operation to be accepted by the full + mempool (however, we are working on providing this information + again). (MR :gl:`!6787`) + Client ------ diff --git a/manifest/main.ml b/manifest/main.ml index d7ec2a5d6758ce3bbd1615b6fe3135b74f1751be..b5d67eaf9fbcc4c9db53a11f7483c27c712c271f 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -5549,15 +5549,8 @@ let hash = Protocol.hash let _plugin_tests = opt_map (both plugin test_helpers) @@ fun (plugin, test_helpers) -> only_if active @@ fun () -> - tezt - [ - "test_consensus_filter"; - "test_filter_state"; - "test_plugin"; - "test_conflict_handler"; - "test_utils"; - "generators"; - ] + tests + ["test_conflict_handler"; "test_consensus_filter"] ~path:(path // "lib_plugin/test") ~with_macos_security_framework:true ~synopsis:"Tezos/Protocol: protocol plugin tests" @@ -6630,6 +6623,7 @@ let _octez_shell_tests = "test_node"; "test_peer_validator"; "test_prevalidation"; + "test_prevalidator_bounding"; "test_prevalidator_classification"; "test_prevalidator_classification_operations"; "test_prevalidator_pending_operations"; diff --git a/opam/tezos-protocol-plugin-016-PtMumbai-tests.opam b/opam/tezos-protocol-plugin-016-PtMumbai-tests.opam index 08253de819e1c38fba08b4c693ed032af5b53e2d..f4384edcb75c7b3a39845ff32a69d2451fb19bba 100644 --- a/opam/tezos-protocol-plugin-016-PtMumbai-tests.opam +++ b/opam/tezos-protocol-plugin-016-PtMumbai-tests.opam @@ -10,7 +10,6 @@ license: "MIT" depends: [ "dune" { >= "3.0" } "ocaml" { >= "4.14" } - "tezt" { with-test & >= "3.1.0" } "tezos-base" {with-test} "tezos-base-test-helpers" {with-test} "octez-alcotezt" {with-test} diff --git a/opam/tezos-protocol-plugin-017-PtNairob-tests.opam b/opam/tezos-protocol-plugin-017-PtNairob-tests.opam index 3422ca9a0473347e8f7b6084a15b204b3a2b43ca..d2ab6c3c062a473bc2cef79c8a56167fd3f41619 100644 --- a/opam/tezos-protocol-plugin-017-PtNairob-tests.opam +++ b/opam/tezos-protocol-plugin-017-PtNairob-tests.opam @@ -10,7 +10,6 @@ license: "MIT" depends: [ "dune" { >= "3.0" } "ocaml" { >= "4.14" } - "tezt" { with-test & >= "3.1.0" } "tezos-base" {with-test} "tezos-base-test-helpers" {with-test} "octez-alcotezt" {with-test} diff --git a/opam/tezos-protocol-plugin-alpha-tests.opam b/opam/tezos-protocol-plugin-alpha-tests.opam index b63b2cc5e8fc5f28e9b7ffb3e44c083ca67a9049..33488fe94cbc24c0f823358dbd1bd99b90ddb229 100644 --- a/opam/tezos-protocol-plugin-alpha-tests.opam +++ b/opam/tezos-protocol-plugin-alpha-tests.opam @@ -10,7 +10,6 @@ license: "MIT" depends: [ "dune" { >= "3.0" } "ocaml" { >= "4.14" } - "tezt" { with-test & >= "3.1.0" } "tezos-base" {with-test} "tezos-base-test-helpers" {with-test} "octez-alcotezt" {with-test} diff --git a/src/lib_shell/prevalidation.ml b/src/lib_shell/prevalidation.ml index 3dd8940456e0c2081e51e8dcd5597a4109b1e7ba..49a45e7f8da94b9b7e1ca94eb982d9a84d36e76b 100644 --- a/src/lib_shell/prevalidation.ml +++ b/src/lib_shell/prevalidation.ml @@ -43,9 +43,11 @@ module type T = sig type validation_state - type filter_state + type config - type filter_config + val default_config : config + + val config_encoding : config Data_encoding.t type chain_store @@ -66,7 +68,7 @@ module type T = sig val pre_filter : t -> - filter_config -> + config -> protocol_operation Shell_operation.operation -> [ `Passed_prefilter of Prevalidator_pending_operations.priority | Prevalidator_classification.error_classification ] @@ -82,51 +84,64 @@ module type T = sig * replacements val add_operation : - t -> filter_config -> protocol_operation operation -> add_result Lwt.t + t -> config -> protocol_operation operation -> add_result Lwt.t val remove_operation : t -> Operation_hash.t -> t module Internal_for_tests : sig val get_mempool_operations : t -> protocol_operation Operation_hash.Map.t - val get_filter_state : t -> filter_state - type mempool val set_mempool : t -> mempool -> t + + type bounding_state + + val get_bounding_state : t -> bounding_state + + val set_bounding_state : t -> bounding_state -> t end end -module MakeAbstract (Chain_store : CHAIN_STORE) (Filter : Shell_plugin.FILTER) : +module MakeAbstract + (Chain_store : CHAIN_STORE) + (Filter : Shell_plugin.FILTER) + (Bounding : Prevalidator_bounding.T + with type protocol_operation = Filter.Proto.operation) : T with type protocol_operation = Filter.Proto.operation and type validation_state = Filter.Proto.validation_state - and type filter_state = Filter.Mempool.state - and type filter_config = Filter.Mempool.config and type chain_store = Chain_store.chain_store - and type Internal_for_tests.mempool = Filter.Proto.Mempool.t = struct + and type Internal_for_tests.mempool = Filter.Proto.Mempool.t + and type Internal_for_tests.bounding_state = Bounding.state = struct module Proto = Filter.Proto type protocol_operation = Proto.operation type validation_state = Proto.validation_state - type filter_state = Filter.Mempool.state + type config = Filter.Mempool.config * Prevalidator_bounding.config + + let default_config = + (Filter.Mempool.default_config, Prevalidator_bounding.default_config) - type filter_config = Filter.Mempool.config + let config_encoding = + Data_encoding.merge_objs + Filter.Mempool.config_encoding + Prevalidator_bounding.config_encoding type chain_store = Chain_store.chain_store type operation = protocol_operation Shell_operation.operation - type create_aux_t = { + type t = { validation_info : Proto.Mempool.validation_info; mempool : Proto.Mempool.t; - head : Block_header.shell_header; - context : Tezos_protocol_environment.Context.t; + bounding_state : Bounding.state; + filter_info : Filter.Mempool.filter_info; } - let create_aux chain_store head timestamp = + let create_aux ?old_state chain_store head timestamp = let open Lwt_result_syntax in let* context = Chain_store.context chain_store head in let head_hash = Store.Block.hash head in @@ -141,35 +156,22 @@ module MakeAbstract (Chain_store : CHAIN_STORE) (Filter : Shell_plugin.FILTER) : let* validation_info, mempool = Proto.Mempool.init context chain_id ~head_hash ~head ~cache:`Lazy in - return {validation_info; mempool; head; context} - - type t = { - validation_info : Proto.Mempool.validation_info; - mempool : Proto.Mempool.t; - filter_state : Filter.Mempool.state; - } + let* filter_info = + match old_state with + | None -> Filter.Mempool.init context ~head + | Some old_state -> Filter.Mempool.flush old_state.filter_info ~head + in + return + {validation_info; mempool; bounding_state = Bounding.empty; filter_info} let create chain_store ~head ~timestamp = - let open Lwt_result_syntax in - let* {validation_info; mempool; head; context} = - create_aux chain_store head timestamp - in - let* filter_state = Filter.Mempool.init context ~head in - return {validation_info; mempool; filter_state} + create_aux chain_store head timestamp let flush chain_store ~head ~timestamp old_state = - let open Lwt_result_syntax in - let* {validation_info; mempool; head; context = _} = - create_aux chain_store head timestamp - in - let* filter_state = Filter.Mempool.flush old_state.filter_state ~head in - return {validation_info; mempool; filter_state} + create_aux ~old_state chain_store head timestamp - let pre_filter state filter_config op = - Filter.Mempool.pre_filter - ~filter_state:state.filter_state - filter_config - op.protocol + let pre_filter state (filter_config, (_ : Prevalidator_bounding.config)) op = + Filter.Mempool.pre_filter state.filter_info filter_config op.protocol type error_classification = Prevalidator_classification.error_classification @@ -206,7 +208,7 @@ module MakeAbstract (Chain_store : CHAIN_STORE) (Filter : Shell_plugin.FILTER) : assert false) let translate_proto_add_result (proto_add_result : Proto.Mempool.add_result) - op : (replacement, error_classification) result = + op : replacement tzresult = let open Result in let open Validation_errors in match proto_add_result with @@ -216,61 +218,50 @@ module MakeAbstract (Chain_store : CHAIN_STORE) (Filter : Shell_plugin.FILTER) : [Operation_replacement {old_hash = removed; new_hash = op.hash}] in return_some (removed, `Outdated trace) - | Unchanged -> - error - (classification_of_trace [Operation_conflict {new_hash = op.hash}]) - - (** Call [Filter.Mempool.add_operation_and_enforce_mempool_bound], - which ensures that the number of manager operations in the - mempool is bounded as specified in [filter_config]. - - The [state] argument is the prevalidation state (which has not - been modified yet). The [mempool] and [proto_add_result] are the - results of the protocol's [add_operation]. - - Maintaining this bound may require the removal of an operation - when the mempool was already full. In this case, this operation, - called [full_mempool_replacement], must also be removed from the - protocol's abstract [mempool]. - - Return the updated [state] (containing the updated protocol - [mempool]) and [filter_state], and the final [replacements], which - may have been mandated either by the protocol's [add_operation] - or by [Filter.Mempool.add_operation_and_enforce_mempool_bound] - (but not both: if the protocol already causes a replacement, then - the mempool is no longer full so there cannot be a - [full_mempool_replacement]. *) - let enforce_mempool_bound_and_update_states state filter_config - (mempool, proto_add_result) op : - (t * replacements, error_classification) result Lwt.t = - let open Lwt_result_syntax in - let*? proto_replacement = translate_proto_add_result proto_add_result op in - let* filter_state, full_mempool_replacement = - Filter.Mempool.add_operation_and_enforce_mempool_bound - ?replace:(Option.map fst proto_replacement) - filter_config - state.filter_state - (op.hash, op.protocol) + | Unchanged -> error [Operation_conflict {new_hash = op.hash}] + + (* Analyze the output of [Proto.Mempool.add_operation] to handle a + potential operation conflict. Then use the [Bounding] module to + ensure that the mempool remains bounded. + + If successful, return the updated [mempool] and [bounding_state], + as well as any operation [replacements] caused by either the + protocol mempool or the [Bounding] module. + + Note that the [mempool] argument, as part of the output of + [Proto.Mempool.add_operation], already contains the new operation + (if it has been accepted). So the only update it may need is the + removal of any operations replaced during [Bounding.add]. *) + let check_conflict_and_bound (mempool, proto_add_result) bounding_state + bounding_config op : + (Proto.Mempool.t * Bounding.state * replacements) tzresult = + let open Result_syntax in + let* proto_replacement = translate_proto_add_result proto_add_result op in + let bounding_state = + match proto_replacement with + | None -> bounding_state + | Some (replaced, _) -> Bounding.remove_operation bounding_state replaced + in + let* bounding_state, removed_by_bounding = + Option.to_result + ~none:[Validation_errors.Rejected_by_full_mempool op.hash] + (Bounding.add_operation bounding_state bounding_config op) in let mempool = - match full_mempool_replacement with - | `No_replace -> mempool - | `Replace (replace_oph, _) -> - Proto.Mempool.remove_operation mempool replace_oph + List.fold_left Proto.Mempool.remove_operation mempool removed_by_bounding in let replacements = - match (proto_replacement, full_mempool_replacement) with - | _, `No_replace -> Option.to_list proto_replacement - | None, `Replace repl -> [repl] - | Some _, `Replace _ -> - (* If there is a [proto_replacement], it gets removed from the - mempool before adding [op] so the mempool cannot be full. *) - assert false + Option.to_list proto_replacement + @ List.map + (fun removed -> + let err = [Validation_errors.Removed_from_full_mempool removed] in + (removed, classification_of_trace err)) + removed_by_bounding in - return ({state with mempool; filter_state}, replacements) + return (mempool, bounding_state, replacements) - let add_operation_result state filter_config op : - (t * operation * classification * replacements) tzresult Lwt.t = + let add_operation_result state (filter_config, bounding_config) op : + add_result tzresult Lwt.t = let open Lwt_result_syntax in let conflict_handler = Filter.Mempool.conflict_handler filter_config in let* proto_output = proto_add_operation ~conflict_handler state op in @@ -281,42 +272,49 @@ module MakeAbstract (Chain_store : CHAIN_STORE) (Filter : Shell_plugin.FILTER) : that the operation is individually valid, in particular its signature is correct. We record this so that any future signature check can be skipped. *) - let op = record_successful_signature_check op in - let*! res = - enforce_mempool_bound_and_update_states - state - filter_config + let valid_op = record_successful_signature_check op in + let res = + catch_e @@ fun () -> + check_conflict_and_bound proto_output - op + state.bounding_state + bounding_config + valid_op in match res with - | Ok (state, replacement) -> return (state, op, `Prechecked, replacement) - | Error err_class -> return (state, op, (err_class :> classification), []) - - let add_operation state filter_config op : add_result Lwt.t = + | Ok (mempool, bounding_state, replacement) -> + let state = {state with mempool; bounding_state} in + return (state, valid_op, `Prechecked, replacement) + | Error trace -> + (* We convert any error from [check_conflict_and_bound] into an + [add_result] here, rather than let [add_operation] below do + the same, so that we can return the updated [valid_op]. *) + return (state, valid_op, classification_of_trace trace, []) + + let add_operation state config op : add_result Lwt.t = let open Lwt_syntax in - let* res = - protect (fun () -> add_operation_result state filter_config op) - in + let* res = protect (fun () -> add_operation_result state config op) in match res with | Ok add_result -> return add_result | Error trace -> return (state, op, classification_of_trace trace, []) let remove_operation state oph = let mempool = Proto.Mempool.remove_operation state.mempool oph in - let filter_state = - Filter.Mempool.remove ~filter_state:state.filter_state oph - in - {state with mempool; filter_state} + let bounding_state = Bounding.remove_operation state.bounding_state oph in + {state with mempool; bounding_state} module Internal_for_tests = struct let get_mempool_operations {mempool; _} = Proto.Mempool.operations mempool - let get_filter_state {filter_state; _} = filter_state - type mempool = Proto.Mempool.t let set_mempool state mempool = {state with mempool} + + type bounding_state = Bounding.state + + let get_bounding_state {bounding_state; _} = bounding_state + + let set_bounding_state state bounding_state = {state with bounding_state} end end @@ -333,10 +331,9 @@ module Make (Filter : Shell_plugin.FILTER) : T with type protocol_operation = Filter.Proto.operation and type validation_state = Filter.Proto.validation_state - and type filter_state = Filter.Mempool.state - and type filter_config = Filter.Mempool.config and type chain_store = Store.chain_store = MakeAbstract (Production_chain_store) (Filter) + (Prevalidator_bounding.Make (Filter.Proto)) module Internal_for_tests = struct module type CHAIN_STORE = CHAIN_STORE diff --git a/src/lib_shell/prevalidation.mli b/src/lib_shell/prevalidation.mli index b891da3864c0b08f6b0690e77e933075d24cf98d..a40b03d2f5c5a26d84be5686470e3be58a0b290e 100644 --- a/src/lib_shell/prevalidation.mli +++ b/src/lib_shell/prevalidation.mli @@ -38,11 +38,18 @@ module type T = sig see {!Tezos_protocol_environment.PROTOCOL} *) type validation_state - (** Type {!Shell_plugin.FILTER.Mempool.state}. *) - type filter_state + (** Mempool configuration that groups both the plugin config + (e.g. minimal fee to pass the {!pre_filter}) and the bounding + config (e.g. max number of valid operations in the mempool). *) + type config - (** Type {!Shell_plugin.FILTER.Mempool.config}. *) - type filter_config + (** Default mempool configuration. *) + val default_config : config + + (** Encoding for {!config}. + + Internally an object without any variable fields. *) + val config_encoding : config Data_encoding.t (** The type implemented by {!Tezos_store.Store.chain_store} in production, and mocked in tests *) @@ -53,7 +60,7 @@ module type T = sig {!remove_operation}. This state notably contains a representation of the protocol - mempool, as well as the filter state. *) + mempool, as well as the bounding state. *) type t (** Create an empty state based on the [head] block. @@ -83,24 +90,22 @@ module type T = sig See [Shell_plugin.FILTER.Mempool.pre_filter]. *) val pre_filter : t -> - filter_config -> + config -> protocol_operation Shell_operation.operation -> [ `Passed_prefilter of Prevalidator_pending_operations.priority | Prevalidator_classification.error_classification ] Lwt.t (** Contain the hash and new classification of any operations that - had to be removed to make room for the newly validated - operation. *) + had to be removed to make room for a newly added operation. *) type replacements = (Operation_hash.t * Prevalidator_classification.error_classification) list (** Result of {!add_operation}. - Contain the updated (or unchanged) state {!t}, - the operation (in which [count_successful_prechecks] - has been incremented if appropriate), its classification, - and the potential {!replacements}. + Contain the updated (or unchanged) state {!t}, the operation (in + which the [signature_checked] field has been updated if + appropriate), its classification, and the potential {!replacements}. Invariant: [replacements] can only be non-empty when the classification is [`Prechecked]. *) @@ -110,24 +115,21 @@ module type T = sig * Prevalidator_classification.classification * replacements - (** Call the protocol [Mempool.add_operation] function, providing it - with the [conflict_handler] from the plugin. - - Then if the protocol accepts the operation, call the plugin - [add_operation_and_enforce_mempool_bound], which is responsible - for bounding the number of manager operations in the mempool. + (** Try and add an operation to the protocol's mempool; also ensure + that this mempool remains bounded (in terms of both operation + count and total byte size; the bounds are specified in the [config]). See {!add_result} for a description of the output. *) val add_operation : t -> - filter_config -> + config -> protocol_operation Shell_operation.operation -> add_result Lwt.t (** Remove an operation from the state. The state remains unchanged when the operation was not - present. *) + present (though not physically equal to the input state). *) val remove_operation : t -> Operation_hash.t -> t module Internal_for_tests : sig @@ -135,14 +137,20 @@ module type T = sig representation of the mempool. *) val get_mempool_operations : t -> protocol_operation Operation_hash.Map.t - (** Return the filter_state component of the state. *) - val get_filter_state : t -> filter_state - (** Type {!Tezos_protocol_environment.PROTOCOL.Mempool.t}. *) type mempool (** Modify the [mempool] field of the internal state [t]. *) val set_mempool : t -> mempool -> t + + (** Type [Prevalidator_bounding.T.state]. *) + type bounding_state + + (** Return the [bounding_state] component of the state. *) + val get_bounding_state : t -> bounding_state + + (** Modify the [bounding_state] component of the state. *) + val set_bounding_state : t -> bounding_state -> t end end @@ -151,8 +159,6 @@ module Make : functor (Filter : Shell_plugin.FILTER) -> T with type protocol_operation = Filter.Proto.operation and type validation_state = Filter.Proto.validation_state - and type filter_state = Filter.Mempool.state - and type filter_config = Filter.Mempool.config and type chain_store = Store.chain_store (**/**) @@ -180,12 +186,13 @@ module Internal_for_tests : sig module Make : functor (Chain_store : CHAIN_STORE) (Filter : Shell_plugin.FILTER) + (Bounding : Prevalidator_bounding.T + with type protocol_operation = Filter.Proto.operation) -> T with type protocol_operation = Filter.Proto.operation and type validation_state = Filter.Proto.validation_state - and type filter_state = Filter.Mempool.state - and type filter_config = Filter.Mempool.config and type chain_store = Chain_store.chain_store and type Internal_for_tests.mempool = Filter.Proto.Mempool.t + and type Internal_for_tests.bounding_state = Bounding.state end diff --git a/src/lib_shell/prevalidator_bounding.ml b/src/lib_shell/prevalidator_bounding.ml new file mode 100644 index 0000000000000000000000000000000000000000..9e7a90e453f0b9084ebfa4d8e07d29067c45bca3 --- /dev/null +++ b/src/lib_shell/prevalidator_bounding.ml @@ -0,0 +1,215 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Nomadic Labs, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +type config = {max_operations : int; max_total_bytes : int} + +let default_max_operations = 10_000 + +let default_max_total_bytes = 10_000_000 + +let default_config = + { + max_operations = default_max_operations; + max_total_bytes = default_max_total_bytes; + } + +let config_encoding : config Data_encoding.t = + let open Data_encoding in + conv + (fun {max_operations; max_total_bytes} -> (max_operations, max_total_bytes)) + (fun (max_operations, max_total_bytes) -> {max_operations; max_total_bytes}) + (obj2 + (dft "max_operations" uint16 default_config.max_operations) + (dft "max_total_bytes" (uint_like_n ()) default_config.max_total_bytes)) + +open Shell_operation + +(* Interface for a [Bounding] module. *) +module type T = sig + type state + + val empty : state + + type protocol_operation + + val add_operation : + state -> + config -> + protocol_operation operation -> + (state * Operation_hash.t list) option + + val remove_operation : state -> Operation_hash.t -> state +end + +(* Include [T] but additionally aware of the state's exact definition: + this is useful for the tests. *) +module type T_for_tests = sig + type protocol_operation + + type operation := protocol_operation Shell_operation.operation + + module Opset : Set.S with type elt = operation + + type state = { + opset : Opset.t; + ophmap : operation Operation_hash.Map.t; + minop : operation option; + cardinal : int; + total_bytes : int; + } + + include + T with type protocol_operation := protocol_operation and type state := state +end + +(* Build a [Bounding] module. *) +module Make (Proto : Tezos_protocol_environment.PROTOCOL) : + T_for_tests with type protocol_operation = Proto.operation = struct + type protocol_operation = Proto.operation + + type operation = protocol_operation Shell_operation.operation + + let compare_ops op1 op2 = + Proto.compare_operations (op1.hash, op1.protocol) (op2.hash, op2.protocol) + + module Opset = Set.Make (struct + type t = operation + + let compare = compare_ops + end) + + (** Internal overview of all the valid operations present in the mempool. + + Structural invariants: + - [opset] and [ophmap] contain the same operations. + - [minop] is the minimum of [opset] (or [None] when [opset] is empty). + - [cardinal] is the cardinal of [opset]. + - [total_bytes] is the sum of the byte sizes of all elements in [opset]. + + Bound invariants: + - [cardinal <= config.max_operations] + - [total_bytes <= config.max_total_bytes] *) + type state = { + opset : Opset.t; + (** Ordered set of valid operations in the mempool. Note that the + operations are ordered by the protocol's [compare_operations] + function, NOT by the size of their bytes. *) + ophmap : operation Operation_hash.Map.t; + (** Contain the same elements as [opset], indexed by their hash. *) + minop : operation option; + (** The smallest operation in [opset] according to the protocol's + [compare_operations] function (not necessarily the one with the + least bytes). This is [None] if and only if [opset] is empty. *) + cardinal : int; (** The number of operations in [opset]. *) + total_bytes : int; + (** The sum of the sizes in bytes of all the operations in [opset]. *) + } + + let empty = + { + opset = Opset.empty; + ophmap = Operation_hash.Map.empty; + minop = None; + cardinal = 0; + total_bytes = 0; + } + + (* Precondition: [op] is present in the [state]. *) + let remove_present state op = + let opset = Opset.remove op state.opset in + let minop = + match state.minop with + | None -> None (* This is impossible since [op] was in the [state]. *) + | Some minop -> + if compare_ops op minop <= 0 then + (* The removed [op] was the minimum. *) + Opset.min_elt opset + else state.minop + in + { + opset; + ophmap = Operation_hash.Map.remove op.hash state.ophmap; + minop; + cardinal = state.cardinal - 1; + total_bytes = state.total_bytes - op.size; + } + + (* Remove [oph] if it is in the [state], otherwise do nothing. *) + let remove_operation state oph = + match Operation_hash.Map.find oph state.ophmap with + | Some op -> remove_present state op + | None -> state + + let check_bound_invariants state config = + state.cardinal <= config.max_operations + && state.total_bytes <= config.max_total_bytes + + (* Remove the minimal operation until the bound invariants are restored. + Return the updated state and the list of removed operation hashes. *) + let enforce_bound_invariants state config = + let rec aux state removed = + if check_bound_invariants state config then (state, removed) + else + (* Invariants are broken: remove the minimal operation. *) + match state.minop with + | None -> + (* Should not happen: the empty set cannot break the invariants. *) + (state, removed) + | Some minop -> aux (remove_present state minop) (minop.hash :: removed) + in + aux state [] + + (* Precondition: [op] is valid (otherwise calling + [Proto.compare_operations] on it may return an error). *) + let add_operation state config op = + if Operation_hash.Map.mem op.hash state.ophmap then Some (state, []) + else + let state = + { + opset = Opset.add op state.opset; + ophmap = Operation_hash.Map.add op.hash op state.ophmap; + minop = + (match state.minop with + | None -> Some op + | Some minop -> + if compare_ops op minop < 0 then Some op else state.minop); + cardinal = state.cardinal + 1; + total_bytes = state.total_bytes + op.size; + } + in + let state, removed = enforce_bound_invariants state config in + if List.mem ~equal:Operation_hash.equal op.hash removed then + (* If the new operation needs to be immediately removed in + order to maintain the mempool bound invariants, then it + should actually be rejected. *) + None + else Some (state, removed) +end + +module Internal_for_tests = struct + module type T = T_for_tests + + module Make = Make +end diff --git a/src/lib_shell/prevalidator_bounding.mli b/src/lib_shell/prevalidator_bounding.mli new file mode 100644 index 0000000000000000000000000000000000000000..883b0b3979b4d73555110a6cf819b157eac904c2 --- /dev/null +++ b/src/lib_shell/prevalidator_bounding.mli @@ -0,0 +1,142 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Nomadic Labs, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(** Bound the valid operations in the mempool by limiting both their + cardinal and their total byte size. *) + +(** Mempool bounds. + + They can be retrieved/set using RPCs GET/POST + [/chains//mempool/filter]. *) +type config = { + max_operations : int; + (** Maximal allowed number of valid operations in the mempool. *) + max_total_bytes : int; + (** Maximal allowed sum of the byte sizes of all valid operations in + the mempool. *) +} + +(** Default {!field-max_total_bytes} is [10_000_000]. + + A block can have at most around 700k bytes of operations (see + [validation_passes] in [proto_xxx/lib_protocol/main.ml]). So with + this bound, a mempool can have the content of more than 10 blocks, + which is more than enough. *) +val default_max_total_bytes : int + +(** Default {!field-max_operations} is [10_000]. + + The smallest operations are around 140 bytes (e.g. 139 bytes for a + (pre)attestation, 146 bytes for a delegation) so a block can have + at most around 5_000 operations. But many operations are much + larger, so in practice a block contains much less operations, so + keeping 10_000 of them in the mempool is enough. *) +val default_max_operations : int + +(** Default bounds. *) +val default_config : config + +(** Encoding for {!config}. + + It is internally an object without any variable field, so it is + possible to use it in a {!Data_encoding.merge_objs}. *) +val config_encoding : config Data_encoding.t + +(** Interface of a mempool bounding module. *) +module type T = sig + (** Internal overview of all the valid operations present in the mempool. *) + type state + + (** Empty state containing no operations. *) + val empty : state + + (** Type [Tezos_protocol_environment.PROTOCOL.operation]. *) + type protocol_operation + + (** Try and add an operation to the state. + + When the mempool is not full (i.e. adding the operation does not + break the {!max_operations} nor the {!max_total_bytes} limits), + return the updated state and an empty list. + + When the mempool is full, return either: + - an updated state and a list of replaced operations, which are + all smaller than the new operation (according to the protocol's + [compare_operations] function) and have been removed from the + state to make room for the new operation, or + - [None] when it is not possible to make room for the new + operation by removing only operations that are smaller than it. + + When the operation is already present in the state, do nothing + i.e. return the unchanged state and an empty replacement list. + + Precondition: the operation has been validated by the protocol + in some context (otherwise calling the protocol's + [compare_operations] function on it may fail). *) + val add_operation : + state -> + config -> + protocol_operation Shell_operation.operation -> + (state * Operation_hash.t list) option + + (** Remove the operation from the state. + + Do nothing if the operation was not in the state + (the returned state is then physically equal to the input state). *) + val remove_operation : state -> Operation_hash.t -> state +end + +(** Build a mempool bounding module. *) +module Make (Proto : Tezos_protocol_environment.PROTOCOL) : + T with type protocol_operation = Proto.operation + +module Internal_for_tests : sig + (** Module type that includes [T] above, but is also aware of the + exact definition of the [state], so that the tests can access its + fields and check its invariants. *) + module type T = sig + type protocol_operation + + type operation := protocol_operation Shell_operation.operation + + module Opset : Set.S with type elt = operation + + type state = { + opset : Opset.t; + ophmap : operation Operation_hash.Map.t; + minop : operation option; + cardinal : int; + total_bytes : int; + } + + include + T + with type protocol_operation := protocol_operation + and type state := state + end + + module Make (Proto : Tezos_protocol_environment.PROTOCOL) : + T with type protocol_operation = Proto.operation +end diff --git a/src/lib_shell/prevalidator_internal.ml b/src/lib_shell/prevalidator_internal.ml index f7347e1d8c12e7973092fdb945a65e6dfd52dd40..cf79f8887d1ca1381570943ea35b4fdf500a7c2e 100644 --- a/src/lib_shell/prevalidator_internal.ml +++ b/src/lib_shell/prevalidator_internal.ml @@ -165,11 +165,8 @@ let mk_chain_tools (chain_db : Distributed_db.chain_db) : (** Module type used both in production and in tests. *) module type S = sig - (** Type instantiated by {!Filter.Mempool.state}. *) - type filter_state - - (** Type instantiated by {!Filter.Mempool.config}. *) - type filter_config + (** Type instantiated by {!Prevalidation.T.config}. *) + type config (** Similar to the type [operation] from the protocol, see {!Tezos_protocol_environment.PROTOCOL} *) @@ -188,7 +185,7 @@ module type S = sig (Classification.classification * protocol_operation operation) Lwt_watcher.input; mutable rpc_directory : types_state Tezos_rpc.Directory.t lazy_t; - mutable filter_config : filter_config; + mutable config : config; lock : Lwt_mutex.t; } @@ -254,17 +251,12 @@ module Make_s (Prevalidation_t : Prevalidation.T with type validation_state = Filter.Proto.validation_state - and type filter_state = Filter.Mempool.state - and type filter_config = Filter.Mempool.config and type protocol_operation = Filter.Proto.operation) : S - with type filter_state = Filter.Mempool.state - and type filter_config = Filter.Mempool.config + with type config = Prevalidation_t.config and type protocol_operation = Filter.Proto.operation and type prevalidation_t = Prevalidation_t.t = struct - type filter_state = Filter.Mempool.state - - type filter_config = Filter.Mempool.config + type config = Prevalidation_t.config type protocol_operation = Filter.Proto.operation @@ -277,7 +269,7 @@ module Make_s (Classification.classification * protocol_operation operation) Lwt_watcher.input; mutable rpc_directory : types_state Tezos_rpc.Directory.t lazy_t; - mutable filter_config : filter_config; + mutable config : config; lock : Lwt_mutex.t; } @@ -332,7 +324,7 @@ module Make_s let pre_filter pv ~notifier parsed_op : [Pending_ops.priority | `Drop] Lwt.t = let open Lwt_syntax in let+ v = - Prevalidation_t.pre_filter pv.validation_state pv.filter_config parsed_op + Prevalidation_t.pre_filter pv.validation_state pv.config parsed_op in match v with | (`Branch_delayed _ | `Branch_refused _ | `Refused _ | `Outdated _) as errs @@ -385,14 +377,14 @@ module Make_s operations whose classes are changed/impacted by this classification (eg. in case of operation replacement). *) - let classify_operation shell ~filter_config ~validation_state mempool op : + let classify_operation shell ~config ~validation_state mempool op : (prevalidation_t * Mempool.t * (protocol_operation operation * Classification.classification) trace) Lwt.t = let open Lwt_syntax in let* v_state, op, classification, replacements = - Prevalidation_t.add_operation validation_state filter_config op + Prevalidation_t.add_operation validation_state config op in let to_replace = List.filter_map @@ -428,7 +420,7 @@ module Make_s operations are advertised to the remote peers. However, if a peer requests our mempool, we advertise all our classified operations and all our pending operations. *) - let classify_pending_operations ~notifier shell filter_config state = + let classify_pending_operations ~notifier shell config state = let open Lwt_syntax in let* r = Pending_ops.fold_es @@ -441,7 +433,7 @@ module Make_s let* new_validation_state, new_mempool, to_handle = classify_operation shell - ~filter_config + ~config ~validation_state:acc_validation_state acc_mempool op @@ -503,7 +495,7 @@ module Make_s classify_pending_operations ~notifier pv.shell - pv.filter_config + pv.config pv.validation_state in pv.validation_state <- validation_state ; @@ -657,7 +649,7 @@ module Make_s let*! validation_state, delta_mempool, to_handle = classify_operation pv.shell - ~filter_config:pv.filter_config + ~config:pv.config ~validation_state:pv.validation_state Mempool.empty parsed_op @@ -869,8 +861,6 @@ module Make (Prevalidation_t : Prevalidation.T with type validation_state = Filter.Proto.validation_state - and type filter_state = Filter.Mempool.state - and type filter_config = Filter.Mempool.config and type protocol_operation = Filter.Proto.operation and type chain_store = Store.chain_store) : T = struct module S = Make_s (Filter) (Prevalidation_t) @@ -901,16 +891,16 @@ module Make type worker = Worker.infinite Worker.queue Worker.t - (** Returns a json describing the prevalidator's [filter_config]. + (** Return a json describing the prevalidator's [config]. The boolean [include_default] ([true] by default) indicates whether the json should include the fields which have a value equal to their default value. *) - let get_filter_config_json ?(include_default = true) pv = + let get_config_json ?(include_default = true) pv = let include_default_fields = if include_default then `Always else `Never in Data_encoding.Json.construct ~include_default_fields - Filter.Mempool.config_encoding - pv.filter_config + Prevalidation_t.config_encoding + pv.config let filter_validation_passes allowed_validation_passes (op : protocol_operation) = @@ -936,10 +926,7 @@ module Make !dir (Proto_services.S.Mempool.get_filter Tezos_rpc.Path.open_root) (fun pv params () -> - return - (get_filter_config_json - ~include_default:params#include_default - pv)) ; + return (get_config_json ~include_default:params#include_default pv)) ; dir := Tezos_rpc.Directory.register !dir @@ -949,13 +936,18 @@ module Make let* () = try let config = - Data_encoding.Json.destruct Filter.Mempool.config_encoding obj + Data_encoding.Json.destruct + Prevalidation_t.config_encoding + obj in - pv.filter_config <- config ; + pv.config <- config ; Lwt.return_unit with _ -> Events.(emit invalid_mempool_filter_configuration) () in - return_ok (get_filter_config_json pv)) ; + (* We return [get_config_json pv] rather than [obj] in + order to show omitted fields (which have been reset to + their default values), and also in case [obj] is invalid. *) + return_ok (get_config_json pv)) ; (* Ban an operation (from its given hash): remove it from the mempool if present. Add it to the set pv.banned_operations to prevent it from being fetched/processed/injected in the @@ -1370,10 +1362,10 @@ module Make validation_state; operation_stream = Lwt_watcher.create_input (); rpc_directory = build_rpc_directory w; - filter_config = + config = (* TODO: https://gitlab.com/tezos/tezos/-/issues/1725 initialize from config file *) - Filter.Mempool.default_config; + Prevalidation_t.default_config; lock = Lwt_mutex.create (); } in diff --git a/src/lib_shell/shell_operation.ml b/src/lib_shell/shell_operation.ml index 330f155077d180b48371d321eb331dea2ce65794..4c5c126658b976b17077e633cb6f8a8b97dd840c 100644 --- a/src/lib_shell/shell_operation.ml +++ b/src/lib_shell/shell_operation.ml @@ -30,6 +30,7 @@ type 'protocol_operation operation = { raw : Operation.t; protocol : 'protocol_operation; signature_checked : bool; + size : int; } let record_successful_signature_check op = {op with signature_checked = true} @@ -70,16 +71,14 @@ module MakeParser (Proto : Tezos_protocol_environment.PROTOCOL) : raw; protocol = {Proto.shell = raw.Operation.shell; protocol_data}; signature_checked = false; + size; } end module Internal_for_tests = struct - let to_raw {raw; _} = raw - - let hash_of {hash; _} = hash - - let make_operation op oph data = - {hash = oph; raw = op; protocol = data; signature_checked = false} + let make_operation ?(signature_checked = false) ?(size = 0) hash raw protocol + = + {hash; raw; protocol; signature_checked; size} let safe_binary_of_bytes = safe_binary_of_bytes end diff --git a/src/lib_shell/shell_operation.mli b/src/lib_shell/shell_operation.mli index defb30d8dd72626cb065453bb5a9a3bc384a92dd..5d75b6e1b2e40d604ef385029fe9577b843eba1c 100644 --- a/src/lib_shell/shell_operation.mli +++ b/src/lib_shell/shell_operation.mli @@ -47,6 +47,7 @@ type 'protocol_operation operation = private { validation context, it notably means that the signature is correct. Therefore, when this field is [true], we can tell the protocol to skip signature checks. *) + size : int; (** Size of the operation in bytes. *) } (** Return the operation with the {!signature_checked} field set to [true]. *) @@ -78,15 +79,22 @@ module MakeParser : functor (Proto : Tezos_protocol_environment.PROTOCOL) -> (**/**) module Internal_for_tests : sig - (** Returns the {!Operation.t} underlying an {!operation} *) - val to_raw : _ operation -> Operation.t + (** A constructor for the [operation] datatype. - (** The hash of an {!operation} *) - val hash_of : _ operation -> Operation_hash.t + It allows tests to bypass the checks and parsing done by the + [parse] function, and/or tune the values of the internal fields. - (** A constructor for the [operation] datatype. It by-passes the - checks done by the [parse] function. *) - val make_operation : Operation.t -> Operation_hash.t -> 'a -> 'a operation + [size] defaults to [0] for tests that don't care about it. + + [signature_checked] defaults to [false] (the initial value + that is always returned by [parse]). *) + val make_operation : + ?signature_checked:bool -> + ?size:int -> + Operation_hash.t -> + Operation.t -> + 'protocol_operation -> + 'protocol_operation operation (** [safe_binary_of_bytes encoding bytes] parses [bytes] using [encoding]. Any error happening during parsing becomes {!Parse_error}. diff --git a/src/lib_shell/shell_plugin.ml b/src/lib_shell/shell_plugin.ml index 6d1ca2a3d4f554f183538ead7a85692899cd4ccc..a99e0558232b0263142029e69fac2de22b753cba 100644 --- a/src/lib_shell/shell_plugin.ml +++ b/src/lib_shell/shell_plugin.ml @@ -34,40 +34,26 @@ module type FILTER = sig val default_config : config - type state + type filter_info val init : Tezos_protocol_environment.Context.t -> head:Tezos_base.Block_header.shell_header -> - state tzresult Lwt.t + filter_info tzresult Lwt.t val flush : - state -> head:Tezos_base.Block_header.shell_header -> state tzresult Lwt.t - - val remove : filter_state:state -> Operation_hash.t -> state + filter_info -> + head:Tezos_base.Block_header.shell_header -> + filter_info tzresult Lwt.t val pre_filter : + filter_info -> config -> - filter_state:state -> Proto.operation -> [ `Passed_prefilter of Prevalidator_pending_operations.priority | Prevalidator_classification.error_classification ] Lwt.t - val add_operation_and_enforce_mempool_bound : - ?replace:Operation_hash.t -> - config -> - state -> - Operation_hash.t * Proto.operation -> - ( state - * [ `No_replace - | `Replace of - Operation_hash.t * Prevalidator_classification.error_classification - ], - Prevalidator_classification.error_classification ) - result - Lwt.t - val conflict_handler : config -> Proto.Mempool.conflict_handler end end @@ -80,7 +66,7 @@ module type RPC = sig end module No_filter (Proto : Registered_protocol.T) : - FILTER with module Proto = Proto and type Mempool.state = unit = struct + FILTER with module Proto = Proto and type Mempool.filter_info = unit = struct module Proto = Proto module Mempool = struct @@ -90,19 +76,13 @@ module No_filter (Proto : Registered_protocol.T) : let default_config = () - type state = unit + type filter_info = unit let init _ ~head:_ = Lwt_result_syntax.return_unit - let remove ~filter_state _ = filter_state - let flush _ ~head:_ = Lwt_result_syntax.return_unit - let pre_filter _ ~filter_state:_ _ = - Lwt.return @@ `Passed_prefilter (`Low []) - - let add_operation_and_enforce_mempool_bound ?replace:_ _ filter_state _ = - Lwt_result.return (filter_state, `No_replace) + let pre_filter _ _ _ = Lwt.return @@ `Passed_prefilter (`Low []) let conflict_handler _ ~existing_operation ~new_operation = if Proto.compare_operations existing_operation new_operation < 0 then diff --git a/src/lib_shell/shell_plugin.mli b/src/lib_shell/shell_plugin.mli index befd3f7faf35a5f9e08b55b7d5e763a0a58b670e..4ac984943cf60742a21babf9a72cd07b8a6a1619 100644 --- a/src/lib_shell/shell_plugin.mli +++ b/src/lib_shell/shell_plugin.mli @@ -35,85 +35,57 @@ module type FILTER = sig val default_config : config - (** Internal state of the prevalidator filter *) - type state + (** Static internal information needed by {!pre_filter}. - (** Create an empty [state]. + It depends on the [head] block upon which a mempool is built. *) + type filter_info - Called only once when a prevalidator starts. *) + (** Create a {!filter_info} based on the [head] block. + + Should be called only once when a new prevalidator is started + for a new protocol. Subsequent {!filter_info}s should be + created using {!flush}. *) val init : Tezos_protocol_environment.Context.t -> head:Tezos_base.Block_header.shell_header -> - state tzresult Lwt.t + filter_info tzresult Lwt.t - (** Create a new empty [state] based on [head]. + (** Create a new {!filter_info} based on the [head] block. - Parts of the old [state] are recycled, so that this function - is more efficient than [init] and does not need a - [Tezos_protocol_environment.Context.t] argument. *) + Parts of the old {!filter_info} (which may have been built on + a different block) are recycled, so that this function is more + efficient than {!init} and does not need a + {!Tezos_protocol_environment.Context.t} argument. *) val flush : - state -> head:Tezos_base.Block_header.shell_header -> state tzresult Lwt.t - - (** [remove ~filter_state oph] removes the operation manager linked to - [oph] from the state of the filter *) - val remove : filter_state:state -> Operation_hash.t -> state - - (** [pre_filter config ~filter_state operation_data] - is called on arrival of an operation and after a flush of - the prevalidator. This function calls the [pre_filter] in the protocol - plugin and returns [`Passed_prefilter priority] if no error occurs during - checking of the [operation_data], where priority is the priority computed by - the protocol filter plug-in. More tests are done using the - [filter_state]. We classify an operation that passes the prefilter as - [`Passed_prefilter] since we do not know yet if the operation is - applicable or not. If an error occurs during the checks, this function - returns an error corresponding to the kind of the error returned by the - protocol. *) + filter_info -> + head:Tezos_base.Block_header.shell_header -> + filter_info tzresult Lwt.t + + (** Perform some light preliminary checks on the operation. + + If successful, return [`Passed_prefilter] with the priority of the + operation, based on the operation kind and potentially its + fee, gas, and size. If not, return a classification containing + the encountered error. + + Should be called on arrival of an operation and after a flush + of the prevalidator. *) val pre_filter : + filter_info -> config -> - filter_state:state -> Proto.operation -> [ `Passed_prefilter of Prevalidator_pending_operations.priority | Prevalidator_classification.error_classification ] Lwt.t - (** Add an operation to the filter {!state}. - - The operation should have been previously validated by the protocol. - - This function is responsible for bounding the number of - manager operations in the mempool. If the mempool is full and - the input operation is a manager operation, then it is compared - with the already present operation with minimal weight. Then - either the minimal operation is replaced, or the new operation - is rejected. - - If successful, return the updated state and possibly the - replaced minimal operation, otherwise return the error - classification for the new operation. - - If [replace] is provided, then it is removed from the state - before processing the new operation (in which case the mempool - can no longer be full, so this function will succeed and return - [`No_replace]). *) - val add_operation_and_enforce_mempool_bound : - ?replace:Operation_hash.t -> - config -> - state -> - Operation_hash.t * Proto.operation -> - ( state - * [ `No_replace - | `Replace of - Operation_hash.t * Prevalidator_classification.error_classification - ], - Prevalidator_classification.error_classification ) - result - Lwt.t - (** Return a conflict handler for [Proto.Mempool.add_operation]. See the documentation of type [Mempool.conflict_handler] in - e.g. [lib_protocol_environment/sigs/v8/updater.mli]. *) + e.g. [lib_protocol_environment/sigs/v8/updater.mli]. + + Precondition: both operations must be individually valid + (required by the protocol's operation comparison on which the + implementation of this function relies). *) val conflict_handler : config -> Proto.Mempool.conflict_handler end end @@ -128,7 +100,7 @@ end (** Dummy filter that does nothing *) module No_filter (Proto : Registered_protocol.T) : - FILTER with module Proto = Proto and type Mempool.state = unit + FILTER with module Proto = Proto (** This is a protocol specific module that is used to collect all the * protocol-specific metrics. This module diff --git a/src/lib_shell/test/dune b/src/lib_shell/test/dune index 11f1b9be1d3eacc91546fef181bc57d3a7eaeafa..6008b89187a16079fe7301cdc4460e0f34fe4fa3 100644 --- a/src/lib_shell/test/dune +++ b/src/lib_shell/test/dune @@ -55,6 +55,7 @@ test_node test_peer_validator test_prevalidation + test_prevalidator_bounding test_prevalidator_classification test_prevalidator_classification_operations test_prevalidator_pending_operations diff --git a/src/lib_shell/test/generators.ml b/src/lib_shell/test/generators.ml index 8075ee937626baa45a0f23c33f728006557cec34..c389cfbabad497b1f3ef96a03230b01cf06e35d0 100644 --- a/src/lib_shell/test/generators.ml +++ b/src/lib_shell/test/generators.ml @@ -113,8 +113,9 @@ let priority_gen () : Prevalidator_pending_operations.priority QCheck2.Gen.t = let operation_with_hash_gen ?proto_gen ?block_hash_t () : unit operation QCheck2.Gen.t = let open QCheck2.Gen in + let* signature_checked = bool in let+ oph, op = raw_operation_with_hash_gen ?proto_gen ?block_hash_t () in - Internal_for_tests.make_operation op oph () + Internal_for_tests.make_operation ~signature_checked oph op () let operation_with_hash_and_priority_gen ?proto_gen ?block_hash_t () : (unit operation * Prevalidator_pending_operations.priority) QCheck2.Gen.t = diff --git a/src/lib_shell/test/generators_tree.ml b/src/lib_shell/test/generators_tree.ml index 3f3b9621b19b54f9562589413b6f860423a8bde0..7f03282fad78a5be3772dcd4dc53d5cdb9c440fb 100644 --- a/src/lib_shell/test/generators_tree.ml +++ b/src/lib_shell/test/generators_tree.ml @@ -393,7 +393,7 @@ let old_mempool_gen (tree : Block.t Tree.tree) : List.map (fun (op : Operation.t) -> let hash = Operation.hash op in - Internal_for_tests.make_operation op hash ()) + Internal_for_tests.make_operation hash op ()) pairs in if elements = [] then QCheck2.Gen.return Operation_hash.Map.empty diff --git a/src/lib_shell/test/test_prevalidation.ml b/src/lib_shell/test/test_prevalidation.ml index 954b210c28588863dea70b9f518bff401fd89b01..2778c1cc0c76756d4a346e039a3281336455ad9f 100644 --- a/src/lib_shell/test/test_prevalidation.ml +++ b/src/lib_shell/test/test_prevalidation.ml @@ -123,7 +123,6 @@ module MakeFilter (Proto : Tezos_protocol_environment.PROTOCOL) : Shell_plugin.FILTER with type Proto.operation_data = Proto.operation_data and type Proto.operation = Proto.operation - and type Mempool.state = unit and type Proto.Mempool.t = Proto.Mempool.t = Shell_plugin.No_filter (struct let hash = Protocol_hash.zero @@ -132,6 +131,20 @@ module MakeFilter (Proto : Tezos_protocol_environment.PROTOCOL) : let complete_b58prefix _ = assert false end) +module Mock_bounding : + Prevalidator_bounding.T with type protocol_operation = Mock_protocol.operation = +struct + type state = unit + + let empty = () + + type protocol_operation = Mock_protocol.operation + + let add_operation _ _ _ = assert false + + let remove_operation _ _ = assert false +end + module MakePrevalidation = Prevalidation.Internal_for_tests.Make let now () = Time.System.to_protocol (Tezos_base.Time.System.now ()) @@ -146,7 +159,7 @@ let () = let open Lwt_result_syntax in let (module Chain_store) = make_chain_store ctxt in let module Filter = MakeFilter (Mock_protocol) in - let module P = MakePrevalidation (Chain_store) (Filter) in + let module P = MakePrevalidation (Chain_store) (Filter) (Mock_bounding) in let timestamp : Time.Protocol.t = now () in let head = Init.genesis_block ~timestamp ctxt in let* _ = P.create chain_store ~head ~timestamp in @@ -213,6 +226,26 @@ let assert_operation_conflict ~__LOC__ = function "Branch_delayed: [Operation_conflict]" cl +let assert_rejected_by_full_mempool ~__LOC__ = function + | `Branch_delayed [Validation_errors.Rejected_by_full_mempool _] -> () + | cl -> + unexpected_classification + ~__LOC__ + "Branch_delayed: [Rejected_by_full_mempool]" + cl + +let assert_replacement ~__LOC__ = function + (* Replaced by the protocol *) + | `Outdated [Validation_errors.Operation_replacement _] -> () + (* Replaced by the filter *) + | `Branch_delayed [Validation_errors.Removed_from_full_mempool _] -> () + | classification -> + unexpected_classification + ~__LOC__ + "Outdated: [Operation_replacement] or Branch_delayed: \ + [Removed_from_full_mempool]" + classification + let assert_exn ~__LOC__ = function | `Branch_delayed [Exn _] -> () | cl -> unexpected_classification ~__LOC__ "Branch_delayed: [Exn]" cl @@ -246,18 +279,6 @@ let () = (function Refused_error -> Some () | _ -> None) (fun () -> Refused_error) -let assert_replacement ~__LOC__ = function - (* Replaced by the protocol *) - | `Outdated [Validation_errors.Operation_replacement _] -> () - (* Replaced by the filter *) - | `Branch_delayed [Branch_delayed_error] -> () - | classification -> - unexpected_classification - ~__LOC__ - "Outdated: [Operation_replacement] or Branch_delayed: \ - [Branch_delayed_error]" - classification - let assert_branch_delayed_error ~__LOC__ = function | `Branch_delayed [Branch_delayed_error] -> () | cl -> @@ -396,106 +417,87 @@ module Toy_proto : end end -(** Possible outcomes of filter's - [Mempool.add_operation_and_enforce_mempool_bound] that we want to test. *) -type filter_add_outcome = - | F_no_replace (** Return [`No_replace]. *) - | F_replace (** Return [`Replace _]. *) - | F_branch_delayed (** Fail with a [`Temporary] error. *) - | F_branch_refused (** Fail with a [`Branch] error. *) - | F_refused (** Fail with a [`Permanent] error. *) - | F_crash (** Raise an exception. *) - -let filter_add_outcome_encoding = - Data_encoding.string_enum - [ - ("No_replace", F_no_replace); - ("Replace", F_replace); - ("Branch_delayed", F_branch_delayed); - ("Branch_refused", F_branch_refused); - ("Refused", F_refused); - ("Crash", F_crash); - ] - -let filter_add_outcome_gen = - (* We try to give higher weights to more usual outcomes, and in - particular to [F_no_replace] so that the number of operations in - the mempool can grow. *) - QCheck2.Gen.frequencyl +module Toy_filter = MakeFilter (Toy_proto) + +type bounding_outcome = + | B_success of int + (** Add the operation successfully, and replace [n] operations in the + process (or replace all operations in the state if there are less + than [n]). + In practice we generate [0 <= n <= 3], and very often [n = 0] + meaning that there is no replacement. *) + | B_none (** Return [None]. *) + | B_crash (** Raise an exception. *) + +(** This generator returns: + - with odds 3/5: [B_success 0] + - with odds 1/15 each: [B_success n], for [1 <= n <= 3] + - with odds 2/15: [B_none] + - with odds 1/15: [B_crash] + + We strongly favor [B_success 0] (aka add the operation without any + replacements) so that the mempool may grow on average. *) +let bounding_outcome_gen = + let open QCheck2.Gen in + frequency [ - (8, F_no_replace); - (4, F_replace); - (1, F_branch_delayed); - (1, F_branch_refused); - (1, F_refused); - (1, F_crash); + ( 12, + let* n = frequencyl [(9, 0); (1, 1); (1, 2); (1, 3)] in + return (B_success n) ); + (2, pure B_none); + (1, pure B_crash); ] -(** Toy mempool filter with an adjustable - [add_operation_and_enforce_mempool_bound] and an actual [state] that - keeps track of added operations. *) -module Toy_filter = struct - include MakeFilter (Toy_proto) - - module Mempool = struct - (* Once again, we hack this type to specify the desired outcome. *) - type config = filter_add_outcome - - let config_encoding = filter_add_outcome_encoding - - let default_config = F_no_replace - - type state = Operation_hash.Set.t - - let init _ ~head:_ = Lwt_result.return Operation_hash.Set.empty - - let flush _ ~head:_ = assert false - - let remove ~filter_state oph = Operation_hash.Set.remove oph filter_state - - let pre_filter _ ~filter_state:_ _ = assert false - - let add_operation_and_enforce_mempool_bound ?replace config filter_state - (oph, _op) = - let filter_state = - match replace with - | None -> filter_state - | Some replace_oph -> Operation_hash.Set.remove replace_oph filter_state +(** Toy bounding with an adjustable [add] function. + + As in [Toy_proto], the [state] has a dual role of tracking the + valid operations and providing [add] with the desired + [bounding_outcome]. + + Note that despite its name, the [Toy_bounding] does not bound the + mempool, but instead may return any output at any time, because + the purpose of the current file is to test the [Prevalidation] + component. The actual Bounding built by [Prevalidator_bounding.Make] + is tested by itself in [test_prevalidator_bounding.ml]. *) +module Toy_bounding : + Prevalidator_bounding.T + with type protocol_operation = Toy_proto.operation + and type state = Operation_hash.Set.t * bounding_outcome = struct + type state = Operation_hash.Set.t * bounding_outcome + + (* Similarly to [Toy_proto.Mempool.t], we initialize the [state] + with the outcome [B_success 0] so that it is easy to add + operations to the mempool and make it grow. *) + let empty = (Operation_hash.Set.empty, B_success 0) + + type protocol_operation = Toy_proto.operation + + (** Remove and return [n] random elements from [set]. + If [n > cardinal set], then return all the elements of [set] instead. + Not tail-rec because in practice it is used with n <= 3. *) + let rec pop_n set n = + if n <= 0 || Operation_hash.Set.is_empty set then (set, []) + else + let oph = + QCheck2.Gen.(generate1 (oneofl (Operation_hash.Set.elements set))) in - match config with - (* To be able to replace an operation, we need the state to be - non-empty and [replace] to be [None]. Indeed, if we have - already removed an operation because of [replace], then the - state is not full and the filter shouldn't also remove an - operation. If these conditions are not fulfilled, then - [F_replace] falls back to the behavior of [F_no_replace]. *) - | F_replace - when (not (Operation_hash.Set.is_empty filter_state)) - && Option.is_none replace -> - let replace_oph = - QCheck2.Gen.( - generate1 (oneofl (Operation_hash.Set.elements filter_state))) - in - let filter_state = - Operation_hash.Set.remove replace_oph filter_state - in - let filter_state = Operation_hash.Set.add oph filter_state in - let replacement = - (replace_oph, `Branch_delayed [Branch_delayed_error]) - in - Lwt_result.return (filter_state, `Replace replacement) - | F_no_replace | F_replace -> - let filter_state = Operation_hash.Set.add oph filter_state in - Lwt_result.return (filter_state, `No_replace) - | F_branch_delayed -> - Lwt_result.fail (`Branch_delayed [Branch_delayed_error]) - | F_branch_refused -> - Lwt_result.fail (`Branch_refused [Branch_refused_error]) - | F_refused -> Lwt_result.fail (`Refused [Refused_error]) - | F_crash -> assert false - - let conflict_handler _ ~existing_operation:_ ~new_operation:_ = assert false - end + let set = Operation_hash.Set.remove oph set in + let set, popped = pop_n set (n - 1) in + (set, oph :: popped) + + let add_operation (set, desired_outcome) _config {Shell_operation.hash; _} = + match desired_outcome with + | B_success n -> + let set, replaced = pop_n set n in + let set = Operation_hash.Set.add hash set in + Some ((set, desired_outcome), replaced) + | B_none -> None + | B_crash -> assert false + + let remove_operation ((set, outcome) as state) oph = + if Operation_hash.Set.mem oph set then + (Operation_hash.Set.remove oph set, outcome) + else state end (** Test [Prevalidation.add_operation]. @@ -512,59 +514,68 @@ let () = (* Number of operations that will be added. *) let nb_ops = 100 in let open Lwt_result_syntax in + let open Shell_operation in let (module Chain_store) = make_chain_store ctxt in - let module P = MakePrevalidation (Chain_store) (Toy_filter) in + let module P = MakePrevalidation (Chain_store) (Toy_filter) (Toy_bounding) in let open P.Internal_for_tests in - let add_op state (op, (proto_outcome, filter_outcome)) = + let add_op state (op, (proto_outcome, bounding_outcome)) = let proto_ophmap_before = get_mempool_operations state in - let filter_state_before = get_filter_state state in - assert ( - not (Operation_hash.Map.mem op.Shell_operation.hash proto_ophmap_before)) ; - assert (not (Operation_hash.Set.mem op.hash filter_state_before)) ; + let bounding_ophset_before = fst (get_bounding_state state) in + assert (not (Operation_hash.Map.mem op.hash proto_ophmap_before)) ; + assert (not (Operation_hash.Set.mem op.hash bounding_ophset_before)) ; let state = set_mempool state (proto_ophmap_before, proto_outcome) in - let*! ( state, - (_op : Mock_protocol.operation Shell_operation.operation), - classification, - replacements ) = - P.add_operation state filter_outcome op + let state = + set_bounding_state state (bounding_ophset_before, bounding_outcome) + in + let*! state, returned_op, classification, replacements = + P.add_operation state P.default_config op in (* Check the classification. *) - (match (proto_outcome, filter_outcome) with - | Proto_success _, (F_no_replace | F_replace) -> - assert_success ~__LOC__ classification + (match (proto_outcome, bounding_outcome) with + | Proto_success _, B_success _ -> assert_success ~__LOC__ classification + | Proto_success _, B_none -> + assert_rejected_by_full_mempool ~__LOC__ classification | Proto_unchanged, _ -> assert_operation_conflict ~__LOC__ classification - | Proto_branch_delayed, _ | Proto_success _, F_branch_delayed -> + | Proto_branch_delayed, _ -> assert_branch_delayed_error ~__LOC__ classification - | Proto_branch_refused, _ | Proto_success _, F_branch_refused -> + | Proto_branch_refused, _ -> assert_branch_refused_error ~__LOC__ classification - | Proto_refused, _ | Proto_success _, F_refused -> - assert_refused_error ~__LOC__ classification - | Proto_crash, _ | Proto_success _, F_crash -> + | Proto_refused, _ -> assert_refused_error ~__LOC__ classification + | Proto_crash, _ | Proto_success _, B_crash -> assert_exn ~__LOC__ classification) ; - (* Check whether the new operation has been added, whether there - is a replacement, and when there is one, whether it has been removed. *) + (* Check that the states have been correctly updated. *) let proto_ophmap = get_mempool_operations state in - let filter_state = get_filter_state state in - (match (proto_outcome, filter_outcome) with - | Proto_success proto_replacement, (F_no_replace | F_replace) -> ( + let bounding_ophset = fst (get_bounding_state state) in + let count_before = Operation_hash.Map.cardinal proto_ophmap_before in + let count = Operation_hash.Map.cardinal proto_ophmap in + (match (proto_outcome, bounding_outcome) with + | Proto_success proto_replacement, B_success nb_replaced_bounding -> assert (Operation_hash.Map.mem op.hash proto_ophmap) ; - assert (Operation_hash.Set.mem op.hash filter_state) ; - match (proto_replacement, filter_outcome) with - | (Replacement, _ | _, F_replace) - when not (Operation_hash.Map.is_empty proto_ophmap_before) -> ( - match replacements with - | [] | _ :: _ :: _ -> assert false - | [(removed, replacement_classification)] -> - assert (Operation_hash.Map.mem removed proto_ophmap_before) ; - assert (Operation_hash.Set.mem removed filter_state_before) ; - assert (not (Operation_hash.Map.mem removed proto_ophmap)) ; - assert (not (Operation_hash.Set.mem removed filter_state)) ; - assert_replacement ~__LOC__ replacement_classification) - | _ -> assert (List.is_empty replacements)) + assert (Operation_hash.Set.mem op.hash bounding_ophset) ; + List.iter + (fun (replaced, replacement_classification) -> + assert (Operation_hash.Map.mem replaced proto_ophmap_before) ; + assert (Operation_hash.Set.mem replaced bounding_ophset_before) ; + assert (not (Operation_hash.Map.mem replaced proto_ophmap)) ; + assert (not (Operation_hash.Set.mem replaced bounding_ophset)) ; + assert_replacement ~__LOC__ replacement_classification) + replacements ; + let nb_replaced_proto = + match proto_replacement with Replacement -> 1 | No_replacement -> 0 + in + let nb_replaced = nb_replaced_proto + nb_replaced_bounding in + let nb_replaced = min nb_replaced count_before in + assert (List.compare_length_with replacements nb_replaced = 0) ; + assert (count = count_before + 1 - nb_replaced) | _ -> assert (not (Operation_hash.Map.mem op.hash proto_ophmap)) ; - assert (not (Operation_hash.Set.mem op.hash filter_state)) ; - assert (List.is_empty replacements)) ; + assert (not (Operation_hash.Set.mem op.hash bounding_ophset)) ; + assert (List.is_empty replacements) ; + assert (count = count_before)) ; + (* Check that the operation has been correctly updated (or left alone). *) + (match proto_outcome with + | Proto_success _ | Proto_unchanged -> assert returned_op.signature_checked + | _ -> assert (returned_op == op)) ; Lwt.return state in let timestamp : Time.Protocol.t = now () in @@ -573,7 +584,7 @@ let () = let ops = mk_ops nb_ops in let outcomes = QCheck2.Gen.( - generate ~n:nb_ops (pair proto_outcome_gen filter_add_outcome_gen)) + generate ~n:nb_ops (pair proto_outcome_gen bounding_outcome_gen)) in let ops_and_outcomes, leftovers = List.combine_with_leftovers ops outcomes in assert (Option.is_none leftovers) ; @@ -581,12 +592,14 @@ let () = List.fold_left_s add_op prevalidation_state ops_and_outcomes in let final_proto_ophmap = get_mempool_operations final_prevalidation_state in - let final_filter_state = get_filter_state final_prevalidation_state in + let final_bounding_ophset = + fst (get_bounding_state final_prevalidation_state) + in assert ( Operation_hash.Map.cardinal final_proto_ophmap - = Operation_hash.Set.cardinal final_filter_state) ; + = Operation_hash.Set.cardinal final_bounding_ophset) ; Operation_hash.Map.iter - (fun oph _ -> assert (Operation_hash.Set.mem oph final_filter_state)) + (fun oph _ -> assert (Operation_hash.Set.mem oph final_bounding_ophset)) final_proto_ophmap ; return_unit @@ -605,7 +618,7 @@ let () = let nb_ops_to_remove = 30 in let open Lwt_result_syntax in let (module Chain_store) = make_chain_store ctxt in - let module P = MakePrevalidation (Chain_store) (Toy_filter) in + let module P = MakePrevalidation (Chain_store) (Toy_filter) (Toy_bounding) in let open P.Internal_for_tests in let timestamp : Time.Protocol.t = now () in let head = Init.genesis_block ~timestamp ctxt in @@ -614,13 +627,13 @@ let () = let oph = QCheck2.Gen.generate1 Generators.operation_hash_gen in let state = P.remove_operation state oph in assert (Operation_hash.Map.is_empty (get_mempool_operations state)) ; - assert (Operation_hash.Set.is_empty (get_filter_state state)) ; + assert (Operation_hash.Set.is_empty (fst (get_bounding_state state))) ; (* Prepare the initial state. *) let*! initial_state = List.fold_left_s (fun state op -> let*! state, _op, _classification, _replacement = - P.add_operation state F_no_replace op + P.add_operation state P.default_config op in Lwt.return state) state @@ -636,25 +649,25 @@ let () = let oph = random_oph_from_map proto_ophmap_before in let state = P.remove_operation state oph in let proto_ophmap = get_mempool_operations state in - let filter_state = get_filter_state state in + let bounding_ophset = fst (get_bounding_state state) in assert (not (Operation_hash.Map.mem oph proto_ophmap)) ; - assert (not (Operation_hash.Set.mem oph filter_state)) ; + assert (not (Operation_hash.Set.mem oph bounding_ophset)) ; let cardinal = Operation_hash.Map.cardinal proto_ophmap in assert (cardinal = cardinal_before - 1) ; - assert (Operation_hash.Set.cardinal filter_state = cardinal) ; + assert (Operation_hash.Set.cardinal bounding_ophset = cardinal) ; (state, proto_ophmap, cardinal)) else (* Remove a fresh operation. *) - let filter_state_before = get_filter_state state in + let bounding_ophset_before = get_bounding_state state in let oph = QCheck2.Gen.generate1 (Generators.fresh_oph_gen proto_ophmap_before) in let state = P.remove_operation state oph in let proto_ophmap = get_mempool_operations state in - let filter_state = get_filter_state state in + let bounding_ophset = get_bounding_state state in (* Internal states are physically unchanged. *) assert (proto_ophmap == proto_ophmap_before) ; - assert (filter_state == filter_state_before) ; + assert (bounding_ophset == bounding_ophset_before) ; (state, proto_ophmap, cardinal_before) in let rec fun_power f x n = if n <= 0 then x else fun_power f (f x) (n - 1) in @@ -664,10 +677,10 @@ let () = (initial_state, initial_proto_ophmap, initial_cardinal) nb_ops_to_remove in - let final_filter_state = get_filter_state final_state in - assert (Operation_hash.Set.cardinal final_filter_state = final_cardinal) ; + let final_bounding_ophset = fst (get_bounding_state final_state) in + assert (Operation_hash.Set.cardinal final_bounding_ophset = final_cardinal) ; assert ( Operation_hash.Map.for_all - (fun oph _op -> Operation_hash.Set.mem oph final_filter_state) + (fun oph _op -> Operation_hash.Set.mem oph final_bounding_ophset) final_proto_ophmap) ; return_unit diff --git a/src/lib_shell/test/test_prevalidator_bounding.ml b/src/lib_shell/test/test_prevalidator_bounding.ml new file mode 100644 index 0000000000000000000000000000000000000000..4bfc325748b5db432e0865dd66c2cfd094aa26e4 --- /dev/null +++ b/src/lib_shell/test/test_prevalidator_bounding.ml @@ -0,0 +1,716 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Nomadic Labs, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(** Testing + ------- + Component: Prevalidator_bounding + Invocation: dune exec src/lib_shell/test/main.exe -- -f test_prevalidator_bounding.ml + Subject: Unit tests for {!Prevalidator_bounding.T} +*) + +let register_test ~title ~additional_tags = + Test.register + ~__FILE__ + ~title:("Shell: Mempool bounding: " ^ title) + ~tags:(["mempool"; "bounding"] @ additional_tags) + +(** {2 Module instantiations} *) + +(** Same as [Mock_all_unit] except that [operation_data] is [int], so + that it can be used by [compare_operations]. We will refer to this + integer as the "weight" of an operation. *) +module Mock_protocol : + Tezos_protocol_environment.PROTOCOL + with type operation_data = int + and type operation_receipt = unit + and type validation_state = unit + and type application_state = unit = struct + include + Tezos_protocol_environment.Internal_for_tests.Environment_protocol_T_test + .Mock_all_unit + + type operation_data = int + + type operation = { + shell : Operation.shell_header; + protocol_data : operation_data; + } + + let operation_data_encoding = Data_encoding.int16 + + let operation_data_and_receipt_encoding = + Data_encoding.conv fst (fun n -> (n, ())) Data_encoding.int16 + + let acceptable_pass _ = assert false + + let compare_operations (oph1, {shell = _; protocol_data = n1}) + (oph2, {shell = _; protocol_data = n2}) = + Compare.or_else (Int.compare n1 n2) (fun () -> + Operation_hash.compare oph1 oph2) + + let validate_operation ?check_signature:_ = assert false + + let apply_operation _ = assert false + + module Mempool = struct + include Mempool + + type conflict_handler = + existing_operation:Tezos_crypto.Hashed.Operation_hash.t * operation -> + new_operation:Tezos_crypto.Hashed.Operation_hash.t * operation -> + [`Keep | `Replace] + + let add_operation ?check_signature:_ ?conflict_handler:_ _ _ _ = + assert false + + let merge ?conflict_handler:_ () () = assert false + + let operations () = assert false + end +end + +module Bounding = Prevalidator_bounding.Internal_for_tests.Make (Mock_protocol) +open Shell_operation +open Prevalidator_bounding +open Bounding + +type operation = protocol_operation Shell_operation.operation + +(** {2 Misc Helpers} *) + +let weight op = op.protocol.Mock_protocol.protocol_data + +let pp_op fmt op = Format.fprintf fmt "{s=%d;w=%d}" op.size (weight op) + +let pp_sep fmt () = Format.fprintf fmt "; " + +let pp_state fmt state = + Format.fprintf + fmt + "{ cardinal = %d; total_bytes = %d; ops are [ %a ]}" + state.cardinal + state.total_bytes + (Format.pp_print_seq ~pp_sep pp_op) + (Opset.to_seq state.opset) + +let pp_option pp fmt = function + | None -> Format.fprintf fmt "None" + | Some x -> Format.fprintf fmt "(Some %a)" pp x + +let pp_op_verbose fmt op = + Format.fprintf + fmt + "{h=%a;s=%d;w=%d}" + Operation_hash.pp_short + op.hash + op.size + (weight op) + +let pp_state_verbose fmt state = + let pp_op_seq = Format.pp_print_seq ~pp_sep pp_op_verbose in + Format.fprintf + fmt + "{ cardinal = %d; total_bytes = %d; min_op = %a;\n\ + \ opset = [ %a ]\n\ + \ ophmap = [ %a ]\n\ + }" + state.cardinal + state.total_bytes + (pp_option pp_op_verbose) + state.minop + pp_op_seq + (Opset.to_seq state.opset) + pp_op_seq + (Seq.map snd (Operation_hash.Map.to_seq state.ophmap)) + +(** Configuration used in the tests. *) +let config_for_tests = {max_operations = 10; max_total_bytes = 100} + +(** Well-typed shell header: it doesn't matter in the tests. *) +let shell_header = {Operation.branch = Block_hash.zero} + +(** Well-typed raw operation: it doesn't matter in the tests. *) +let default_raw_op = {Operation.shell = shell_header; proto = Bytes.empty} + +let compare_ops op1 op2 = + Mock_protocol.compare_operations + (op1.hash, op1.protocol) + (op2.hash, op2.protocol) + +(** Check {!Bounding.state} invariants (both structural invariants and + bound constraints specified in the given [config]). *) +let check_invariants ~__LOC__ config state = + let broken_invariant format = + Format.kasprintf + (fun description -> + Test.fail + "%s:\nBroken state invariant: %s\nstate = %a@." + __LOC__ + description + pp_state_verbose + state) + format + in + (* Structural invariants *) + (* Check cardinal. *) + if not (Opset.cardinal state.opset = state.cardinal) then + broken_invariant + "cardinal = %d should be equal to cardinal(opset) = %d" + state.cardinal + (Opset.cardinal state.opset) ; + if not (Operation_hash.Map.cardinal state.ophmap = state.cardinal) then + broken_invariant + "cardinal = %d should be equal to cardinal(ophmap) = %d" + state.cardinal + (Operation_hash.Map.cardinal state.ophmap) ; + (* Check minop. *) + if + not + (Option.equal + (fun op1 op2 -> Operation_hash.equal op1.hash op2.hash) + (Opset.min_elt state.opset) + state.minop) + then + broken_invariant + "minop = %a should be minimum(opset) = %a" + (pp_option pp_op) + state.minop + (pp_option pp_op) + (Opset.min_elt state.opset) ; + (* Check that ophmap and opset contain the same operations (we + already know that they have the same cardinal, so we only need to + test that one is included in the other), and check total_bytes. *) + let computed_total_bytes = + Opset.fold + (fun op sum_sizes -> + if not (Operation_hash.Map.mem op.hash state.ophmap) then + broken_invariant + "opset and ophmap should contain the same operations, but %a is \ + present is opset and not in ophmap." + Operation_hash.pp_short + op.hash ; + sum_sizes + op.size) + state.opset + 0 + in + if not (computed_total_bytes = state.total_bytes) then + broken_invariant + "total_bytes = %d should be the sum of the sizes of the elements of \ + opset which is %d." + state.total_bytes + computed_total_bytes ; + (* Bound invariants *) + if state.cardinal > config.max_operations then + broken_invariant + "cardinal = %d should be <= config.max_operations = %d." + state.cardinal + config.max_operations ; + if state.total_bytes > config.max_total_bytes then + broken_invariant + "total_bytes = %d should be <= config.max_total_bytes = %d." + state.total_bytes + config.max_total_bytes + +(** {2 Generators} *) + +let oph_gen = Generators.operation_hash_gen + +(** Default operation size generator: high chance of a "small" + operation, medium chance of a "medium" operation, and low chance of + a "large" operation. If [may_exceed_max_total_bytes] is true, then + there is also a low chance that the generated size is higher than + the maximal total size allowed by [config_for_tests]. *) +let size_gen ~may_exceed_max_total_bytes = + let max_size = config_for_tests.max_total_bytes in + let open QCheck2.Gen in + let frequences = + [(5, int_range 1 5); (3, int_range 6 30); (1, int_range 31 max_size)] + in + let frequences = + if may_exceed_max_total_bytes then + (1, int_range (max_size + 1) (10 * max_size)) :: frequences + else frequences + in + frequency frequences + +(** Default generator for weights. The bounds are chosen so that we + can manually create operations with weight around 1000 that are + smaller than all other generated operations. Always using weights + in the thousands range lowers the risk of confusing them with sizes + (which are in the 1-100 range). *) +let weight_gen = QCheck2.Gen.int_range 2000 3000 + +let make_op oph ~size ~weight = + let signature_checked = QCheck2.Gen.(generate1 bool) in + let data = {Mock_protocol.shell = shell_header; protocol_data = weight} in + Shell_operation.Internal_for_tests.make_operation + ~signature_checked + ~size + oph + default_raw_op + data + +let op_gen ~may_exceed_max_total_bytes = + let open QCheck2.Gen in + let* oph = oph_gen in + let* size = size_gen ~may_exceed_max_total_bytes in + let* weight = weight_gen in + return (make_op oph ~size ~weight) + +(* Precondition: maximal_size >= 1 *) +let size_gen_for_initial_state maximal_size = + let open QCheck2.Gen in + (* All bounds must be >= 1 and <= maximal_size, and the min must + be <= the max. *) + let low_min = 1 in + let low_max = max (maximal_size / 3) 1 in + let medium_min = min (low_max + 1) maximal_size in + let medium_max = max (maximal_size / 3) medium_min in + let high_min = min (medium_max + 1) maximal_size in + let high_max = maximal_size in + frequency + [ + (3, int_range low_min low_max); + (2, int_range medium_min medium_max); + (1, int_range high_min high_max); + ] + +(* Preconditions: n >= 1 and total_bytes >= n *) +let size_list_gen ~n ~total_bytes = + let open QCheck2.Gen in + let rec loop n remaining_size acc = + if n <= 1 then return (remaining_size :: acc) + else + let* size = size_gen_for_initial_state (remaining_size - (n - 1)) in + loop (n - 1) (remaining_size - size) (size :: acc) + in + loop n total_bytes [] + +let generate_fresh_oph ophmap = + QCheck2.Gen.generate1 (Generators.fresh_oph_gen ophmap) + +let rec fill_ophmap sizes weights ophmap = + match (sizes, weights) with + | [], [] -> ophmap + | size :: sizes, weight :: weights -> + let oph = generate_fresh_oph ophmap in + let ophmap = + Operation_hash.Map.add oph (make_op oph ~size ~weight) ophmap + in + fill_ophmap sizes weights ophmap + | _ -> + invalid_arg + "fill_ophmap: inconsistent number of hashes, sizes, and weights" + +let make_state sizes weights = + let ophmap = fill_ophmap sizes weights Operation_hash.Map.empty in + let opset, cardinal, total_bytes = + Operation_hash.Map.fold + (fun _oph op (opset, cardinal, size) -> + (Opset.add op opset, cardinal + 1, size + op.size)) + ophmap + (Opset.empty, 0, 0) + in + {opset; ophmap; minop = Opset.min_elt opset; cardinal; total_bytes} + +(** Generate a non-empty {!type-state} that satisfies {!val-config}. + (NB: Tests on an empty state are done separately.) *) +let state_gen ?cardinal ?total_bytes () = + let open QCheck2.Gen in + let* cardinal = + match cardinal with + | Some v -> pure v + | None -> int_range 1 config_for_tests.max_operations + in + let* total_bytes = + match total_bytes with + | Some v -> pure v + | None -> int_range cardinal config_for_tests.max_total_bytes + in + let* sizes = size_list_gen ~n:cardinal ~total_bytes in + let* weights = list_size (pure cardinal) weight_gen in + let state = make_state sizes weights in + assert (state.cardinal = cardinal && state.total_bytes = total_bytes) ; + return state + +type state_and_ops = { + state : state; + present_op : operation; + fresh_op : operation; +} + +let state_and_ops_gen_aux ?cardinal ?total_bytes () = + let open QCheck2.Gen in + let* state = state_gen ?cardinal ?total_bytes () in + let* present_op = oneofl (Opset.elements state.opset) in + let* fresh_oph = Generators.fresh_oph_gen state.ophmap in + let* size = size_gen ~may_exceed_max_total_bytes:true in + let* weight = weight_gen in + let fresh_op = make_op fresh_oph ~size ~weight in + return {state; present_op; fresh_op} + +let state_and_ops_gen = state_and_ops_gen_aux () + +(** Generate a full state, as well as present and fresh operations. + The state is full in terms of both operation cardinal and total size + with probability 1/2, or only either one with probability 1/4 each. *) +let full_state_and_ops_gen = + let open QCheck2.Gen in + let* both_full_cardinal_and_full_size = bool in + if both_full_cardinal_and_full_size then + state_and_ops_gen_aux + ~cardinal:config_for_tests.max_operations + ~total_bytes:config_for_tests.max_total_bytes + () + else + let* full_cardinal = bool in + if full_cardinal then + state_and_ops_gen_aux ~cardinal:config_for_tests.max_operations () + else state_and_ops_gen_aux ~total_bytes:config_for_tests.max_total_bytes () + +(** {2 Tests} *) + +(* Test addition to and removal from an empty state, and check that + removing the added operation goes back to an empty state. *) +let () = + register_test ~title:"add/remove (empty state)" ~additional_tags:["empty"] + @@ fun () -> + let config = config_for_tests in + let op = QCheck2.Gen.generate1 (op_gen ~may_exceed_max_total_bytes:false) in + (* Removal from an empty state does nothing: + the same state (physically) is returned. *) + let state = remove_operation empty op.hash in + assert (state == empty) ; + (* Add an operation to the empty state. *) + let state, replacements = + WithExceptions.Option.get ~loc:__LOC__ (add_operation empty config op) + in + (* Check that the state contains a single operation which is [op]. + Then the other fields are correct thanks to [check_invariants]. *) + assert (state.cardinal = 1) ; + assert (Opset.mem op state.opset) ; + check_invariants ~__LOC__ config state ; + assert (List.is_empty replacements) ; + (* Removing the added operation goes back to an empty state. *) + let state = remove_operation state op.hash in + assert (state = empty) ; + (* Adding an operation larger than the max_total_bytes fails. *) + let over_max_op = + make_op op.hash ~size:(config.max_total_bytes + 1) ~weight:(weight op) + in + assert (Option.is_none (add_operation empty config over_max_op)) ; + (* Adding an operation with exactly the max_total_bytes succeeds. *) + let max_op = + make_op op.hash ~size:config.max_total_bytes ~weight:(weight op) + in + assert (Option.is_some (add_operation empty config max_op)) ; + unit + +(** Return the cardinal and total size of all operations in opset that + are smaller than op. *) +let nb_and_size_smaller op opset = + let rec loop opset acc_nb acc_size = + match Opset.min_elt opset with + | Some smaller_op when compare_ops smaller_op op < 0 -> + let opset = Opset.remove smaller_op opset in + loop opset (acc_nb + 1) (acc_size + smaller_op.size) + | Some _ | None -> (acc_nb, acc_size) + in + loop opset 0 0 + +let check_successful_add ?expected_nb_replacements ~state_before ~state + ~replacements op = + check_invariants ~__LOC__ config_for_tests state ; + assert (Opset.mem op state.opset) ; + assert (Operation_hash.Map.mem op.hash state.ophmap) ; + let minop = WithExceptions.Option.get ~loc:__LOC__ state.minop in + let nb_replaced, size_replaced = + List.fold_left + (fun (nb_repl, size_repl) replaced_oph -> + (* Each replaced operation should be in state_before but not in + state, and should be smaller than the new minimal operation. *) + let replaced_op = + WithExceptions.Option.get + ~loc:__LOC__ + (Operation_hash.Map.find replaced_oph state_before.ophmap) + in + assert (not (Opset.mem replaced_op state.opset)) ; + assert (not (Operation_hash.Map.mem replaced_oph state.ophmap)) ; + assert (compare_ops replaced_op minop < 0) ; + (nb_repl + 1, size_repl + replaced_op.size)) + (0, 0) + replacements + in + assert (state.cardinal = state_before.cardinal - nb_replaced + 1) ; + assert (state.total_bytes = state_before.total_bytes - size_replaced + op.size) ; + Option.iter + (fun expected_nb_replacements -> + Check.((expected_nb_replacements = nb_replaced) int) + ~error_msg:"Expected %L replacements but got %R.") + expected_nb_replacements + +(* Preconditions: + - [state_before] verifies the state invariants + - [op] is not in [state_before] *) +let check_add_fresh state_before op = + assert (not (Operation_hash.Map.mem op.hash state_before.ophmap)) ; + match add_operation state_before config_for_tests op with + | Some (state, replacements) -> + (* The new operation has been successfully added. *) + check_successful_add ~state_before ~state ~replacements op + | None -> + (* The new operation could not be added. This should only happen + when removing all operations that are smaller is not enough to + make room for it. *) + let nb_smaller, size_smaller = + nb_and_size_smaller op state_before.opset + in + assert ( + state_before.cardinal - nb_smaller + 1 > config_for_tests.max_operations + || state_before.total_bytes - size_smaller + op.size + > config_for_tests.max_total_bytes) + +(* Preconditions: + - [state_before] verifies the state invariants + - [op] is present in [state_before] *) +let check_add_present state_before op = + assert (Opset.mem op state_before.opset) ; + let res = add_operation state_before config_for_tests op in + (* add_operation should return the same state (physical equality + intended) and an empty list of replacements. *) + let state, replacements = WithExceptions.Option.get ~loc:__LOC__ res in + assert (state == state_before) ; + assert (List.is_empty replacements) + +(* Preconditions: + - [state_before] verifies the state invariants + - [op] is present in [state_before] *) +let check_remove_present state_before op = + assert (Opset.mem op state_before.opset) ; + let state = remove_operation state_before op.hash in + check_invariants ~__LOC__ config_for_tests state ; + assert (not (Opset.mem op state.opset)) ; + assert (not (Operation_hash.Map.mem op.hash state.ophmap)) ; + assert (state.cardinal = state_before.cardinal - 1) ; + assert (state.total_bytes = state_before.total_bytes - op.size) + +(* Preconditions: + - [state_before] verifies the state invariants + - [op] is not in [state_before] *) +let check_remove_fresh state_before op = + assert (not (Operation_hash.Map.mem op.hash state_before.ophmap)) ; + (* Should return the same state (physical equality intended). *) + let state = remove_operation state_before op.hash in + assert (state == state_before) + +let test_add_remove {state; present_op; fresh_op} = + Log.debug + "Testing with\n state: %a\n present_op: %a\n fresh_op: %a" + pp_state + state + pp_op + present_op + pp_op + fresh_op ; + check_add_fresh state fresh_op ; + check_add_present state present_op ; + check_remove_present state present_op ; + check_remove_fresh state fresh_op + +(* Test the [add_operation] and [remove_operation] functions on random + non-empty states. Various scenarios involving either function are + grouped in this single test in order to mutualize the generation of + random states. *) +let () = + register_test ~title:"add/remove (random state)" ~additional_tags:["random"] + @@ fun () -> + let state_and_ops_list = QCheck2.Gen.generate ~n:50 state_and_ops_gen in + List.iter test_add_remove state_and_ops_list ; + unit + +(* Test the [add_operation] and [remove_operation] functions on random + full states (full cardinal or full total byte size or + both). Various scenarios involving either function are grouped in + this single test in order to mutualize the generation of random + full states. *) +let () = + register_test + ~title:"add/remove (random full state)" + ~additional_tags:["random"; "full"] + @@ fun () -> + let state_and_ops_list = QCheck2.Gen.generate ~n:20 full_state_and_ops_gen in + List.iter test_add_remove state_and_ops_list ; + unit + +let check_add_fails ~__LOC__ state op = + Log.debug "Check failure: add op %a to %a" pp_op op pp_state state ; + if Option.is_some (add_operation state config_for_tests op) then + Test.fail "%s: add_operation unexpectedly succeeded." __LOC__ + +let add_successfully ~__LOC__ ~expected_nb_replacements state_before op = + Log.debug "Check success: add op %a to %a" pp_op op pp_state state_before ; + let ((state, replacements) as res) = + WithExceptions.Option.get ~loc:__LOC__ + @@ add_operation state_before config_for_tests op + in + (try + check_successful_add + ~expected_nb_replacements + ~state_before + ~state + ~replacements + op + with exn -> + Test.fail + "%s: Unexpected output while adding operation:\n%s" + __LOC__ + (Printexc.to_string exn)) ; + res + +let check_add_succeeds ~__LOC__ ~expected_nb_replacements state op = + ignore (add_successfully ~__LOC__ ~expected_nb_replacements state op) + +(* Test addition to a state that has the maximal number of operations. *) +let () = + register_test + ~title:"add to full state (full cardinal)" + ~additional_tags:["full"] + @@ fun () -> + (* State with 9 operations (out of config_for_tests.max_operations = 10) + and total size 50 (out of max_total_bytes = 100) and all operation + weights between 2000 and 3000 (see [weight_gen]). *) + let state = + QCheck2.Gen.generate1 (state_gen ~cardinal:9 ~total_bytes:50 ()) + in + (* Add an operation with weight 1000 and size 1. The state is now full. *) + let oph1000 = generate_fresh_oph state.ophmap in + let op1000 = make_op oph1000 ~size:1 ~weight:1000 in + let state, _replacements = + add_successfully ~__LOC__ ~expected_nb_replacements:0 state op1000 + in + assert (state.minop = Some op1000) ; + assert (state.cardinal = config_for_tests.max_operations) ; + (* Adding an operation with weight 999 fails. *) + let fresh_oph = generate_fresh_oph state.ophmap in + check_add_fails ~__LOC__ state (make_op fresh_oph ~size:1 ~weight:999) ; + (* Adding an operation with weight 1001 succeeds and replaces the + operation with weight 1000. *) + let op1001 = make_op fresh_oph ~size:1 ~weight:1001 in + let state, replacements = + add_successfully ~__LOC__ ~expected_nb_replacements:1 state op1001 + in + assert (state.minop = Some op1001) ; + assert (replacements = [oph1000]) ; + unit + +(* Test addition to a state that has the maximal total size, or close to it. *) +let () = + register_test + ~title:"add to (almost) full state (full bytes)" + ~additional_tags:["full"] + @@ fun () -> + (* Craft a state with full total size. *) + let sizes = [10; 40; 20; 1; 29] in + let weights = [1000; 1100; 1200; 1300; 1400] in + let state = make_state sizes weights in + assert (state.total_bytes = config_for_tests.max_total_bytes) ; + let fresh_oph = generate_fresh_oph state.ophmap in + (* Adding operations with insufficient weight for their size. *) + check_add_fails ~__LOC__ state (make_op fresh_oph ~size:1 ~weight:999) ; + check_add_fails ~__LOC__ state (make_op fresh_oph ~size:11 ~weight:1050) ; + check_add_fails ~__LOC__ state (make_op fresh_oph ~size:51 ~weight:1150) ; + check_add_fails ~__LOC__ state (make_op fresh_oph ~size:90 ~weight:1350) ; + (* Adding operations with enough weight for their size. *) + check_add_succeeds ~__LOC__ ~expected_nb_replacements:1 state + @@ make_op fresh_oph ~size:10 ~weight:1001 ; + check_add_succeeds ~__LOC__ ~expected_nb_replacements:2 state + @@ make_op fresh_oph ~size:50 ~weight:1101 ; + check_add_succeeds ~__LOC__ ~expected_nb_replacements:4 state + @@ make_op fresh_oph ~size:71 ~weight:1301 ; + (* Replace the operation with weight 1000 and size 10 with an operation + with weight 1001 and size 5 to have an almost full state. *) + let state, _replacements = + add_successfully ~__LOC__ ~expected_nb_replacements:1 state + @@ make_op fresh_oph ~size:5 ~weight:1001 + in + assert (state.total_bytes = 95) ; + let fresh_oph = generate_fresh_oph state.ophmap in + (* Adding operations with insufficient weight for their size. *) + check_add_fails ~__LOC__ state (make_op fresh_oph ~size:6 ~weight:1000) ; + check_add_fails ~__LOC__ state (make_op fresh_oph ~size:30 ~weight:1099) ; + check_add_fails ~__LOC__ state (make_op fresh_oph ~size:60 ~weight:1199) ; + (* Adding operations with enough weight for their size. *) + check_add_succeeds ~__LOC__ ~expected_nb_replacements:0 state + @@ make_op fresh_oph ~size:5 ~weight:0 ; + check_add_succeeds ~__LOC__ ~expected_nb_replacements:1 state + @@ make_op fresh_oph ~size:10 ~weight:1002 ; + check_add_succeeds ~__LOC__ ~expected_nb_replacements:3 state + @@ make_op fresh_oph ~size:60 ~weight:1201 ; + check_add_succeeds ~__LOC__ ~expected_nb_replacements:5 state + @@ make_op fresh_oph ~size:100 ~weight:2000 ; + unit + +let init_list n f = + WithExceptions.Result.get_ok + ~loc:__LOC__ + (List.init ~when_negative_length:() n f) + +(* Test addition to a state that has both the maximal number of + operations and the maximal total size. *) +let () = + register_test + ~title:"add to full state (full cardinal and full bytes)" + ~additional_tags:["full"] + @@ fun () -> + (* Craft a state with full cardinal and full total size. Operations all + have size 10 and their weights are [1000; 1100; 1200; ... ; 1900]. *) + let sizes = init_list 10 (fun _n -> 10) in + let weights = init_list 10 (fun n -> 1000 + (100 * n)) in + let state = make_state sizes weights in + assert (state.cardinal = config_for_tests.max_operations) ; + assert (state.total_bytes = config_for_tests.max_total_bytes) ; + let fresh_oph = generate_fresh_oph state.ophmap in + (* Adding operations with insufficient weight for their size. *) + check_add_fails ~__LOC__ state (make_op fresh_oph ~size:1 ~weight:999) ; + check_add_fails ~__LOC__ state (make_op fresh_oph ~size:11 ~weight:1099) ; + check_add_fails ~__LOC__ state (make_op fresh_oph ~size:31 ~weight:1299) ; + check_add_fails ~__LOC__ state (make_op fresh_oph ~size:91 ~weight:1899) ; + (* Insufficient weight; the cardinal is full, regardless of the size. *) + check_add_fails ~__LOC__ state (make_op fresh_oph ~size:0 ~weight:999) ; + (* Adding operations with enough weight for their size. *) + check_add_succeeds ~__LOC__ ~expected_nb_replacements:1 state + @@ make_op fresh_oph ~size:1 ~weight:1001 ; + check_add_succeeds ~__LOC__ ~expected_nb_replacements:2 state + @@ make_op fresh_oph ~size:11 ~weight:1101 ; + check_add_succeeds ~__LOC__ ~expected_nb_replacements:4 state + @@ make_op fresh_oph ~size:31 ~weight:1301 ; + check_add_succeeds ~__LOC__ ~expected_nb_replacements:10 state + @@ make_op fresh_oph ~size:91 ~weight:1901 ; + (* Don't replace more operations than necessary, even with a high weight. *) + check_add_succeeds ~__LOC__ ~expected_nb_replacements:1 state + @@ make_op fresh_oph ~size:1 ~weight:3000 ; + check_add_succeeds ~__LOC__ ~expected_nb_replacements:4 state + @@ make_op fresh_oph ~size:40 ~weight:3000 ; + unit diff --git a/src/lib_shell/test/test_prevalidator_classification_operations.ml b/src/lib_shell/test/test_prevalidator_classification_operations.ml index cceef875844455f7993a9d736cafc97a25181151..b7e44c077df552d9bd55b64b4eddc8a551c8b438 100644 --- a/src/lib_shell/test/test_prevalidator_classification_operations.ml +++ b/src/lib_shell/test/test_prevalidator_classification_operations.ml @@ -40,7 +40,7 @@ module Tree = Generators_tree.Tree module List_extra = Generators_tree.List_extra module Block = Generators_tree.Block -let make_operation = Shell_operation.Internal_for_tests.make_operation +let parse hash raw = Some (Internal_for_tests.make_operation hash raw ()) (** Function to unwrap an [option] when it MUST be a [Some] *) let force_opt ~loc = function @@ -119,8 +119,6 @@ module Handle_operations = struct Classification.( create {map_size_limit = 1; on_discarded_operation = (fun _oph -> ())}) - let parse raw hash = Some (make_operation hash raw ()) - (** Test that operations returned by [handle_live_operations] are all in the alive branch. *) let test_handle_live_operations_is_branch_alive = @@ -415,7 +413,6 @@ module Recyle_operations = struct assume @@ Option.is_some pair_blocks_opt ; let from_branch, to_branch = force_opt ~loc:__LOC__ pair_blocks_opt in let chain = Generators_tree.classification_chain_tools tree in - let parse raw hash = Some (make_operation hash raw ()) in let actual : unit operation Op_map.t = Classification.recycle_operations ~block_store:Block.tools @@ -482,7 +479,6 @@ module Recyle_operations = struct expected_from_classification) expected_from_pending in - let parse raw hash = Some (make_operation hash raw ()) in let actual : Operation_hash.Set.t = Classification.recycle_operations ~block_store:Block.tools @@ -527,7 +523,6 @@ module Recyle_operations = struct in let from_branch, to_branch = force_opt ~loc:__LOC__ pair_blocks_opt in let chain = Generators_tree.classification_chain_tools tree in - let parse raw hash = Some (make_operation hash raw ()) in let () = Classification.recycle_operations ~block_store:Block.tools diff --git a/src/lib_shell/test/test_prevalidator_pending_operations.ml b/src/lib_shell/test/test_prevalidator_pending_operations.ml index 1336bf734c25e908163b19064365d00c191e9bdf..8eb7f2cd0a35cb07904e9abec1247c32aba500f1 100644 --- a/src/lib_shell/test/test_prevalidator_pending_operations.ml +++ b/src/lib_shell/test/test_prevalidator_pending_operations.ml @@ -37,12 +37,9 @@ module CompareListQ = Compare.List (Q) let pending_of_list = List.fold_left - (fun pendings (op, priority) -> - if - Operation_hash.Set.mem - (Shell_operation.Internal_for_tests.hash_of op) - (Pending_ops.hashes pendings) - then (* no duplicate hashes *) + (fun pendings ((op : _ Shell_operation.operation), priority) -> + if Operation_hash.Set.mem op.hash (Pending_ops.hashes pendings) then + (* no duplicate hashes *) pendings else Pending_ops.add op priority pendings) Pending_ops.empty diff --git a/src/lib_shell/test/test_shell_operation.ml b/src/lib_shell/test/test_shell_operation.ml index 455451eab6eee2cd51cb3fcb8eba1c535c345648..73b50bf227d48cfdb0d3ddbce959e4c61da5481c 100644 --- a/src/lib_shell/test/test_shell_operation.ml +++ b/src/lib_shell/test/test_shell_operation.ml @@ -90,7 +90,7 @@ let init_full_requester_disk ?global_input () : let init_full_requester ?global_input () : Test_Requester.t = fst (init_full_requester_disk ?global_input ()) -let mk_operation n : Operation.t = +let mk_operation n = let base = "BLuxtLkkNKWgV8xTzBuGcHJRPukgk4nY" in let base_len = String.length base in let n_string = Int.to_string n in @@ -101,7 +101,10 @@ let mk_operation n : Operation.t = assert (String.length hash_string = base_len) ; let branch = Block_hash.of_string_exn hash_string in let proto = Bytes.of_string n_string in - {shell = {branch}; proto} + let op : Operation.t = {shell = {branch}; proto} in + let oph = Operation.hash op in + let shell_op = Shell_operation.Internal_for_tests.make_operation oph op () in + (oph, shell_op) (** Check that when doing a succession of inject/classify operations, * the memory table of the requester stays small. In the past, this @@ -121,9 +124,7 @@ let test_db_leak f (nb_ops : int) (_ : unit) = in let classes = Classification.create parameters in let handle i = - let op = mk_operation i in - let oph = Operation.hash op in - let op = Shell_operation.Internal_for_tests.make_operation op oph () in + let oph, op = mk_operation i in let injected = Lwt_main.run @@ Test_Requester.inject requester oph i in assert injected ; f [] op classes @@ -158,9 +159,7 @@ let test_in_mempool_leak f (nb_ops : int) (_ : unit) = in let classes = Classification.create parameters in let handle i = - let op = mk_operation i in - let oph = Operation.hash op in - let op = Shell_operation.Internal_for_tests.make_operation op oph () in + let oph, op = mk_operation i in let injected = Lwt_main.run @@ Test_Requester.inject requester oph i in assert injected ; f [] op classes @@ -194,9 +193,7 @@ let test_db_do_not_clear_right_away f (nb_ops : int) (_ : unit) = in let classes = Classification.create parameters in let handle i = - let op = mk_operation i in - let oph = Operation.hash op in - let op = Shell_operation.Internal_for_tests.make_operation op oph () in + let oph, op = mk_operation i in Format.printf "Injecting op: %a\n" Operation_hash.pp oph ; let injected = Lwt_main.run @@ Test_Requester.inject requester oph i in assert injected ; diff --git a/src/lib_shell_services/block_services.ml b/src/lib_shell_services/block_services.ml index 9da34320ea382e927cf6257e03b980d502b2215d..df373835e6ef7761f45a667a7b9e06308a880421 100644 --- a/src/lib_shell_services/block_services.ml +++ b/src/lib_shell_services/block_services.ml @@ -1383,7 +1383,7 @@ module Make (Proto : PROTO) (Next_proto : PROTO) = struct let get_filter path = Tezos_rpc.Service.get_service ~description: - {|Get the configuration of the mempool filter. The minimal_fees are in mutez. Each field minimal_nanotez_per_xxx is a rational number given as a numerator and a denominator, e.g. "minimal_nanotez_per_gas_unit": [ "100", "1" ].|} + {|Get the configuration of the mempool's filter and bounds. Values of the form [ "21", "20" ] are rational numbers given as a numerator and a denominator, e.g. 21/20 = 1.05. The minimal_fees (in mutez), minimal_nanotez_per_gas_unit, and minimal_nanotez_per_byte are requirements that a manager operation must meet to be considered by the mempool. replace_by_fee_factor is how much better a manager operation must be to replace a previous valid operation **from the same manager** (both its fee and its fee/gas ratio must exceed the old operation's by at least this factor). max_operations and max_total_bytes are the bounds on respectively the number of valid operations in the mempool and the sum of their sizes in bytes.|} ~query:get_filter_query ~output:json Tezos_rpc.Path.(path / "filter") @@ -1391,7 +1391,7 @@ module Make (Proto : PROTO) (Next_proto : PROTO) = struct let set_filter path = Tezos_rpc.Service.post_service ~description: - {|Set the configuration of the mempool filter. **If any of the fields is absent from the input JSON, then it is set to the default value for this field (i.e. its value in the default configuration), even if it previously had a different value.** If the input JSON does not describe a valid configuration, then the configuration is left unchanged. Also return the new configuration (which may differ from the input if it had omitted fields or was invalid). You may call [./octez-client rpc get '/chains/main/mempool/filter?include_default=true'] to see an example of JSON describing a valid configuration.|} + {|Set the configuration of the mempool's filter and bounds. **If any of the fields is absent from the input JSON, then it is set to the default value for this field (i.e. its value in the default configuration), even if it previously had a different value.** If the input JSON does not describe a valid configuration, then the configuration is left unchanged. This RPC also returns the new configuration of the mempool (which may differ from the input if the latter omits fields or is invalid). You may call [octez-client rpc get '/chains/main/mempool/filter?include_default=true'] to see an example of JSON describing a valid configuration. See the description of that RPC for details on each configurable value.|} ~query:Tezos_rpc.Query.empty ~input:json ~output:json diff --git a/src/lib_shell_services/validation_errors.ml b/src/lib_shell_services/validation_errors.ml index 374c1fccd3986ae31f70c797187c32039398f402..0d70f8375687bf01f42f523ef1679278e0ba6731 100644 --- a/src/lib_shell_services/validation_errors.ml +++ b/src/lib_shell_services/validation_errors.ml @@ -34,6 +34,8 @@ type error += old_hash : Operation_hash.t; new_hash : Operation_hash.t; } + | Rejected_by_full_mempool of Operation_hash.t + | Removed_from_full_mempool of Operation_hash.t type error += Too_many_operations @@ -102,6 +104,39 @@ let () = | Operation_replacement {old_hash; new_hash} -> Some (old_hash, new_hash) | _ -> None) (fun (old_hash, new_hash) -> Operation_replacement {old_hash; new_hash}) ; + (* Rejected_by_full_mempool *) + register_error_kind + `Temporary + ~id:"node.mempool.rejected_by_full_mempool" + ~title:"Operation fees are too low to be considered in full mempool" + ~description:"Operation fees are too low to be considered in full mempool" + ~pp:(fun ppf oph -> + Format.fprintf + ppf + "The mempool is full and operation %a does not have enough fees to \ + replace existing operations." + Operation_hash.pp + oph) + Data_encoding.(obj1 (req "operation_hash" Operation_hash.encoding)) + (function Rejected_by_full_mempool oph -> Some oph | _ -> None) + (fun oph -> Rejected_by_full_mempool oph) ; + (* Removed_from_full_mempool *) + register_error_kind + `Temporary + ~id:"node.mempool.removed_from_full_mempool" + ~title:"Operation removed from full mempool because its fees are too low" + ~description: + "Operation removed from full mempool because its fees are too low" + ~pp:(fun ppf oph -> + Format.fprintf + ppf + "Operation %a has been removed from a full mempool because it had the \ + lowest fee/gas ratio." + Operation_hash.pp + oph) + Data_encoding.(obj1 (req "operation_hash" Operation_hash.encoding)) + (function Removed_from_full_mempool oph -> Some oph | _ -> None) + (fun oph -> Removed_from_full_mempool oph) ; (* Too many operations *) register_error_kind `Temporary diff --git a/src/lib_shell_services/validation_errors.mli b/src/lib_shell_services/validation_errors.mli index 906477663a5100f3236fc7f8dbcb84b46145369e..7ac0343a7ae46bb77d76be244745f28bc061feec 100644 --- a/src/lib_shell_services/validation_errors.mli +++ b/src/lib_shell_services/validation_errors.mli @@ -34,6 +34,8 @@ type error += old_hash : Operation_hash.t; new_hash : Operation_hash.t; } + | Rejected_by_full_mempool of Operation_hash.t + | Removed_from_full_mempool of Operation_hash.t type error += Too_many_operations diff --git a/src/proto_016_PtMumbai/lib_plugin/mempool.ml b/src/proto_016_PtMumbai/lib_plugin/mempool.ml index bdec72bfb9389890b39273f9046c351029107f38..7391d705d8f4a641f9fe29f375af1dbe7d369645 100644 --- a/src/proto_016_PtMumbai/lib_plugin/mempool.ml +++ b/src/proto_016_PtMumbai/lib_plugin/mempool.ml @@ -28,12 +28,6 @@ open Protocol open Alpha_context -type error_classification = - [ `Branch_delayed of tztrace - | `Branch_refused of tztrace - | `Refused of tztrace - | `Outdated of tztrace ] - type nanotez = Q.t let nanotez_enc : nanotez Data_encoding.t = @@ -62,22 +56,12 @@ type config = { minimal_fees : Tez.t; minimal_nanotez_per_gas_unit : nanotez; minimal_nanotez_per_byte : nanotez; - allow_script_failure : bool; - (** If [true], this makes [post_filter_manager] unconditionally return - [`Passed_postfilter filter_state], no matter the operation's - success. *) clock_drift : Period.t option; replace_by_fee_factor : Q.t; - (** This field determines the amount of additional fees (given as a - factor of the declared fees) a manager should add to an operation - in order to (eventually) replace an existing (prechecked) one - in the mempool. Note that other criteria, such as the gas ratio, - are also taken into account to decide whether to accept the - replacement or not. *) - max_prechecked_manager_operations : int; - (** Maximal number of prechecked operations to keep. The mempool only - keeps the [max_prechecked_manager_operations] operations with the - highest fee/gas and fee/size ratios. *) + (** Factor by which the fee and fee/gas ratio of an old operation in + the mempool are both multiplied to determine the values that a new + operation must exceed in order to replace the old operation. See + the [better_fees_and_ratio] function further below. *) } let default_minimal_fees = @@ -101,12 +85,10 @@ let default_config = minimal_fees = default_minimal_fees; minimal_nanotez_per_gas_unit = default_minimal_nanotez_per_gas_unit; minimal_nanotez_per_byte = default_minimal_nanotez_per_byte; - allow_script_failure = true; clock_drift = None; replace_by_fee_factor = Q.make (Z.of_int 105) (Z.of_int 100) (* Default value of [replace_by_fee_factor] is set to 5% *); - max_prechecked_manager_operations = 5_000; } let config_encoding : config Data_encoding.t = @@ -116,35 +98,27 @@ let config_encoding : config Data_encoding.t = minimal_fees; minimal_nanotez_per_gas_unit; minimal_nanotez_per_byte; - allow_script_failure; clock_drift; replace_by_fee_factor; - max_prechecked_manager_operations; } -> ( minimal_fees, minimal_nanotez_per_gas_unit, minimal_nanotez_per_byte, - allow_script_failure, clock_drift, - replace_by_fee_factor, - max_prechecked_manager_operations )) + replace_by_fee_factor )) (fun ( minimal_fees, minimal_nanotez_per_gas_unit, minimal_nanotez_per_byte, - allow_script_failure, clock_drift, - replace_by_fee_factor, - max_prechecked_manager_operations ) -> + replace_by_fee_factor ) -> { minimal_fees; minimal_nanotez_per_gas_unit; minimal_nanotez_per_byte; - allow_script_failure; clock_drift; replace_by_fee_factor; - max_prechecked_manager_operations; }) - (obj7 + (obj5 (dft "minimal_fees" Tez.encoding default_config.minimal_fees) (dft "minimal_nanotez_per_gas_unit" @@ -154,59 +128,14 @@ let config_encoding : config Data_encoding.t = "minimal_nanotez_per_byte" nanotez_enc default_config.minimal_nanotez_per_byte) - (dft "allow_script_failure" bool default_config.allow_script_failure) (opt "clock_drift" Period.encoding) (dft "replace_by_fee_factor" manager_op_replacement_factor_enc - default_config.replace_by_fee_factor) - (dft - "max_prechecked_manager_operations" - int31 - default_config.max_prechecked_manager_operations)) - -(** An Alpha_context manager operation, packed so that the type is not - parametrized by ['kind]. *) -type manager_op = Manager_op : 'kind Kind.manager operation -> manager_op - -(** Information stored for each prechecked manager operation. - - Note that this record does not include the operation hash because - it is instead used as key in the map that stores this information - in the [state] below. *) -type manager_op_info = { - manager_op : manager_op; - (** Used when we want to remove the operation with - {!Validate.remove_manager_operation}. *) - fee : Tez.t; - gas_limit : Fixed_point_repr.integral_tag Gas.Arith.t; - (** Both [fee] and [gas_limit] are used to determine whether a new - operation from the same manager should replace this one. *) - weight : Q.t; - (** Used to update [ops_prechecked] and [min_prechecked_op_weight] - in [state] when appropriate. *) -} - -type manager_op_weight = {operation_hash : Operation_hash.t; weight : Q.t} - -(** Build a {!manager_op_weight} from operation hash and {!manager_op_info}. *) -let mk_op_weight oph (info : manager_op_info) = - {operation_hash = oph; weight = info.weight} - -let compare_manager_op_weight op1 op2 = - let c = Q.compare op1.weight op2.weight in - if c <> 0 then c - else Operation_hash.compare op1.operation_hash op2.operation_hash - -module ManagerOpWeightSet = Set.Make (struct - type t = manager_op_weight - - (* Sort by weight *) - let compare = compare_manager_op_weight -end) + default_config.replace_by_fee_factor)) (** Static information to store in the filter state. *) -type state_info = { +type filter_info = { head : Block_header.shell_header; round_durations : Round.round_durations; hard_gas_limit_per_block : Gas.Arith.integral; @@ -215,36 +144,6 @@ type state_info = { grandparent_level_start : Timestamp.t; } -(** State that tracks validated manager operations. *) -type ops_state = { - prechecked_manager_op_count : int; - (** Number of prechecked manager operations. - Invariants: - - [prechecked_manager_op_count - = Operation_hash.Map.cardinal prechecked_manager_ops - = ManagerOpWeightSet.cardinal prechecked_op_weights] - - [prechecked_manager_op_count <= max_prechecked_manager_operations] *) - prechecked_manager_ops : manager_op_info Operation_hash.Map.t; - (** All prechecked manager operations. See {!manager_op_info}. *) - prechecked_op_weights : ManagerOpWeightSet.t; - (** The {!manager_op_weight} of all prechecked manager operations. *) - min_prechecked_op_weight : manager_op_weight option; - (** The prechecked operation in [op_prechecked_managers], if any, with - the minimal weight. - Invariant: - - [min_prechecked_op_weight = min { x | x \in prechecked_op_weights }] *) -} - -type state = {state_info : state_info; ops_state : ops_state} - -let empty_ops_state = - { - prechecked_manager_op_count = 0; - prechecked_manager_ops = Operation_hash.Map.empty; - prechecked_op_weights = ManagerOpWeightSet.empty; - min_prechecked_op_weight = None; - } - let init_state_prototzresult ~head round_durations hard_gas_limit_per_block = let open Lwt_result_syntax in let*? head_round = @@ -266,7 +165,7 @@ let init_state_prototzresult ~head round_durations hard_gas_limit_per_block = Period.(add proposal_level_offset proposal_round_offset) in let grandparent_level_start = Timestamp.(head.timestamp - proposal_offset) in - let state_info = + return { head; round_durations; @@ -275,8 +174,6 @@ let init_state_prototzresult ~head round_durations hard_gas_limit_per_block = round_zero_duration; grandparent_level_start; } - in - return {state_info; ops_state = empty_ops_state} let init_state ~head round_durations hard_gas_limit_per_block = Lwt.map @@ -299,7 +196,7 @@ let init context ~(head : Tezos_base.Block_header.shell_header) = let hard_gas_limit_per_block = Constants.hard_gas_limit_per_block ctxt in init_state ~head round_durations hard_gas_limit_per_block -let flush old_state ~head = +let flush old_filter_info ~head = (* To avoid the need to prepare a context as in [init], we retrieve the [round_durations] from the previous state. Indeed, they are only determined by the [minimal_block_delay] and @@ -310,8 +207,8 @@ let flush old_state ~head = constant. *) init_state ~head - old_state.state_info.round_durations - old_state.state_info.hard_gas_limit_per_block + old_filter_info.round_durations + old_filter_info.hard_gas_limit_per_block let manager_prio p = `Low p @@ -348,125 +245,6 @@ let () = (function Fees_too_low -> Some () | _ -> None) (fun () -> Fees_too_low) -type Environment.Error_monad.error += - | Manager_restriction of {oph : Operation_hash.t; fee : Tez.t} - -let () = - Environment.Error_monad.register_error_kind - `Temporary - ~id:"prefilter.manager_restriction" - ~title:"Only one manager operation per manager per block allowed" - ~description:"Only one manager operation per manager per block allowed" - ~pp:(fun ppf (oph, fee) -> - Format.fprintf - ppf - "Only one manager operation per manager per block allowed (found %a \ - with %atez fee. You may want to use --replace to provide adequate fee \ - and replace it)." - Operation_hash.pp - oph - Tez.pp - fee) - Data_encoding.( - obj2 - (req "operation_hash" Operation_hash.encoding) - (req "operation_fee" Tez.encoding)) - (function Manager_restriction {oph; fee} -> Some (oph, fee) | _ -> None) - (fun (oph, fee) -> Manager_restriction {oph; fee}) - -type Environment.Error_monad.error += - | Manager_operation_replaced of { - old_hash : Operation_hash.t; - new_hash : Operation_hash.t; - } - -let () = - Environment.Error_monad.register_error_kind - `Permanent - ~id:"plugin.manager_operation_replaced" - ~title:"Manager operation replaced" - ~description:"The manager operation has been replaced" - ~pp:(fun ppf (old_hash, new_hash) -> - Format.fprintf - ppf - "The manager operation %a has been replaced with %a" - Operation_hash.pp - old_hash - Operation_hash.pp - new_hash) - (Data_encoding.obj2 - (Data_encoding.req "old_hash" Operation_hash.encoding) - (Data_encoding.req "new_hash" Operation_hash.encoding)) - (function - | Manager_operation_replaced {old_hash; new_hash} -> - Some (old_hash, new_hash) - | _ -> None) - (fun (old_hash, new_hash) -> - Manager_operation_replaced {old_hash; new_hash}) - -type Environment.Error_monad.error += Fees_too_low_for_mempool of Tez.t - -let () = - Environment.Error_monad.register_error_kind - `Temporary - ~id:"prefilter.fees_too_low_for_mempool" - ~title:"Operation fees are too low to be considered in full mempool" - ~description:"Operation fees are too low to be considered in full mempool" - ~pp:(fun ppf required_fees -> - Format.fprintf - ppf - "The mempool is full, the number of prechecked manager operations has \ - reached the limit max_prechecked_manager_operations set by the \ - filter. Increase operation fees to at least %atz for the operation to \ - be considered and propagated by THIS node. Note that the operations \ - with the minimum fees in the mempool risk being removed if better \ - ones are received." - Tez.pp - required_fees) - Data_encoding.(obj1 (req "required_fees" Tez.encoding)) - (function - | Fees_too_low_for_mempool required_fees -> Some required_fees | _ -> None) - (fun required_fees -> Fees_too_low_for_mempool required_fees) - -type Environment.Error_monad.error += Removed_fees_too_low_for_mempool - -let () = - Environment.Error_monad.register_error_kind - `Temporary - ~id:"plugin.removed_fees_too_low_for_mempool" - ~title:"Operation removed because fees are too low for full mempool" - ~description:"Operation removed because fees are too low for full mempool" - ~pp:(fun ppf () -> - Format.fprintf - ppf - "The mempool is full, the number of prechecked manager operations has \ - reached the limit max_prechecked_manager_operations set by the \ - filter. Operation was removed because another operation with a better \ - fees/gas-size ratio was received and accepted by the mempool.") - Data_encoding.unit - (function Removed_fees_too_low_for_mempool -> Some () | _ -> None) - (fun () -> Removed_fees_too_low_for_mempool) - -(* TODO: https://gitlab.com/tezos/tezos/-/issues/2238 - Write unit tests for the feature 'replace-by-fee' and for other changes - introduced by other MRs in the plugin. *) -(* In order to decide if the new operation can replace an old one from the - same manager, we check if its fees (resp. fees/gas ratio) are greater than - (or equal to) the old operations's fees (resp. fees/gas ratio), bumped by - the factor [config.replace_by_fee_factor]. -*) -let better_fees_and_ratio = - let bump config q = Q.mul q config.replace_by_fee_factor in - fun config old_gas old_fee new_gas new_fee -> - let old_fee = Tez.to_mutez old_fee |> Z.of_int64 |> Q.of_bigint in - let old_gas = Gas.Arith.integral_to_z old_gas |> Q.of_bigint in - let new_fee = Tez.to_mutez new_fee |> Z.of_int64 |> Q.of_bigint in - let new_gas = Gas.Arith.integral_to_z new_gas |> Q.of_bigint in - let old_ratio = Q.div old_fee old_gas in - let new_ratio = Q.div new_fee new_gas in - Q.compare new_ratio (bump config old_ratio) >= 0 - && Q.compare new_fee (bump config old_fee) >= 0 - let size_of_operation op = (WithExceptions.Option.get ~loc:__LOC__ @@ Data_encoding.Binary.fixed_length @@ -491,54 +269,10 @@ let weight_and_resources_manager_operation ~hard_gas_limit_per_block ?size ~fee let resources = Q.max size_ratio gas_ratio in (Q.(fee_f / resources), resources) -(** Return fee for an operation that consumes [op_resources] for its weight to - be strictly greater than [min_weight]. *) -let required_fee_manager_operation_weight ~op_resources ~min_weight = - let req_mutez_q = Q.((min_weight * op_resources) + Q.one) in - Tez.of_mutez_exn @@ Q.to_int64 req_mutez_q - -(** Check if an operation as a weight (fees w.r.t gas and size) large enough to - be prechecked and return said weight. In the case where the prechecked - mempool is full, return an error if the weight is too small, or return the - operation to be replaced otherwise. *) -let check_minimal_weight config state ~fee ~gas_limit op = - let weight, op_resources = - weight_and_resources_manager_operation - ~hard_gas_limit_per_block:state.state_info.hard_gas_limit_per_block - ~fee - ~gas:gas_limit - op - in - if - state.ops_state.prechecked_manager_op_count - < config.max_prechecked_manager_operations - then - (* The precheck mempool is not full yet *) - `Weight_ok (`No_replace, [weight]) - else - match state.ops_state.min_prechecked_op_weight with - | None -> - (* The precheck mempool is empty *) - `Weight_ok (`No_replace, [weight]) - | Some {weight = min_weight; operation_hash = min_oph} -> - if Q.(weight > min_weight) then - (* The operation has a weight greater than the minimal - prechecked operation, replace the latest with the new one *) - `Weight_ok (`Replace min_oph, [weight]) - else - (* Otherwise fail and give indication as to what to fee should - be for the operation to be prechecked *) - let required_fee = - required_fee_manager_operation_weight ~op_resources ~min_weight - in - `Fail - (`Branch_delayed - [Environment.wrap_tzerror (Fees_too_low_for_mempool required_fee)]) - let pre_filter_manager : type t. + filter_info -> config -> - state -> Operation.packed_protocol_data -> t Kind.manager contents_list -> [ `Passed_prefilter of Q.t list @@ -546,7 +280,7 @@ let pre_filter_manager : | `Branch_delayed of tztrace | `Refused of tztrace | `Outdated of tztrace ] = - fun config filter_state packed_op op -> + fun filter_info config packed_op op -> let size = size_of_operation packed_op in let check_gas_and_fee fee gas_limit = let fees_in_nanotez = @@ -580,12 +314,15 @@ let pre_filter_manager : | Ok (fee, gas_limit) -> ( match check_gas_and_fee fee gas_limit with | `Refused _ as err -> err - | `Fees_ok -> ( - match - check_minimal_weight config filter_state ~fee ~gas_limit packed_op - with - | `Fail errs -> errs - | `Weight_ok (_, weight) -> `Passed_prefilter weight)) + | `Fees_ok -> + let weight, _op_resources = + weight_and_resources_manager_operation + ~hard_gas_limit_per_block:filter_info.hard_gas_limit_per_block + ~fee + ~gas:gas_limit + packed_op + in + `Passed_prefilter [weight]) type Environment.Error_monad.error += Wrong_operation @@ -751,23 +488,20 @@ let acceptable_op ~config ~round_durations ~round_zero_duration ~proposal_level acceptable *) acceptable ~drift ~op_earliest_ts ~now_timestamp -let pre_filter_far_future_consensus_ops config ~filter_state +let pre_filter_far_future_consensus_ops filter_info config ({level = op_level; round = op_round; _} : consensus_content) : bool Lwt.t = let res = let open Result_syntax in let now_timestamp = Time.System.now () |> Time.System.to_protocol in - let* proposal_level = - Raw_level.of_int32 filter_state.state_info.head.level - in + let* proposal_level = Raw_level.of_int32 filter_info.head.level in acceptable_op ~config - ~round_durations:filter_state.state_info.round_durations - ~round_zero_duration:filter_state.state_info.round_zero_duration + ~round_durations:filter_info.round_durations + ~round_zero_duration:filter_info.round_zero_duration ~proposal_level - ~proposal_round:filter_state.state_info.head_round - ~proposal_timestamp:filter_state.state_info.head.timestamp - ~proposal_predecessor_level_start: - filter_state.state_info.grandparent_level_start + ~proposal_round:filter_info.head_round + ~proposal_timestamp:filter_info.head.timestamp + ~proposal_predecessor_level_start:filter_info.grandparent_level_start ~op_level ~op_round ~now_timestamp @@ -783,13 +517,13 @@ let pre_filter_far_future_consensus_ops config ~filter_state We add [config.clock_drift] time as a safety margin. *) -let pre_filter config ~filter_state +let pre_filter filter_info config ({shell = _; protocol_data = Operation_data {contents; _} as op} : Main.operation) = let prefilter_manager_op manager_op = Lwt.return @@ - match pre_filter_manager config filter_state op manager_op with + match pre_filter_manager filter_info config op manager_op with | `Passed_prefilter prio -> `Passed_prefilter (manager_prio prio) | (`Branch_refused _ | `Branch_delayed _ | `Refused _ | `Outdated _) as err -> @@ -800,7 +534,7 @@ let pre_filter config ~filter_state Lwt.return (`Refused [Environment.wrap_tzerror Wrong_operation]) | Single (Preendorsement consensus_content) | Single (Endorsement consensus_content) -> - pre_filter_far_future_consensus_ops ~filter_state config consensus_content + pre_filter_far_future_consensus_ops filter_info config consensus_content >>= fun keep -> if keep then Lwt.return @@ `Passed_prefilter consensus_prio else @@ -821,171 +555,28 @@ let pre_filter config ~filter_state | Single (Manager_operation _) as op -> prefilter_manager_op op | Cons (Manager_operation _, _) as op -> prefilter_manager_op op -(** Remove a manager operation hash from the ops_state. - Do nothing if the operation was not in the state. *) -let remove_operation ops_state oph = - match Operation_hash.Map.find oph ops_state.prechecked_manager_ops with - | None -> - (* Not present in the ops_state: nothing to do. *) - ops_state - | Some info -> - let prechecked_manager_ops = - Operation_hash.Map.remove oph ops_state.prechecked_manager_ops - in - let prechecked_manager_op_count = - ops_state.prechecked_manager_op_count - 1 - in - let prechecked_op_weights = - ManagerOpWeightSet.remove - (mk_op_weight oph info) - ops_state.prechecked_op_weights - in - let min_prechecked_op_weight = - match ops_state.min_prechecked_op_weight with - | None -> None - | Some min_op_weight -> - if Operation_hash.equal min_op_weight.operation_hash oph then - ManagerOpWeightSet.min_elt prechecked_op_weights - else Some min_op_weight - in - { - prechecked_manager_op_count; - prechecked_manager_ops; - prechecked_op_weights; - min_prechecked_op_weight; - } - -(** Remove a manager operation hash from the ops_state. - Do nothing if the operation was not in the state. *) -let remove ~filter_state oph = - {filter_state with ops_state = remove_operation filter_state.ops_state oph} - -(** Add a manager operation hash and information to the filter state. - Do nothing if the operation is already present in the state. *) -let add_manager_op ops_state oph info replacement = - let ops_state = - match replacement with - | `No_replace -> ops_state - | `Replace (oph, _classification) -> remove_operation ops_state oph - in - if Operation_hash.Map.mem oph ops_state.prechecked_manager_ops then - (* Already present in the ops_state: nothing to do. *) - ops_state - else - let prechecked_manager_op_count = - ops_state.prechecked_manager_op_count + 1 - in - let prechecked_manager_ops = - Operation_hash.Map.add oph info ops_state.prechecked_manager_ops - in - let op_weight = mk_op_weight oph info in - let prechecked_op_weights = - ManagerOpWeightSet.add op_weight ops_state.prechecked_op_weights - in - let min_prechecked_op_weight = - match ops_state.min_prechecked_op_weight with - | Some old_min when compare_manager_op_weight old_min op_weight <= 0 -> - Some old_min - | _ -> Some op_weight - in - { - prechecked_manager_op_count; - prechecked_manager_ops; - prechecked_op_weights; - min_prechecked_op_weight; - } - -let add_manager_op_and_enforce_mempool_bound config filter_state oph - (op : 'manager_kind Kind.manager operation) = - let open Lwt_result_syntax in - let*? fee, gas_limit = - Result.map_error - (fun err -> `Refused (Environment.wrap_tztrace err)) - (get_manager_operation_gas_and_fee op.protocol_data.contents) - in - let* replacement, weight = - match - check_minimal_weight - config - filter_state - ~fee - ~gas_limit - (Operation_data op.protocol_data) - with - | `Weight_ok (`No_replace, weight) -> - (* The mempool is not full: no need to replace any operation. *) - return (`No_replace, weight) - | `Weight_ok (`Replace min_weight_oph, weight) -> - (* The mempool is full yet the new operation has enough weight - to be included: the old operation with the lowest weight is - reclassified as [Branch_delayed]. *) - (* TODO: https://gitlab.com/tezos/tezos/-/issues/2347 The - branch_delayed ring is bounded to 1000, so we may loose - operations. We can probably do better. *) - let replace_err = - Environment.wrap_tzerror Removed_fees_too_low_for_mempool - in - let replacement = - `Replace (min_weight_oph, `Branch_delayed [replace_err]) - in - return (replacement, weight) - | `Fail err -> - (* The mempool is full and the weight of the new operation is - too low: raise the error returned by {!check_minimal_weight}. *) - fail err - in - let weight = match weight with [x] -> x | _ -> assert false in - let info = {manager_op = Manager_op op; gas_limit; fee; weight} in - let ops_state = add_manager_op filter_state.ops_state oph info replacement in - return ({filter_state with ops_state}, replacement) - -(** If the provided operation is a manager operation, add it to the - filter_state. If the mempool is full, either return an error if the - operation does not have enough weight to be included, or return the - operation with minimal weight that gets removed to make room. - - Do nothing on non-manager operations. - - If [replace] is provided, then it is removed from [filter_state] - before processing [op]. (If [replace] is a non-manager operation, - this does nothing since it was never in [filter_state] to begin with.) - Note that when this happens, the mempool can no longer be full after - the operation has been removed, so this function always returns - [`No_replace]. - - This function is designed to be called by the shell each time a - new operation has been validated by the protocol. It will be - removed in the future once the shell is able to bound the number of - operations in the mempool by itself. *) -let add_operation_and_enforce_mempool_bound ?replace config filter_state - (oph, op) = - let filter_state = - match replace with - | Some replace_oph -> - (* If the operation to replace is not a manager operation, then - it cannot be present in the [filter_state]. But then, - [remove] does nothing anyway. *) - remove ~filter_state replace_oph - | None -> filter_state - in - let {protocol_data = Operation_data protocol_data; _} = op in - let call_manager protocol_data = - add_manager_op_and_enforce_mempool_bound - config - filter_state - oph - {shell = op.shell; protocol_data} - in - match protocol_data.contents with - | Single (Manager_operation _) -> call_manager protocol_data - | Cons (Manager_operation _, _) -> call_manager protocol_data - | Single _ -> return (filter_state, `No_replace) - let is_manager_operation op = match Operation.acceptable_pass op with | Some pass -> Compare.Int.equal pass Operation_repr.manager_pass | None -> false +(** Determine whether the new manager operation is sufficiently better + than the old manager operation to replace it. Sufficiently better + means that the new operation's fee and fee/gas ratio are both + greater than or equal to the old operation's same metrics bumped by + the factor [config.replace_by_fee_factor]. *) +let better_fees_and_ratio = + let bump config q = Q.mul q config.replace_by_fee_factor in + fun config old_gas old_fee new_gas new_fee -> + let old_fee = Tez.to_mutez old_fee |> Z.of_int64 |> Q.of_bigint in + let old_gas = Gas.Arith.integral_to_z old_gas |> Q.of_bigint in + let new_fee = Tez.to_mutez new_fee |> Z.of_int64 |> Q.of_bigint in + let new_gas = Gas.Arith.integral_to_z new_gas |> Q.of_bigint in + let old_ratio = Q.div old_fee old_gas in + let new_ratio = Q.div new_fee new_gas in + Q.compare new_ratio (bump config old_ratio) >= 0 + && Q.compare new_fee (bump config old_fee) >= 0 + (** [conflict_handler config] returns a conflict handler for {!Mempool.add_operation} (see {!Mempool.conflict_handler}). @@ -1026,3 +617,15 @@ let conflict_handler config : Mempool.conflict_handler = | Ok _ | Error _ -> `Keep else if Operation.compare existing_operation new_operation < 0 then `Replace else `Keep + +module Internal_for_tests = struct + let default_config_with_clock_drift clock_drift = + {default_config with clock_drift} + + let default_config_with_replace_factor replace_by_fee_factor = + {default_config with replace_by_fee_factor} + + let get_clock_drift {clock_drift; _} = clock_drift + + let acceptable_op = acceptable_op +end diff --git a/src/proto_016_PtMumbai/lib_plugin/mempool.mli b/src/proto_016_PtMumbai/lib_plugin/mempool.mli new file mode 100644 index 0000000000000000000000000000000000000000..7562f4aedc0527430cbe973419eda5b47b14fb9d --- /dev/null +++ b/src/proto_016_PtMumbai/lib_plugin/mempool.mli @@ -0,0 +1,150 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2018 Nomadic Development. *) +(* Copyright (c) 2021 Nomadic Labs, *) +(* Copyright (c) 2022 TriliTech *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(** Plugin for the shell mempool. It must include the signature + [FILTER.Mempool] from [lib_shell/shell_plugin.mli]. *) + +(** Settings for the {!pre_filter}: + - minimal fees to accept an operation (absolute, relative to the + gas limit, and relative to the byte size) + - clock drift for the prefiltering of consensus operations + + and for the {!conflict_handler}: + - replacement factor, that is, how much better a new manager + operation needs to be, in terms of both absolute fees and fee/gas + ratios, in order to replace an old conflicting manager operation. *) +type config + +(** Default parameters. *) +val default_config : config + +(** Encoding for {!config}. *) +val config_encoding : config Data_encoding.t + +(** Static information needed by {!pre_filter}. + + It depends on the [head] block upon which a mempool is built. *) +type filter_info + +(** Create a {!filter_info} based on the [head] block and the current + context. *) +val init : + Environment.Context.t -> + head:Block_header.shell_header -> + (filter_info, tztrace) result Lwt.t + +(** Create a new {!filter_info} based on the [head] block. + + Parts of the old {!filter_info} (which may have been built on + a different block) are recycled, so that this function is more + efficient than {!init} and does not need an + {!Environment.Context.t} argument. *) +val flush : + filter_info -> head:Block_header.shell_header -> filter_info tzresult Lwt.t + +(** Perform some preliminary checks on an operation. + + For manager operations, check that its fee, fee/gas ratio, and + fee/size ratio all meet the minimal requirements specified in the + {!config}. + + For consensus operations, check that it is possible for the + operation to have been produced before now (plus additional time + equal to the [clock_drift] from {!config}, as a safety margin). + Indeed, without this check, a baker could flood the network with + consensus operations for any future rounds or levels. The ml file + contains more detailled explanations with diagrams. *) +val pre_filter : + filter_info -> + config -> + Protocol.Alpha_context.packed_operation -> + [ `Passed_prefilter of [`High | `Medium | `Low of Q.t list] + | `Branch_delayed of tztrace + | `Branch_refused of tztrace + | `Refused of tztrace + | `Outdated of tztrace ] + Lwt.t + +(** Return a conflict handler for {!Protocol.Mempool.add_operation} + (see {!Protocol.Mempool.conflict_handler}). + + For non-manager operations, select the greater operation according + to {!Protocol.Alpha_context.Operation.compare}. + + A manager operation is replaced only when the new operation's fee + and fee/gas ratio both exceed the old operation's by at least a + factor specified in the {!config}. + + Precondition: both operations must be individually valid (to be + able to call {!Protocol.Alpha_context.Operation.compare}). *) +val conflict_handler : config -> Protocol.Mempool.conflict_handler + +(** The following type, encoding, and default values are exported for + [bin_sc_rollup_node/configuration.ml]. *) + +(** An amount of fees in nanotez. *) +type nanotez = Q.t + +(** Encoding for {!nanotez}. *) +val nanotez_enc : nanotez Data_encoding.t + +(** Minimal absolute fees in {!default_config}. *) +val default_minimal_fees : Protocol.Alpha_context.Tez.t + +(** Minimal fee over gas_limit ratio in {!default_config}. *) +val default_minimal_nanotez_per_gas_unit : nanotez + +(** Minimal fee over byte size ratio in {!default_config}. *) +val default_minimal_nanotez_per_byte : nanotez + +module Internal_for_tests : sig + open Protocol.Alpha_context + + (** {!default_config} with a custom value for the [clock_drift] field. *) + val default_config_with_clock_drift : Period.t option -> config + + (** {!default_config} with a custom [replace_by_fee_factor]. *) + val default_config_with_replace_factor : nanotez -> config + + (** Return the [clock_drift] field of the given {!config}. *) + val get_clock_drift : config -> Period.t option + + (** The main auxiliary function for {!pre_filter} regarding + consensus operations. *) + val acceptable_op : + config:config -> + round_durations:Round.round_durations -> + round_zero_duration:Period.t -> + proposal_level:Raw_level.t -> + proposal_round:Round.t -> + proposal_timestamp:Timestamp.time -> + proposal_predecessor_level_start:Timestamp.time -> + op_level:Raw_level.t -> + op_round:Round.t -> + now_timestamp:Timestamp.time -> + bool Environment.Error_monad.tzresult +end diff --git a/src/proto_016_PtMumbai/lib_plugin/test/dune b/src/proto_016_PtMumbai/lib_plugin/test/dune index 1ca373c52a4b9f001856c7de7eeb3ba40608896c..5b57bcb6dfec4392e6f5d9dfb05c998b20653130 100644 --- a/src/proto_016_PtMumbai/lib_plugin/test/dune +++ b/src/proto_016_PtMumbai/lib_plugin/test/dune @@ -1,11 +1,9 @@ ; This file was automatically generated, do not edit. ; Edit file manifest/main.ml instead. -(library - (name src_proto_016_PtMumbai_lib_plugin_test_tezt_lib) - (instrumentation (backend bisect_ppx)) +(executables + (names test_conflict_handler test_consensus_filter) (libraries - tezt.core tezos-base tezos-base-test-helpers tezos-base.unix @@ -18,11 +16,11 @@ tezos-protocol-016-PtMumbai tezos-protocol-016-PtMumbai.parameters tezos-016-PtMumbai-test-helpers) - (library_flags (:standard -linkall)) + (link_flags + (:standard) + (:include %{workspace_root}/macos-link-flags.sexp)) (flags (:standard) - -open Tezt_core - -open Tezt_core.Base -open Tezos_base.TzPervasives -open Tezos_base.TzPervasives.Error_monad.Legacy_monad_globals -open Tezos_base_test_helpers @@ -33,32 +31,14 @@ -open Tezos_protocol_016_PtMumbai -open Tezos_protocol_016_PtMumbai.Protocol -open Tezos_protocol_016_PtMumbai_parameters - -open Tezos_016_PtMumbai_test_helpers) - (modules - test_consensus_filter - test_filter_state - test_plugin - test_conflict_handler - test_utils - generators)) - -(executable - (name main) - (instrumentation (backend bisect_ppx --bisect-sigterm)) - (libraries - src_proto_016_PtMumbai_lib_plugin_test_tezt_lib - tezt) - (link_flags - (:standard) - (:include %{workspace_root}/macos-link-flags.sexp)) - (modules main)) + -open Tezos_016_PtMumbai_test_helpers)) (rule (alias runtest) (package tezos-protocol-plugin-016-PtMumbai-tests) - (enabled_if (<> false %{env:RUNTEZTALIAS=true})) - (action (run %{dep:./main.exe}))) + (action (run %{dep:./test_conflict_handler.exe}))) (rule - (targets main.ml) - (action (with-stdout-to %{targets} (echo "let () = Tezt.Test.run ()")))) + (alias runtest) + (package tezos-protocol-plugin-016-PtMumbai-tests) + (action (run %{dep:./test_consensus_filter.exe}))) diff --git a/src/proto_016_PtMumbai/lib_plugin/test/generators.ml b/src/proto_016_PtMumbai/lib_plugin/test/generators.ml deleted file mode 100644 index affc95dd5a7729d204895fe86b9b44a1e4e2ebdb..0000000000000000000000000000000000000000 --- a/src/proto_016_PtMumbai/lib_plugin/test/generators.ml +++ /dev/null @@ -1,140 +0,0 @@ -(*****************************************************************************) -(* *) -(* Open Source License *) -(* Copyright (c) 2022 Nomadic Labs, *) -(* *) -(* Permission is hereby granted, free of charge, to any person obtaining a *) -(* copy of this software and associated documentation files (the "Software"),*) -(* to deal in the Software without restriction, including without limitation *) -(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) -(* and/or sell copies of the Software, and to permit persons to whom the *) -(* Software is furnished to do so, subject to the following conditions: *) -(* *) -(* The above copyright notice and this permission notice shall be included *) -(* in all copies or substantial portions of the Software. *) -(* *) -(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) -(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) -(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) -(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) -(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) -(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) -(* DEALINGS IN THE SOFTWARE. *) -(* *) -(*****************************************************************************) - -module Mempool = Plugin.Mempool - -let string_gen = QCheck2.Gen.small_string ?gen:None - -let public_key_hash_gen : - (Tezos_crypto.Signature.public_key_hash - * Tezos_crypto.Signature.public_key - * Tezos_crypto.Signature.secret_key) - QCheck2.Gen.t = - let open QCheck2.Gen in - let+ seed = string_size (32 -- 64) in - let seed = Bytes.of_string seed in - Tezos_crypto.Signature.generate_key ~seed () - -(* TODO: https://gitlab.com/tezos/tezos/-/issues/2407 - move this function to an helper file? *) -let operation_hash_gen : Operation_hash.t QCheck2.Gen.t = - let open QCheck2.Gen in - let+ s = QCheck2.Gen.string_size (return 32) in - Operation_hash.of_string_exn s - -let dummy_manager_op_info = - let fee = Alpha_context.Tez.zero in - let gas_limit = Alpha_context.Gas.Arith.zero in - let manager_op = - let open Alpha_context in - let source = Tezos_crypto.Signature.Public_key_hash.zero in - let counter = Manager_counter.Internal_for_tests.of_int 0 in - let storage_limit = Z.zero in - let operation = Set_deposits_limit None in - let contents = - Manager_operation - {source; fee; counter; operation; gas_limit; storage_limit} - in - let contents = Single contents in - let protocol_data = - {contents; signature = Some Tezos_crypto.Signature.zero} - in - let branch = Block_hash.zero in - Mempool.Manager_op {shell = {branch}; protocol_data} - in - Mempool.{manager_op; fee; gas_limit; weight = Q.zero} - -let oph_and_info_gen = - let open QCheck2.Gen in - let+ oph = operation_hash_gen in - (oph, dummy_manager_op_info) - -let filter_state_gen : Plugin.Mempool.ops_state QCheck2.Gen.t = - let open QCheck2.Gen in - let open Plugin.Mempool in - let+ ops = small_list oph_and_info_gen in - List.fold_left - (fun state (oph, info) -> - if Operation_hash.Map.mem oph state.prechecked_manager_ops then state - else - let prechecked_manager_op_count = - state.prechecked_manager_op_count + 1 - in - let op_weight = mk_op_weight oph info in - let min_prechecked_op_weight = - match state.min_prechecked_op_weight with - | Some old_min - when Mempool.compare_manager_op_weight old_min op_weight <= 0 -> - Some old_min - | Some _ | None -> Some op_weight - in - { - prechecked_manager_op_count; - prechecked_manager_ops = - Operation_hash.Map.add oph info state.prechecked_manager_ops; - prechecked_op_weights = - ManagerOpWeightSet.add op_weight state.prechecked_op_weights; - min_prechecked_op_weight; - }) - Plugin.Mempool.empty_ops_state - ops - -(** Generate a pair of operation hash and manager_op_info, that has - even odds of belonging to the given filter_state or being fresh. *) -let with_filter_state_operation_gen : - Plugin.Mempool.ops_state -> - (Operation_hash.t * Plugin.Mempool.manager_op_info) QCheck2.Gen.t = - fun state -> - let open QCheck2.Gen in - let* use_fresh = bool in - if use_fresh || Operation_hash.Map.is_empty state.prechecked_manager_ops then - oph_and_info_gen - else oneofl (Operation_hash.Map.bindings state.prechecked_manager_ops) - -(** Generate both a filter_state, and a pair of operation hash and - manager_op_info. The pair has even odds of belonging to the - filter_state or being fresh. *) -let filter_state_with_operation_gen : - (Plugin.Mempool.ops_state - * (Operation_hash.t * Plugin.Mempool.manager_op_info)) - QCheck2.Gen.t = - let open QCheck2.Gen in - filter_state_gen >>= fun state -> - pair (return state) (with_filter_state_operation_gen state) - -(** Generate a filter_state, and two pairs of operation hash and - manager_op_info. The pairs have indepedent, even odds of belonging - to the filter_state or being fresh. *) -let filter_state_with_two_operations_gen : - (Plugin.Mempool.ops_state - * (Operation_hash.t * Plugin.Mempool.manager_op_info) - * (Operation_hash.t * Plugin.Mempool.manager_op_info)) - QCheck2.Gen.t = - let open QCheck2.Gen in - let* filter_state = filter_state_gen in - triple - (return filter_state) - (with_filter_state_operation_gen filter_state) - (with_filter_state_operation_gen filter_state) diff --git a/src/proto_016_PtMumbai/lib_plugin/test/test_conflict_handler.ml b/src/proto_016_PtMumbai/lib_plugin/test/test_conflict_handler.ml index 07468626b1e21d292b4d7d4042bf921c1fc68ff0..dbed73715fa8f7a302702386de69df6eb83c7558 100644 --- a/src/proto_016_PtMumbai/lib_plugin/test/test_conflict_handler.ml +++ b/src/proto_016_PtMumbai/lib_plugin/test/test_conflict_handler.ml @@ -175,18 +175,17 @@ let test_manager_ops () = (* Config that requires 10% better fee and ratio to replace. *) let config10 = - { - Plugin.Mempool.default_config with - replace_by_fee_factor = Q.make (Z.of_int 11) (Z.of_int 10); - } + Plugin.Mempool.Internal_for_tests.default_config_with_replace_factor + (Q.make (Z.of_int 11) (Z.of_int 10)) in + check_conflict_handler ~__LOC__ config10 ~old ~nw:op_same `Keep ; check_conflict_handler ~__LOC__ config10 ~old ~nw:op_fee5 `Keep ; check_conflict_handler ~__LOC__ config10 ~old ~nw:op_ratio5 `Keep ; check_conflict_handler ~__LOC__ config10 ~old ~nw:op_both5 `Keep ; (* Config that replaces when the new op has at least as much fee and ratio. *) let config0 = - {Plugin.Mempool.default_config with replace_by_fee_factor = Q.one} + Plugin.Mempool.Internal_for_tests.default_config_with_replace_factor Q.one in check_conflict_handler ~__LOC__ config0 ~old ~nw:op_same `Replace ; check_conflict_handler ~__LOC__ config0 ~old ~nw:op_fee5 `Replace ; diff --git a/src/proto_016_PtMumbai/lib_plugin/test/test_consensus_filter.ml b/src/proto_016_PtMumbai/lib_plugin/test/test_consensus_filter.ml index a476f53797bb49c87bdcba169536854ddedf0d03..069fe88f28e492fa8c8ed450a671d011bd64ec4c 100644 --- a/src/proto_016_PtMumbai/lib_plugin/test/test_consensus_filter.ml +++ b/src/proto_016_PtMumbai/lib_plugin/test/test_consensus_filter.ml @@ -32,9 +32,8 @@ *) open Qcheck2_helpers -open Plugin.Mempool open Alpha_context -open Test_utils +open Plugin.Mempool.Internal_for_tests (** {2. Conversion helpers} *) @@ -48,15 +47,20 @@ module Generator = struct prefix ^ printer d ^ suffix let config = - let+ x = opt small_nat in - config x + let* drift_int_opt = opt small_nat in + let clock_drift = + Option.map + (fun drift_int -> Period.of_seconds_exn (Int64.of_int drift_int)) + drift_int_opt + in + return (default_config_with_clock_drift clock_drift) let print_config = decorate ~prefix:"clock_drift " (fun config -> Option.fold ~none:"round_0 duration" ~some:(fun drift -> Int64.to_string @@ Period.to_seconds drift) - config.clock_drift) + (get_clock_drift config)) let of_result = Result.value_f ~default:(fun _ -> assert false) @@ -282,7 +286,7 @@ let test_acceptable_current_level = ~proposal_round ~round:op_round >>? fun expected_time -> - max_ts config.clock_drift proposal_timestamp now_timestamp + max_ts (get_clock_drift config) proposal_timestamp now_timestamp >>? fun max_timestamp -> ok Timestamp.(expected_time <= max_timestamp) ) ==> no_error @@ -321,7 +325,7 @@ let test_not_acceptable_current_level = ~proposal_round ~round:op_round >>? fun expected_time -> - max_ts config.clock_drift proposal_timestamp now_timestamp + max_ts (get_clock_drift config) proposal_timestamp now_timestamp >>? fun max_timestamp -> ok Timestamp.( @@ -370,7 +374,7 @@ let test_acceptable_next_level = ~predecessor_round:Round.zero ~round:op_round >>? fun expected_time -> - max_ts config.clock_drift proposal_timestamp now_timestamp + max_ts (get_clock_drift config) proposal_timestamp now_timestamp >>? fun max_timestamp -> ok Timestamp.(expected_time <= max_timestamp) ) ==> no_error @@ -421,7 +425,7 @@ let test_not_acceptable_next_level = proposal_timestamp +? Round.round_duration round_durations proposal_round) >>? fun next_level_ts -> - max_ts config.clock_drift next_level_ts now_timestamp + max_ts (get_clock_drift config) next_level_ts now_timestamp >>? fun max_timestamp -> ok Timestamp.(expected_time > max_timestamp) ) ; no_error diff --git a/src/proto_016_PtMumbai/lib_plugin/test/test_filter_state.ml b/src/proto_016_PtMumbai/lib_plugin/test/test_filter_state.ml deleted file mode 100644 index ee39e877a54cc911b79a8a2924e6a7bfc5edda54..0000000000000000000000000000000000000000 --- a/src/proto_016_PtMumbai/lib_plugin/test/test_filter_state.ml +++ /dev/null @@ -1,194 +0,0 @@ -(*****************************************************************************) -(* *) -(* Open Source License *) -(* Copyright (c) 2022 Nomadic Labs, *) -(* *) -(* Permission is hereby granted, free of charge, to any person obtaining a *) -(* copy of this software and associated documentation files (the "Software"),*) -(* to deal in the Software without restriction, including without limitation *) -(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) -(* and/or sell copies of the Software, and to permit persons to whom the *) -(* Software is furnished to do so, subject to the following conditions: *) -(* *) -(* The above copyright notice and this permission notice shall be included *) -(* in all copies or substantial portions of the Software. *) -(* *) -(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) -(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) -(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) -(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) -(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) -(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) -(* DEALINGS IN THE SOFTWARE. *) -(* *) -(*****************************************************************************) - -(** Testing - ------- - Component: Shell (Plugin) - Invocation: dune exec src/proto_016_PtMumbai/lib_plugin/test/main.exe \ - -- --file test_filter_state.ml - Subject: Unit tests the filter state functions of the plugin -*) - -open Qcheck2_helpers -open Plugin.Mempool -open Test_utils - -let count = 1000 - -let check_filter_state_invariants filter_state = - qcheck_cond - ~pp:(fun fmt filter_state -> - Format.fprintf - fmt - "The following filter_state breaks invariants:@.%a" - pp_state - filter_state) - ~cond:(fun filter_state -> - filter_state.prechecked_manager_op_count - = Operation_hash.Map.cardinal filter_state.prechecked_manager_ops - && filter_state.prechecked_manager_op_count - = ManagerOpWeightSet.cardinal filter_state.prechecked_op_weights - && ManagerOpWeightSet.for_all - (fun {operation_hash; weight} -> - match - Operation_hash.Map.find - operation_hash - filter_state.prechecked_manager_ops - with - | None -> false - | Some info -> Q.equal weight info.weight) - filter_state.prechecked_op_weights - && eq_op_weight_opt - (ManagerOpWeightSet.min_elt filter_state.prechecked_op_weights) - filter_state.min_prechecked_op_weight) - filter_state - () - -(** Test that [add_manager_op] adds the given operation to the filter - state, and removes the replaced operation if applicable. *) -let test_add_manager_op = - let open QCheck2 in - Test.make - ~count - ~name:"Add a manager operation" - (Gen.pair Generators.filter_state_with_two_operations_gen Gen.bool) - (fun ((filter_state, (oph, op_info), (oph_to_replace, _)), should_replace) - -> - (* Both [oph] and [oph_to_replace] have even odds of being - already present in [filter_state] or fresh. *) - let replacement = - if should_replace then ( - assume (not (Operation_hash.equal oph_to_replace oph)) ; - `Replace (oph_to_replace, ())) - else `No_replace - in - let filter_state = add_manager_op filter_state oph op_info replacement in - check_filter_state_invariants filter_state - && qcheck_cond - ~pp:(fun fmt set -> - Format.fprintf - fmt - "%a was not found in prechecked_manager_ops: %a" - Operation_hash.pp - oph - pp_prechecked_manager_ops - set) - ~cond:(Operation_hash.Map.mem oph) - filter_state.prechecked_manager_ops - () - && - if should_replace then - qcheck_cond - ~pp:(fun fmt () -> - Format.fprintf - fmt - "%a should have been removed from prechecked_manager_ops." - Operation_hash.pp - oph_to_replace) - ~cond:(fun () -> - not - (Operation_hash.Map.mem - oph_to_replace - filter_state.prechecked_manager_ops)) - () - () - else true) - -(** Test that [remove] removes the operation from the filter state if - it was present, and that adding then removing is the same as doing - nothing but removing the replaced operation if there is one. *) -let test_remove_present = - let open QCheck2 in - Test.make - ~count - ~name:"Remove an existing operation hash" - (Gen.triple - Generators.filter_state_with_operation_gen - Generators.oph_and_info_gen - Gen.bool) - (fun ((initial_state, (oph_to_replace, _)), (oph, op_info), should_replace) - -> - (* Add a fresh operation [oph] to the state. *) - assume - (not (Operation_hash.Map.mem oph initial_state.prechecked_manager_ops)) ; - let replacement = - if should_replace then `Replace (oph_to_replace, ()) else `No_replace - in - let filter_state = add_manager_op initial_state oph op_info replacement in - (* Remove [oph] from the state, in which it was present. *) - let filter_state = remove_operation filter_state oph in - let (_ : bool) = - (* Check that the state invariants are preserved and that - [oph] has been removed. *) - check_filter_state_invariants filter_state - && qcheck_cond - ~pp:(fun fmt () -> - Format.fprintf - fmt - "%a should have been removed from prechecked_manager_ops." - Operation_hash.pp - oph) - ~cond:(fun () -> - not - (Operation_hash.Map.mem - oph - filter_state.prechecked_manager_ops)) - () - () - in - (* Check that adding a fresh operation then removing it is the - same as doing nothing except removing any replaced operation. *) - let initial_state_without_replaced_op = - if should_replace then remove_operation initial_state oph_to_replace - else initial_state - in - qcheck_eq - ~pp:pp_state - ~eq:eq_state - initial_state_without_replaced_op - filter_state) - -(** Test that [remove] leaves the filter state intact if the operation - hash is unknown. *) -let test_remove_unknown = - let open QCheck2 in - Test.make - ~count - ~name:"Remove an unknown operation hash" - (Gen.pair Generators.filter_state_gen Generators.operation_hash_gen) - (fun (initial_state, oph) -> - assume - (not (Operation_hash.Map.mem oph initial_state.prechecked_manager_ops)) ; - let filter_state = remove_operation initial_state oph in - qcheck_eq ~pp:pp_state ~eq:eq_state initial_state filter_state) - -let () = - Alcotest.run - ~__FILE__ - Protocol.name - [ - ("add_manager_op", qcheck_wrap [test_add_manager_op]); - ("remove", qcheck_wrap [test_remove_present; test_remove_unknown]); - ] diff --git a/src/proto_016_PtMumbai/lib_plugin/test/test_plugin.ml b/src/proto_016_PtMumbai/lib_plugin/test/test_plugin.ml deleted file mode 100644 index dbb55a3ec11d26971a5b2604812ff047670c17b0..0000000000000000000000000000000000000000 --- a/src/proto_016_PtMumbai/lib_plugin/test/test_plugin.ml +++ /dev/null @@ -1,84 +0,0 @@ -(*****************************************************************************) -(* *) -(* Open Source License *) -(* Copyright (c) 2022 Nomadic Labs, *) -(* *) -(* Permission is hereby granted, free of charge, to any person obtaining a *) -(* copy of this software and associated documentation files (the "Software"),*) -(* to deal in the Software without restriction, including without limitation *) -(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) -(* and/or sell copies of the Software, and to permit persons to whom the *) -(* Software is furnished to do so, subject to the following conditions: *) -(* *) -(* The above copyright notice and this permission notice shall be included *) -(* in all copies or substantial portions of the Software. *) -(* *) -(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) -(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) -(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) -(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) -(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) -(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) -(* DEALINGS IN THE SOFTWARE. *) -(* *) -(*****************************************************************************) - -(** Testing - ------- - Component: Shell (Plugin) - Invocation: dune exec src/proto_016_PtMumbai/lib_plugin/test/main.exe \ - -- --file test_plugin.ml - Subject: Unit tests the plugin -*) - -open Plugin.Mempool -open Test_utils - -let count = Some 1000 - -(** This test checks that [flush] empties the filter state. *) -let test_flush () = - let l = - QCheck2.Gen.generate - ~n:(Option.value ~default:1 count) - Generators.filter_state_with_operation_gen - in - List.iter_es - (fun (ops_state, (oph, (op_info : manager_op_info))) -> - let ops_state = add_manager_op ops_state oph op_info `No_replace in - let open Lwt_result_syntax in - let* block, _contract = Context.init1 () in - let* incremental = Incremental.begin_construction block in - let ctxt = Incremental.alpha_ctxt incremental in - let head = block.header.shell in - let round_durations = Alpha_context.Constants.round_durations ctxt in - let hard_gas_limit_per_block = - Alpha_context.Constants.hard_gas_limit_per_block ctxt - in - let* filter_state = - init_state ~head round_durations hard_gas_limit_per_block - in - let filter_state = {filter_state with ops_state} in - let* flushed_state = flush filter_state ~head in - if eq_state flushed_state.ops_state empty_ops_state then return_unit - else - failwith - "Flushed state is not empty : %a" - pp_state - flushed_state.ops_state) - l - -let () = - Alcotest_lwt.run - ~__FILE__ - Protocol.name - [ - ( "on_flush", - [ - Tztest.tztest - "[on_flush ~validation_state ...] yields an empty state " - `Quick - test_flush; - ] ); - ] - |> Lwt_main.run diff --git a/src/proto_016_PtMumbai/lib_plugin/test/test_utils.ml b/src/proto_016_PtMumbai/lib_plugin/test/test_utils.ml deleted file mode 100644 index 57402233a49e2a128e60f1bc831509bef25e12d0..0000000000000000000000000000000000000000 --- a/src/proto_016_PtMumbai/lib_plugin/test/test_utils.ml +++ /dev/null @@ -1,116 +0,0 @@ -(*****************************************************************************) -(* *) -(* Open Source License *) -(* Copyright (c) 2022 Nomadic Labs, *) -(* *) -(* Permission is hereby granted, free of charge, to any person obtaining a *) -(* copy of this software and associated documentation files (the "Software"),*) -(* to deal in the Software without restriction, including without limitation *) -(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) -(* and/or sell copies of the Software, and to permit persons to whom the *) -(* Software is furnished to do so, subject to the following conditions: *) -(* *) -(* The above copyright notice and this permission notice shall be included *) -(* in all copies or substantial portions of the Software. *) -(* *) -(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) -(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) -(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) -(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) -(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) -(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) -(* DEALINGS IN THE SOFTWARE. *) -(* *) -(*****************************************************************************) - -open Plugin.Mempool -open Alpha_context - -let config drift_opt = - { - default_config with - clock_drift = - Option.map - (fun drift -> Period.of_seconds_exn (Int64.of_int drift)) - drift_opt; - replace_by_fee_factor = Q.make (Z.of_int 105) (Z.of_int 100); - } - -let pp_prechecked_manager_ops fmt set = - Format.fprintf - fmt - "[%a]" - (Format.pp_print_list (fun ppf (oph, (op_info : manager_op_info)) -> - Format.fprintf - ppf - "(%a -> {fee: %a; gas: %a; weight: %a})" - Operation_hash.pp - oph - Alpha_context.Tez.pp - op_info.fee - Alpha_context.Gas.Arith.pp - op_info.gas_limit - Q.pp_print - op_info.weight)) - (Operation_hash.Map.bindings set) - -let pp_manager_op_weight fmt weight = - Format.fprintf - fmt - "{oph: %a; weight: %a}" - Operation_hash.pp - weight.operation_hash - Q.pp_print - weight.weight - -let pp_prechecked_op_weights fmt set = - Format.fprintf - fmt - "[%a]" - (Format.pp_print_list (fun ppf manager_op_weight -> - pp_manager_op_weight ppf manager_op_weight)) - (ManagerOpWeightSet.elements set) - -let pp_op_weight_opt fmt = function - | None -> Format.fprintf fmt "None" - | Some op_weight -> - Format.fprintf fmt "Some %a" pp_manager_op_weight op_weight - -let pp_state fmt state = - Format.fprintf - fmt - "@[state:@,\ - {@,\ - prechecked_manager_op_count: %d@,\ - prechecked_manager_ops: %a;@,\ - prechecked_op_weights: %a;@,\ - min_prechecked_op_weight: %a@,\ - }@]" - state.prechecked_manager_op_count - pp_prechecked_manager_ops - state.prechecked_manager_ops - pp_prechecked_op_weights - state.prechecked_op_weights - pp_op_weight_opt - state.min_prechecked_op_weight - -let eq_prechecked_manager_ops = - Operation_hash.Map.equal - (fun - {manager_op = _; fee = fee1; gas_limit = gas1; weight = w1} - {manager_op = _; fee = fee2; gas_limit = gas2; weight = w2} - -> Tez.equal fee1 fee2 && Gas.Arith.equal gas1 gas2 && Q.equal w1 w2) - -let eq_op_weight_opt = - Option.equal (fun op_weight1 op_weight2 -> - Operation_hash.equal op_weight1.operation_hash op_weight2.operation_hash - && Q.equal op_weight1.weight op_weight2.weight) - -(* This function needs to be updated if the filter state is extended *) -let eq_state s1 s2 = - s1.prechecked_manager_op_count = s2.prechecked_manager_op_count - && eq_prechecked_manager_ops - s1.prechecked_manager_ops - s2.prechecked_manager_ops - && ManagerOpWeightSet.equal s1.prechecked_op_weights s2.prechecked_op_weights - && eq_op_weight_opt s1.min_prechecked_op_weight s2.min_prechecked_op_weight diff --git a/src/proto_017_PtNairob/lib_plugin/mempool.ml b/src/proto_017_PtNairob/lib_plugin/mempool.ml index bdec72bfb9389890b39273f9046c351029107f38..7391d705d8f4a641f9fe29f375af1dbe7d369645 100644 --- a/src/proto_017_PtNairob/lib_plugin/mempool.ml +++ b/src/proto_017_PtNairob/lib_plugin/mempool.ml @@ -28,12 +28,6 @@ open Protocol open Alpha_context -type error_classification = - [ `Branch_delayed of tztrace - | `Branch_refused of tztrace - | `Refused of tztrace - | `Outdated of tztrace ] - type nanotez = Q.t let nanotez_enc : nanotez Data_encoding.t = @@ -62,22 +56,12 @@ type config = { minimal_fees : Tez.t; minimal_nanotez_per_gas_unit : nanotez; minimal_nanotez_per_byte : nanotez; - allow_script_failure : bool; - (** If [true], this makes [post_filter_manager] unconditionally return - [`Passed_postfilter filter_state], no matter the operation's - success. *) clock_drift : Period.t option; replace_by_fee_factor : Q.t; - (** This field determines the amount of additional fees (given as a - factor of the declared fees) a manager should add to an operation - in order to (eventually) replace an existing (prechecked) one - in the mempool. Note that other criteria, such as the gas ratio, - are also taken into account to decide whether to accept the - replacement or not. *) - max_prechecked_manager_operations : int; - (** Maximal number of prechecked operations to keep. The mempool only - keeps the [max_prechecked_manager_operations] operations with the - highest fee/gas and fee/size ratios. *) + (** Factor by which the fee and fee/gas ratio of an old operation in + the mempool are both multiplied to determine the values that a new + operation must exceed in order to replace the old operation. See + the [better_fees_and_ratio] function further below. *) } let default_minimal_fees = @@ -101,12 +85,10 @@ let default_config = minimal_fees = default_minimal_fees; minimal_nanotez_per_gas_unit = default_minimal_nanotez_per_gas_unit; minimal_nanotez_per_byte = default_minimal_nanotez_per_byte; - allow_script_failure = true; clock_drift = None; replace_by_fee_factor = Q.make (Z.of_int 105) (Z.of_int 100) (* Default value of [replace_by_fee_factor] is set to 5% *); - max_prechecked_manager_operations = 5_000; } let config_encoding : config Data_encoding.t = @@ -116,35 +98,27 @@ let config_encoding : config Data_encoding.t = minimal_fees; minimal_nanotez_per_gas_unit; minimal_nanotez_per_byte; - allow_script_failure; clock_drift; replace_by_fee_factor; - max_prechecked_manager_operations; } -> ( minimal_fees, minimal_nanotez_per_gas_unit, minimal_nanotez_per_byte, - allow_script_failure, clock_drift, - replace_by_fee_factor, - max_prechecked_manager_operations )) + replace_by_fee_factor )) (fun ( minimal_fees, minimal_nanotez_per_gas_unit, minimal_nanotez_per_byte, - allow_script_failure, clock_drift, - replace_by_fee_factor, - max_prechecked_manager_operations ) -> + replace_by_fee_factor ) -> { minimal_fees; minimal_nanotez_per_gas_unit; minimal_nanotez_per_byte; - allow_script_failure; clock_drift; replace_by_fee_factor; - max_prechecked_manager_operations; }) - (obj7 + (obj5 (dft "minimal_fees" Tez.encoding default_config.minimal_fees) (dft "minimal_nanotez_per_gas_unit" @@ -154,59 +128,14 @@ let config_encoding : config Data_encoding.t = "minimal_nanotez_per_byte" nanotez_enc default_config.minimal_nanotez_per_byte) - (dft "allow_script_failure" bool default_config.allow_script_failure) (opt "clock_drift" Period.encoding) (dft "replace_by_fee_factor" manager_op_replacement_factor_enc - default_config.replace_by_fee_factor) - (dft - "max_prechecked_manager_operations" - int31 - default_config.max_prechecked_manager_operations)) - -(** An Alpha_context manager operation, packed so that the type is not - parametrized by ['kind]. *) -type manager_op = Manager_op : 'kind Kind.manager operation -> manager_op - -(** Information stored for each prechecked manager operation. - - Note that this record does not include the operation hash because - it is instead used as key in the map that stores this information - in the [state] below. *) -type manager_op_info = { - manager_op : manager_op; - (** Used when we want to remove the operation with - {!Validate.remove_manager_operation}. *) - fee : Tez.t; - gas_limit : Fixed_point_repr.integral_tag Gas.Arith.t; - (** Both [fee] and [gas_limit] are used to determine whether a new - operation from the same manager should replace this one. *) - weight : Q.t; - (** Used to update [ops_prechecked] and [min_prechecked_op_weight] - in [state] when appropriate. *) -} - -type manager_op_weight = {operation_hash : Operation_hash.t; weight : Q.t} - -(** Build a {!manager_op_weight} from operation hash and {!manager_op_info}. *) -let mk_op_weight oph (info : manager_op_info) = - {operation_hash = oph; weight = info.weight} - -let compare_manager_op_weight op1 op2 = - let c = Q.compare op1.weight op2.weight in - if c <> 0 then c - else Operation_hash.compare op1.operation_hash op2.operation_hash - -module ManagerOpWeightSet = Set.Make (struct - type t = manager_op_weight - - (* Sort by weight *) - let compare = compare_manager_op_weight -end) + default_config.replace_by_fee_factor)) (** Static information to store in the filter state. *) -type state_info = { +type filter_info = { head : Block_header.shell_header; round_durations : Round.round_durations; hard_gas_limit_per_block : Gas.Arith.integral; @@ -215,36 +144,6 @@ type state_info = { grandparent_level_start : Timestamp.t; } -(** State that tracks validated manager operations. *) -type ops_state = { - prechecked_manager_op_count : int; - (** Number of prechecked manager operations. - Invariants: - - [prechecked_manager_op_count - = Operation_hash.Map.cardinal prechecked_manager_ops - = ManagerOpWeightSet.cardinal prechecked_op_weights] - - [prechecked_manager_op_count <= max_prechecked_manager_operations] *) - prechecked_manager_ops : manager_op_info Operation_hash.Map.t; - (** All prechecked manager operations. See {!manager_op_info}. *) - prechecked_op_weights : ManagerOpWeightSet.t; - (** The {!manager_op_weight} of all prechecked manager operations. *) - min_prechecked_op_weight : manager_op_weight option; - (** The prechecked operation in [op_prechecked_managers], if any, with - the minimal weight. - Invariant: - - [min_prechecked_op_weight = min { x | x \in prechecked_op_weights }] *) -} - -type state = {state_info : state_info; ops_state : ops_state} - -let empty_ops_state = - { - prechecked_manager_op_count = 0; - prechecked_manager_ops = Operation_hash.Map.empty; - prechecked_op_weights = ManagerOpWeightSet.empty; - min_prechecked_op_weight = None; - } - let init_state_prototzresult ~head round_durations hard_gas_limit_per_block = let open Lwt_result_syntax in let*? head_round = @@ -266,7 +165,7 @@ let init_state_prototzresult ~head round_durations hard_gas_limit_per_block = Period.(add proposal_level_offset proposal_round_offset) in let grandparent_level_start = Timestamp.(head.timestamp - proposal_offset) in - let state_info = + return { head; round_durations; @@ -275,8 +174,6 @@ let init_state_prototzresult ~head round_durations hard_gas_limit_per_block = round_zero_duration; grandparent_level_start; } - in - return {state_info; ops_state = empty_ops_state} let init_state ~head round_durations hard_gas_limit_per_block = Lwt.map @@ -299,7 +196,7 @@ let init context ~(head : Tezos_base.Block_header.shell_header) = let hard_gas_limit_per_block = Constants.hard_gas_limit_per_block ctxt in init_state ~head round_durations hard_gas_limit_per_block -let flush old_state ~head = +let flush old_filter_info ~head = (* To avoid the need to prepare a context as in [init], we retrieve the [round_durations] from the previous state. Indeed, they are only determined by the [minimal_block_delay] and @@ -310,8 +207,8 @@ let flush old_state ~head = constant. *) init_state ~head - old_state.state_info.round_durations - old_state.state_info.hard_gas_limit_per_block + old_filter_info.round_durations + old_filter_info.hard_gas_limit_per_block let manager_prio p = `Low p @@ -348,125 +245,6 @@ let () = (function Fees_too_low -> Some () | _ -> None) (fun () -> Fees_too_low) -type Environment.Error_monad.error += - | Manager_restriction of {oph : Operation_hash.t; fee : Tez.t} - -let () = - Environment.Error_monad.register_error_kind - `Temporary - ~id:"prefilter.manager_restriction" - ~title:"Only one manager operation per manager per block allowed" - ~description:"Only one manager operation per manager per block allowed" - ~pp:(fun ppf (oph, fee) -> - Format.fprintf - ppf - "Only one manager operation per manager per block allowed (found %a \ - with %atez fee. You may want to use --replace to provide adequate fee \ - and replace it)." - Operation_hash.pp - oph - Tez.pp - fee) - Data_encoding.( - obj2 - (req "operation_hash" Operation_hash.encoding) - (req "operation_fee" Tez.encoding)) - (function Manager_restriction {oph; fee} -> Some (oph, fee) | _ -> None) - (fun (oph, fee) -> Manager_restriction {oph; fee}) - -type Environment.Error_monad.error += - | Manager_operation_replaced of { - old_hash : Operation_hash.t; - new_hash : Operation_hash.t; - } - -let () = - Environment.Error_monad.register_error_kind - `Permanent - ~id:"plugin.manager_operation_replaced" - ~title:"Manager operation replaced" - ~description:"The manager operation has been replaced" - ~pp:(fun ppf (old_hash, new_hash) -> - Format.fprintf - ppf - "The manager operation %a has been replaced with %a" - Operation_hash.pp - old_hash - Operation_hash.pp - new_hash) - (Data_encoding.obj2 - (Data_encoding.req "old_hash" Operation_hash.encoding) - (Data_encoding.req "new_hash" Operation_hash.encoding)) - (function - | Manager_operation_replaced {old_hash; new_hash} -> - Some (old_hash, new_hash) - | _ -> None) - (fun (old_hash, new_hash) -> - Manager_operation_replaced {old_hash; new_hash}) - -type Environment.Error_monad.error += Fees_too_low_for_mempool of Tez.t - -let () = - Environment.Error_monad.register_error_kind - `Temporary - ~id:"prefilter.fees_too_low_for_mempool" - ~title:"Operation fees are too low to be considered in full mempool" - ~description:"Operation fees are too low to be considered in full mempool" - ~pp:(fun ppf required_fees -> - Format.fprintf - ppf - "The mempool is full, the number of prechecked manager operations has \ - reached the limit max_prechecked_manager_operations set by the \ - filter. Increase operation fees to at least %atz for the operation to \ - be considered and propagated by THIS node. Note that the operations \ - with the minimum fees in the mempool risk being removed if better \ - ones are received." - Tez.pp - required_fees) - Data_encoding.(obj1 (req "required_fees" Tez.encoding)) - (function - | Fees_too_low_for_mempool required_fees -> Some required_fees | _ -> None) - (fun required_fees -> Fees_too_low_for_mempool required_fees) - -type Environment.Error_monad.error += Removed_fees_too_low_for_mempool - -let () = - Environment.Error_monad.register_error_kind - `Temporary - ~id:"plugin.removed_fees_too_low_for_mempool" - ~title:"Operation removed because fees are too low for full mempool" - ~description:"Operation removed because fees are too low for full mempool" - ~pp:(fun ppf () -> - Format.fprintf - ppf - "The mempool is full, the number of prechecked manager operations has \ - reached the limit max_prechecked_manager_operations set by the \ - filter. Operation was removed because another operation with a better \ - fees/gas-size ratio was received and accepted by the mempool.") - Data_encoding.unit - (function Removed_fees_too_low_for_mempool -> Some () | _ -> None) - (fun () -> Removed_fees_too_low_for_mempool) - -(* TODO: https://gitlab.com/tezos/tezos/-/issues/2238 - Write unit tests for the feature 'replace-by-fee' and for other changes - introduced by other MRs in the plugin. *) -(* In order to decide if the new operation can replace an old one from the - same manager, we check if its fees (resp. fees/gas ratio) are greater than - (or equal to) the old operations's fees (resp. fees/gas ratio), bumped by - the factor [config.replace_by_fee_factor]. -*) -let better_fees_and_ratio = - let bump config q = Q.mul q config.replace_by_fee_factor in - fun config old_gas old_fee new_gas new_fee -> - let old_fee = Tez.to_mutez old_fee |> Z.of_int64 |> Q.of_bigint in - let old_gas = Gas.Arith.integral_to_z old_gas |> Q.of_bigint in - let new_fee = Tez.to_mutez new_fee |> Z.of_int64 |> Q.of_bigint in - let new_gas = Gas.Arith.integral_to_z new_gas |> Q.of_bigint in - let old_ratio = Q.div old_fee old_gas in - let new_ratio = Q.div new_fee new_gas in - Q.compare new_ratio (bump config old_ratio) >= 0 - && Q.compare new_fee (bump config old_fee) >= 0 - let size_of_operation op = (WithExceptions.Option.get ~loc:__LOC__ @@ Data_encoding.Binary.fixed_length @@ -491,54 +269,10 @@ let weight_and_resources_manager_operation ~hard_gas_limit_per_block ?size ~fee let resources = Q.max size_ratio gas_ratio in (Q.(fee_f / resources), resources) -(** Return fee for an operation that consumes [op_resources] for its weight to - be strictly greater than [min_weight]. *) -let required_fee_manager_operation_weight ~op_resources ~min_weight = - let req_mutez_q = Q.((min_weight * op_resources) + Q.one) in - Tez.of_mutez_exn @@ Q.to_int64 req_mutez_q - -(** Check if an operation as a weight (fees w.r.t gas and size) large enough to - be prechecked and return said weight. In the case where the prechecked - mempool is full, return an error if the weight is too small, or return the - operation to be replaced otherwise. *) -let check_minimal_weight config state ~fee ~gas_limit op = - let weight, op_resources = - weight_and_resources_manager_operation - ~hard_gas_limit_per_block:state.state_info.hard_gas_limit_per_block - ~fee - ~gas:gas_limit - op - in - if - state.ops_state.prechecked_manager_op_count - < config.max_prechecked_manager_operations - then - (* The precheck mempool is not full yet *) - `Weight_ok (`No_replace, [weight]) - else - match state.ops_state.min_prechecked_op_weight with - | None -> - (* The precheck mempool is empty *) - `Weight_ok (`No_replace, [weight]) - | Some {weight = min_weight; operation_hash = min_oph} -> - if Q.(weight > min_weight) then - (* The operation has a weight greater than the minimal - prechecked operation, replace the latest with the new one *) - `Weight_ok (`Replace min_oph, [weight]) - else - (* Otherwise fail and give indication as to what to fee should - be for the operation to be prechecked *) - let required_fee = - required_fee_manager_operation_weight ~op_resources ~min_weight - in - `Fail - (`Branch_delayed - [Environment.wrap_tzerror (Fees_too_low_for_mempool required_fee)]) - let pre_filter_manager : type t. + filter_info -> config -> - state -> Operation.packed_protocol_data -> t Kind.manager contents_list -> [ `Passed_prefilter of Q.t list @@ -546,7 +280,7 @@ let pre_filter_manager : | `Branch_delayed of tztrace | `Refused of tztrace | `Outdated of tztrace ] = - fun config filter_state packed_op op -> + fun filter_info config packed_op op -> let size = size_of_operation packed_op in let check_gas_and_fee fee gas_limit = let fees_in_nanotez = @@ -580,12 +314,15 @@ let pre_filter_manager : | Ok (fee, gas_limit) -> ( match check_gas_and_fee fee gas_limit with | `Refused _ as err -> err - | `Fees_ok -> ( - match - check_minimal_weight config filter_state ~fee ~gas_limit packed_op - with - | `Fail errs -> errs - | `Weight_ok (_, weight) -> `Passed_prefilter weight)) + | `Fees_ok -> + let weight, _op_resources = + weight_and_resources_manager_operation + ~hard_gas_limit_per_block:filter_info.hard_gas_limit_per_block + ~fee + ~gas:gas_limit + packed_op + in + `Passed_prefilter [weight]) type Environment.Error_monad.error += Wrong_operation @@ -751,23 +488,20 @@ let acceptable_op ~config ~round_durations ~round_zero_duration ~proposal_level acceptable *) acceptable ~drift ~op_earliest_ts ~now_timestamp -let pre_filter_far_future_consensus_ops config ~filter_state +let pre_filter_far_future_consensus_ops filter_info config ({level = op_level; round = op_round; _} : consensus_content) : bool Lwt.t = let res = let open Result_syntax in let now_timestamp = Time.System.now () |> Time.System.to_protocol in - let* proposal_level = - Raw_level.of_int32 filter_state.state_info.head.level - in + let* proposal_level = Raw_level.of_int32 filter_info.head.level in acceptable_op ~config - ~round_durations:filter_state.state_info.round_durations - ~round_zero_duration:filter_state.state_info.round_zero_duration + ~round_durations:filter_info.round_durations + ~round_zero_duration:filter_info.round_zero_duration ~proposal_level - ~proposal_round:filter_state.state_info.head_round - ~proposal_timestamp:filter_state.state_info.head.timestamp - ~proposal_predecessor_level_start: - filter_state.state_info.grandparent_level_start + ~proposal_round:filter_info.head_round + ~proposal_timestamp:filter_info.head.timestamp + ~proposal_predecessor_level_start:filter_info.grandparent_level_start ~op_level ~op_round ~now_timestamp @@ -783,13 +517,13 @@ let pre_filter_far_future_consensus_ops config ~filter_state We add [config.clock_drift] time as a safety margin. *) -let pre_filter config ~filter_state +let pre_filter filter_info config ({shell = _; protocol_data = Operation_data {contents; _} as op} : Main.operation) = let prefilter_manager_op manager_op = Lwt.return @@ - match pre_filter_manager config filter_state op manager_op with + match pre_filter_manager filter_info config op manager_op with | `Passed_prefilter prio -> `Passed_prefilter (manager_prio prio) | (`Branch_refused _ | `Branch_delayed _ | `Refused _ | `Outdated _) as err -> @@ -800,7 +534,7 @@ let pre_filter config ~filter_state Lwt.return (`Refused [Environment.wrap_tzerror Wrong_operation]) | Single (Preendorsement consensus_content) | Single (Endorsement consensus_content) -> - pre_filter_far_future_consensus_ops ~filter_state config consensus_content + pre_filter_far_future_consensus_ops filter_info config consensus_content >>= fun keep -> if keep then Lwt.return @@ `Passed_prefilter consensus_prio else @@ -821,171 +555,28 @@ let pre_filter config ~filter_state | Single (Manager_operation _) as op -> prefilter_manager_op op | Cons (Manager_operation _, _) as op -> prefilter_manager_op op -(** Remove a manager operation hash from the ops_state. - Do nothing if the operation was not in the state. *) -let remove_operation ops_state oph = - match Operation_hash.Map.find oph ops_state.prechecked_manager_ops with - | None -> - (* Not present in the ops_state: nothing to do. *) - ops_state - | Some info -> - let prechecked_manager_ops = - Operation_hash.Map.remove oph ops_state.prechecked_manager_ops - in - let prechecked_manager_op_count = - ops_state.prechecked_manager_op_count - 1 - in - let prechecked_op_weights = - ManagerOpWeightSet.remove - (mk_op_weight oph info) - ops_state.prechecked_op_weights - in - let min_prechecked_op_weight = - match ops_state.min_prechecked_op_weight with - | None -> None - | Some min_op_weight -> - if Operation_hash.equal min_op_weight.operation_hash oph then - ManagerOpWeightSet.min_elt prechecked_op_weights - else Some min_op_weight - in - { - prechecked_manager_op_count; - prechecked_manager_ops; - prechecked_op_weights; - min_prechecked_op_weight; - } - -(** Remove a manager operation hash from the ops_state. - Do nothing if the operation was not in the state. *) -let remove ~filter_state oph = - {filter_state with ops_state = remove_operation filter_state.ops_state oph} - -(** Add a manager operation hash and information to the filter state. - Do nothing if the operation is already present in the state. *) -let add_manager_op ops_state oph info replacement = - let ops_state = - match replacement with - | `No_replace -> ops_state - | `Replace (oph, _classification) -> remove_operation ops_state oph - in - if Operation_hash.Map.mem oph ops_state.prechecked_manager_ops then - (* Already present in the ops_state: nothing to do. *) - ops_state - else - let prechecked_manager_op_count = - ops_state.prechecked_manager_op_count + 1 - in - let prechecked_manager_ops = - Operation_hash.Map.add oph info ops_state.prechecked_manager_ops - in - let op_weight = mk_op_weight oph info in - let prechecked_op_weights = - ManagerOpWeightSet.add op_weight ops_state.prechecked_op_weights - in - let min_prechecked_op_weight = - match ops_state.min_prechecked_op_weight with - | Some old_min when compare_manager_op_weight old_min op_weight <= 0 -> - Some old_min - | _ -> Some op_weight - in - { - prechecked_manager_op_count; - prechecked_manager_ops; - prechecked_op_weights; - min_prechecked_op_weight; - } - -let add_manager_op_and_enforce_mempool_bound config filter_state oph - (op : 'manager_kind Kind.manager operation) = - let open Lwt_result_syntax in - let*? fee, gas_limit = - Result.map_error - (fun err -> `Refused (Environment.wrap_tztrace err)) - (get_manager_operation_gas_and_fee op.protocol_data.contents) - in - let* replacement, weight = - match - check_minimal_weight - config - filter_state - ~fee - ~gas_limit - (Operation_data op.protocol_data) - with - | `Weight_ok (`No_replace, weight) -> - (* The mempool is not full: no need to replace any operation. *) - return (`No_replace, weight) - | `Weight_ok (`Replace min_weight_oph, weight) -> - (* The mempool is full yet the new operation has enough weight - to be included: the old operation with the lowest weight is - reclassified as [Branch_delayed]. *) - (* TODO: https://gitlab.com/tezos/tezos/-/issues/2347 The - branch_delayed ring is bounded to 1000, so we may loose - operations. We can probably do better. *) - let replace_err = - Environment.wrap_tzerror Removed_fees_too_low_for_mempool - in - let replacement = - `Replace (min_weight_oph, `Branch_delayed [replace_err]) - in - return (replacement, weight) - | `Fail err -> - (* The mempool is full and the weight of the new operation is - too low: raise the error returned by {!check_minimal_weight}. *) - fail err - in - let weight = match weight with [x] -> x | _ -> assert false in - let info = {manager_op = Manager_op op; gas_limit; fee; weight} in - let ops_state = add_manager_op filter_state.ops_state oph info replacement in - return ({filter_state with ops_state}, replacement) - -(** If the provided operation is a manager operation, add it to the - filter_state. If the mempool is full, either return an error if the - operation does not have enough weight to be included, or return the - operation with minimal weight that gets removed to make room. - - Do nothing on non-manager operations. - - If [replace] is provided, then it is removed from [filter_state] - before processing [op]. (If [replace] is a non-manager operation, - this does nothing since it was never in [filter_state] to begin with.) - Note that when this happens, the mempool can no longer be full after - the operation has been removed, so this function always returns - [`No_replace]. - - This function is designed to be called by the shell each time a - new operation has been validated by the protocol. It will be - removed in the future once the shell is able to bound the number of - operations in the mempool by itself. *) -let add_operation_and_enforce_mempool_bound ?replace config filter_state - (oph, op) = - let filter_state = - match replace with - | Some replace_oph -> - (* If the operation to replace is not a manager operation, then - it cannot be present in the [filter_state]. But then, - [remove] does nothing anyway. *) - remove ~filter_state replace_oph - | None -> filter_state - in - let {protocol_data = Operation_data protocol_data; _} = op in - let call_manager protocol_data = - add_manager_op_and_enforce_mempool_bound - config - filter_state - oph - {shell = op.shell; protocol_data} - in - match protocol_data.contents with - | Single (Manager_operation _) -> call_manager protocol_data - | Cons (Manager_operation _, _) -> call_manager protocol_data - | Single _ -> return (filter_state, `No_replace) - let is_manager_operation op = match Operation.acceptable_pass op with | Some pass -> Compare.Int.equal pass Operation_repr.manager_pass | None -> false +(** Determine whether the new manager operation is sufficiently better + than the old manager operation to replace it. Sufficiently better + means that the new operation's fee and fee/gas ratio are both + greater than or equal to the old operation's same metrics bumped by + the factor [config.replace_by_fee_factor]. *) +let better_fees_and_ratio = + let bump config q = Q.mul q config.replace_by_fee_factor in + fun config old_gas old_fee new_gas new_fee -> + let old_fee = Tez.to_mutez old_fee |> Z.of_int64 |> Q.of_bigint in + let old_gas = Gas.Arith.integral_to_z old_gas |> Q.of_bigint in + let new_fee = Tez.to_mutez new_fee |> Z.of_int64 |> Q.of_bigint in + let new_gas = Gas.Arith.integral_to_z new_gas |> Q.of_bigint in + let old_ratio = Q.div old_fee old_gas in + let new_ratio = Q.div new_fee new_gas in + Q.compare new_ratio (bump config old_ratio) >= 0 + && Q.compare new_fee (bump config old_fee) >= 0 + (** [conflict_handler config] returns a conflict handler for {!Mempool.add_operation} (see {!Mempool.conflict_handler}). @@ -1026,3 +617,15 @@ let conflict_handler config : Mempool.conflict_handler = | Ok _ | Error _ -> `Keep else if Operation.compare existing_operation new_operation < 0 then `Replace else `Keep + +module Internal_for_tests = struct + let default_config_with_clock_drift clock_drift = + {default_config with clock_drift} + + let default_config_with_replace_factor replace_by_fee_factor = + {default_config with replace_by_fee_factor} + + let get_clock_drift {clock_drift; _} = clock_drift + + let acceptable_op = acceptable_op +end diff --git a/src/proto_017_PtNairob/lib_plugin/mempool.mli b/src/proto_017_PtNairob/lib_plugin/mempool.mli new file mode 100644 index 0000000000000000000000000000000000000000..7562f4aedc0527430cbe973419eda5b47b14fb9d --- /dev/null +++ b/src/proto_017_PtNairob/lib_plugin/mempool.mli @@ -0,0 +1,150 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2018 Nomadic Development. *) +(* Copyright (c) 2021 Nomadic Labs, *) +(* Copyright (c) 2022 TriliTech *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(** Plugin for the shell mempool. It must include the signature + [FILTER.Mempool] from [lib_shell/shell_plugin.mli]. *) + +(** Settings for the {!pre_filter}: + - minimal fees to accept an operation (absolute, relative to the + gas limit, and relative to the byte size) + - clock drift for the prefiltering of consensus operations + + and for the {!conflict_handler}: + - replacement factor, that is, how much better a new manager + operation needs to be, in terms of both absolute fees and fee/gas + ratios, in order to replace an old conflicting manager operation. *) +type config + +(** Default parameters. *) +val default_config : config + +(** Encoding for {!config}. *) +val config_encoding : config Data_encoding.t + +(** Static information needed by {!pre_filter}. + + It depends on the [head] block upon which a mempool is built. *) +type filter_info + +(** Create a {!filter_info} based on the [head] block and the current + context. *) +val init : + Environment.Context.t -> + head:Block_header.shell_header -> + (filter_info, tztrace) result Lwt.t + +(** Create a new {!filter_info} based on the [head] block. + + Parts of the old {!filter_info} (which may have been built on + a different block) are recycled, so that this function is more + efficient than {!init} and does not need an + {!Environment.Context.t} argument. *) +val flush : + filter_info -> head:Block_header.shell_header -> filter_info tzresult Lwt.t + +(** Perform some preliminary checks on an operation. + + For manager operations, check that its fee, fee/gas ratio, and + fee/size ratio all meet the minimal requirements specified in the + {!config}. + + For consensus operations, check that it is possible for the + operation to have been produced before now (plus additional time + equal to the [clock_drift] from {!config}, as a safety margin). + Indeed, without this check, a baker could flood the network with + consensus operations for any future rounds or levels. The ml file + contains more detailled explanations with diagrams. *) +val pre_filter : + filter_info -> + config -> + Protocol.Alpha_context.packed_operation -> + [ `Passed_prefilter of [`High | `Medium | `Low of Q.t list] + | `Branch_delayed of tztrace + | `Branch_refused of tztrace + | `Refused of tztrace + | `Outdated of tztrace ] + Lwt.t + +(** Return a conflict handler for {!Protocol.Mempool.add_operation} + (see {!Protocol.Mempool.conflict_handler}). + + For non-manager operations, select the greater operation according + to {!Protocol.Alpha_context.Operation.compare}. + + A manager operation is replaced only when the new operation's fee + and fee/gas ratio both exceed the old operation's by at least a + factor specified in the {!config}. + + Precondition: both operations must be individually valid (to be + able to call {!Protocol.Alpha_context.Operation.compare}). *) +val conflict_handler : config -> Protocol.Mempool.conflict_handler + +(** The following type, encoding, and default values are exported for + [bin_sc_rollup_node/configuration.ml]. *) + +(** An amount of fees in nanotez. *) +type nanotez = Q.t + +(** Encoding for {!nanotez}. *) +val nanotez_enc : nanotez Data_encoding.t + +(** Minimal absolute fees in {!default_config}. *) +val default_minimal_fees : Protocol.Alpha_context.Tez.t + +(** Minimal fee over gas_limit ratio in {!default_config}. *) +val default_minimal_nanotez_per_gas_unit : nanotez + +(** Minimal fee over byte size ratio in {!default_config}. *) +val default_minimal_nanotez_per_byte : nanotez + +module Internal_for_tests : sig + open Protocol.Alpha_context + + (** {!default_config} with a custom value for the [clock_drift] field. *) + val default_config_with_clock_drift : Period.t option -> config + + (** {!default_config} with a custom [replace_by_fee_factor]. *) + val default_config_with_replace_factor : nanotez -> config + + (** Return the [clock_drift] field of the given {!config}. *) + val get_clock_drift : config -> Period.t option + + (** The main auxiliary function for {!pre_filter} regarding + consensus operations. *) + val acceptable_op : + config:config -> + round_durations:Round.round_durations -> + round_zero_duration:Period.t -> + proposal_level:Raw_level.t -> + proposal_round:Round.t -> + proposal_timestamp:Timestamp.time -> + proposal_predecessor_level_start:Timestamp.time -> + op_level:Raw_level.t -> + op_round:Round.t -> + now_timestamp:Timestamp.time -> + bool Environment.Error_monad.tzresult +end diff --git a/src/proto_017_PtNairob/lib_plugin/test/dune b/src/proto_017_PtNairob/lib_plugin/test/dune index 53ad3cdfaa879e85ac5a531110a23354f24b7a34..1b6756e593131ff72f05379102b55598c5667138 100644 --- a/src/proto_017_PtNairob/lib_plugin/test/dune +++ b/src/proto_017_PtNairob/lib_plugin/test/dune @@ -1,11 +1,9 @@ ; This file was automatically generated, do not edit. ; Edit file manifest/main.ml instead. -(library - (name src_proto_017_PtNairob_lib_plugin_test_tezt_lib) - (instrumentation (backend bisect_ppx)) +(executables + (names test_conflict_handler test_consensus_filter) (libraries - tezt.core tezos-base tezos-base-test-helpers tezos-base.unix @@ -18,11 +16,11 @@ tezos-protocol-017-PtNairob tezos-protocol-017-PtNairob.parameters tezos-017-PtNairob-test-helpers) - (library_flags (:standard -linkall)) + (link_flags + (:standard) + (:include %{workspace_root}/macos-link-flags.sexp)) (flags (:standard) - -open Tezt_core - -open Tezt_core.Base -open Tezos_base.TzPervasives -open Tezos_base.TzPervasives.Error_monad.Legacy_monad_globals -open Tezos_base_test_helpers @@ -33,32 +31,14 @@ -open Tezos_protocol_017_PtNairob -open Tezos_protocol_017_PtNairob.Protocol -open Tezos_protocol_017_PtNairob_parameters - -open Tezos_017_PtNairob_test_helpers) - (modules - test_consensus_filter - test_filter_state - test_plugin - test_conflict_handler - test_utils - generators)) - -(executable - (name main) - (instrumentation (backend bisect_ppx --bisect-sigterm)) - (libraries - src_proto_017_PtNairob_lib_plugin_test_tezt_lib - tezt) - (link_flags - (:standard) - (:include %{workspace_root}/macos-link-flags.sexp)) - (modules main)) + -open Tezos_017_PtNairob_test_helpers)) (rule (alias runtest) (package tezos-protocol-plugin-017-PtNairob-tests) - (enabled_if (<> false %{env:RUNTEZTALIAS=true})) - (action (run %{dep:./main.exe}))) + (action (run %{dep:./test_conflict_handler.exe}))) (rule - (targets main.ml) - (action (with-stdout-to %{targets} (echo "let () = Tezt.Test.run ()")))) + (alias runtest) + (package tezos-protocol-plugin-017-PtNairob-tests) + (action (run %{dep:./test_consensus_filter.exe}))) diff --git a/src/proto_017_PtNairob/lib_plugin/test/generators.ml b/src/proto_017_PtNairob/lib_plugin/test/generators.ml deleted file mode 100644 index 2987fab6c3ca28554e002b1f78dd7259421c7e39..0000000000000000000000000000000000000000 --- a/src/proto_017_PtNairob/lib_plugin/test/generators.ml +++ /dev/null @@ -1,136 +0,0 @@ -(*****************************************************************************) -(* *) -(* Open Source License *) -(* Copyright (c) 2022 Nomadic Labs, *) -(* *) -(* Permission is hereby granted, free of charge, to any person obtaining a *) -(* copy of this software and associated documentation files (the "Software"),*) -(* to deal in the Software without restriction, including without limitation *) -(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) -(* and/or sell copies of the Software, and to permit persons to whom the *) -(* Software is furnished to do so, subject to the following conditions: *) -(* *) -(* The above copyright notice and this permission notice shall be included *) -(* in all copies or substantial portions of the Software. *) -(* *) -(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) -(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) -(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) -(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) -(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) -(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) -(* DEALINGS IN THE SOFTWARE. *) -(* *) -(*****************************************************************************) - -module Mempool = Plugin.Mempool - -let string_gen = QCheck2.Gen.small_string ?gen:None - -let public_key_hash_gen : - (Signature.public_key_hash * Signature.public_key * Signature.secret_key) - QCheck2.Gen.t = - let open QCheck2.Gen in - let+ seed = string_size (32 -- 64) in - let seed = Bytes.of_string seed in - Signature.generate_key ~seed () - -(* TODO: https://gitlab.com/tezos/tezos/-/issues/2407 - move this function to an helper file? *) -let operation_hash_gen : Operation_hash.t QCheck2.Gen.t = - let open QCheck2.Gen in - let+ s = QCheck2.Gen.string_size (return 32) in - Operation_hash.of_string_exn s - -let dummy_manager_op_info = - let fee = Alpha_context.Tez.zero in - let gas_limit = Alpha_context.Gas.Arith.zero in - let manager_op = - let open Alpha_context in - let source = Signature.Public_key_hash.zero in - let counter = Manager_counter.Internal_for_tests.of_int 0 in - let storage_limit = Z.zero in - let operation = Set_deposits_limit None in - let contents = - Manager_operation - {source; fee; counter; operation; gas_limit; storage_limit} - in - let contents = Single contents in - let protocol_data = {contents; signature = Some Signature.zero} in - let branch = Block_hash.zero in - Mempool.Manager_op {shell = {branch}; protocol_data} - in - Mempool.{manager_op; fee; gas_limit; weight = Q.zero} - -let oph_and_info_gen = - let open QCheck2.Gen in - let+ oph = operation_hash_gen in - (oph, dummy_manager_op_info) - -let filter_state_gen : Plugin.Mempool.ops_state QCheck2.Gen.t = - let open QCheck2.Gen in - let open Plugin.Mempool in - let+ ops = small_list oph_and_info_gen in - List.fold_left - (fun state (oph, info) -> - if Operation_hash.Map.mem oph state.prechecked_manager_ops then state - else - let prechecked_manager_op_count = - state.prechecked_manager_op_count + 1 - in - let op_weight = mk_op_weight oph info in - let min_prechecked_op_weight = - match state.min_prechecked_op_weight with - | Some old_min - when Mempool.compare_manager_op_weight old_min op_weight <= 0 -> - Some old_min - | Some _ | None -> Some op_weight - in - { - prechecked_manager_op_count; - prechecked_manager_ops = - Operation_hash.Map.add oph info state.prechecked_manager_ops; - prechecked_op_weights = - ManagerOpWeightSet.add op_weight state.prechecked_op_weights; - min_prechecked_op_weight; - }) - Plugin.Mempool.empty_ops_state - ops - -(** Generate a pair of operation hash and manager_op_info, that has - even odds of belonging to the given filter_state or being fresh. *) -let with_filter_state_operation_gen : - Plugin.Mempool.ops_state -> - (Operation_hash.t * Plugin.Mempool.manager_op_info) QCheck2.Gen.t = - fun state -> - let open QCheck2.Gen in - let* use_fresh = bool in - if use_fresh || Operation_hash.Map.is_empty state.prechecked_manager_ops then - oph_and_info_gen - else oneofl (Operation_hash.Map.bindings state.prechecked_manager_ops) - -(** Generate both a filter_state, and a pair of operation hash and - manager_op_info. The pair has even odds of belonging to the - filter_state or being fresh. *) -let filter_state_with_operation_gen : - (Plugin.Mempool.ops_state - * (Operation_hash.t * Plugin.Mempool.manager_op_info)) - QCheck2.Gen.t = - let open QCheck2.Gen in - filter_state_gen >>= fun state -> - pair (return state) (with_filter_state_operation_gen state) - -(** Generate a filter_state, and two pairs of operation hash and - manager_op_info. The pairs have indepedent, even odds of belonging - to the filter_state or being fresh. *) -let filter_state_with_two_operations_gen : - (Plugin.Mempool.ops_state - * (Operation_hash.t * Plugin.Mempool.manager_op_info) - * (Operation_hash.t * Plugin.Mempool.manager_op_info)) - QCheck2.Gen.t = - let open QCheck2.Gen in - let* filter_state = filter_state_gen in - triple - (return filter_state) - (with_filter_state_operation_gen filter_state) - (with_filter_state_operation_gen filter_state) diff --git a/src/proto_017_PtNairob/lib_plugin/test/test_conflict_handler.ml b/src/proto_017_PtNairob/lib_plugin/test/test_conflict_handler.ml index 543acb81727d318588d18470ebcadf1267d2d841..d53b599ea22647f77d43584e101790b9076705a8 100644 --- a/src/proto_017_PtNairob/lib_plugin/test/test_conflict_handler.ml +++ b/src/proto_017_PtNairob/lib_plugin/test/test_conflict_handler.ml @@ -175,18 +175,17 @@ let test_manager_ops () = (* Config that requires 10% better fee and ratio to replace. *) let config10 = - { - Plugin.Mempool.default_config with - replace_by_fee_factor = Q.make (Z.of_int 11) (Z.of_int 10); - } + Plugin.Mempool.Internal_for_tests.default_config_with_replace_factor + (Q.make (Z.of_int 11) (Z.of_int 10)) in + check_conflict_handler ~__LOC__ config10 ~old ~nw:op_same `Keep ; check_conflict_handler ~__LOC__ config10 ~old ~nw:op_fee5 `Keep ; check_conflict_handler ~__LOC__ config10 ~old ~nw:op_ratio5 `Keep ; check_conflict_handler ~__LOC__ config10 ~old ~nw:op_both5 `Keep ; (* Config that replaces when the new op has at least as much fee and ratio. *) let config0 = - {Plugin.Mempool.default_config with replace_by_fee_factor = Q.one} + Plugin.Mempool.Internal_for_tests.default_config_with_replace_factor Q.one in check_conflict_handler ~__LOC__ config0 ~old ~nw:op_same `Replace ; check_conflict_handler ~__LOC__ config0 ~old ~nw:op_fee5 `Replace ; diff --git a/src/proto_017_PtNairob/lib_plugin/test/test_consensus_filter.ml b/src/proto_017_PtNairob/lib_plugin/test/test_consensus_filter.ml index 0fdecad309a40399475019f4e543cfeadf2d5ed1..ae09c8356c23df30fb6d9f40e4e2affd4740a691 100644 --- a/src/proto_017_PtNairob/lib_plugin/test/test_consensus_filter.ml +++ b/src/proto_017_PtNairob/lib_plugin/test/test_consensus_filter.ml @@ -32,9 +32,8 @@ *) open Qcheck2_helpers -open Plugin.Mempool open Alpha_context -open Test_utils +open Plugin.Mempool.Internal_for_tests (** {2. Conversion helpers} *) @@ -48,15 +47,20 @@ module Generator = struct prefix ^ printer d ^ suffix let config = - let+ x = opt small_nat in - config x + let* drift_int_opt = opt small_nat in + let clock_drift = + Option.map + (fun drift_int -> Period.of_seconds_exn (Int64.of_int drift_int)) + drift_int_opt + in + return (default_config_with_clock_drift clock_drift) let print_config = decorate ~prefix:"clock_drift " (fun config -> Option.fold ~none:"round_0 duration" ~some:(fun drift -> Int64.to_string @@ Period.to_seconds drift) - config.clock_drift) + (get_clock_drift config)) let of_result = Result.value_f ~default:(fun _ -> assert false) @@ -282,7 +286,7 @@ let test_acceptable_current_level = ~proposal_round ~round:op_round >>? fun expected_time -> - max_ts config.clock_drift proposal_timestamp now_timestamp + max_ts (get_clock_drift config) proposal_timestamp now_timestamp >>? fun max_timestamp -> ok Timestamp.(expected_time <= max_timestamp) ) ==> no_error @@ -321,7 +325,7 @@ let test_not_acceptable_current_level = ~proposal_round ~round:op_round >>? fun expected_time -> - max_ts config.clock_drift proposal_timestamp now_timestamp + max_ts (get_clock_drift config) proposal_timestamp now_timestamp >>? fun max_timestamp -> ok Timestamp.( @@ -370,7 +374,7 @@ let test_acceptable_next_level = ~predecessor_round:Round.zero ~round:op_round >>? fun expected_time -> - max_ts config.clock_drift proposal_timestamp now_timestamp + max_ts (get_clock_drift config) proposal_timestamp now_timestamp >>? fun max_timestamp -> ok Timestamp.(expected_time <= max_timestamp) ) ==> no_error @@ -421,7 +425,7 @@ let test_not_acceptable_next_level = proposal_timestamp +? Round.round_duration round_durations proposal_round) >>? fun next_level_ts -> - max_ts config.clock_drift next_level_ts now_timestamp + max_ts (get_clock_drift config) next_level_ts now_timestamp >>? fun max_timestamp -> ok Timestamp.(expected_time > max_timestamp) ) ; no_error diff --git a/src/proto_017_PtNairob/lib_plugin/test/test_filter_state.ml b/src/proto_017_PtNairob/lib_plugin/test/test_filter_state.ml deleted file mode 100644 index 1e0b20ee0e8567f7fa4f8d119334d7b0274a9bc4..0000000000000000000000000000000000000000 --- a/src/proto_017_PtNairob/lib_plugin/test/test_filter_state.ml +++ /dev/null @@ -1,194 +0,0 @@ -(*****************************************************************************) -(* *) -(* Open Source License *) -(* Copyright (c) 2022 Nomadic Labs, *) -(* *) -(* Permission is hereby granted, free of charge, to any person obtaining a *) -(* copy of this software and associated documentation files (the "Software"),*) -(* to deal in the Software without restriction, including without limitation *) -(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) -(* and/or sell copies of the Software, and to permit persons to whom the *) -(* Software is furnished to do so, subject to the following conditions: *) -(* *) -(* The above copyright notice and this permission notice shall be included *) -(* in all copies or substantial portions of the Software. *) -(* *) -(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) -(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) -(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) -(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) -(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) -(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) -(* DEALINGS IN THE SOFTWARE. *) -(* *) -(*****************************************************************************) - -(** Testing - ------- - Component: Shell (Plugin) - Invocation: dune exec src/proto_017_PtNairob/lib_plugin/test/main.exe \ - -- --file test_filter_state.ml - Subject: Unit tests the filter state functions of the plugin -*) - -open Qcheck2_helpers -open Plugin.Mempool -open Test_utils - -let count = 1000 - -let check_filter_state_invariants filter_state = - qcheck_cond - ~pp:(fun fmt filter_state -> - Format.fprintf - fmt - "The following filter_state breaks invariants:@.%a" - pp_state - filter_state) - ~cond:(fun filter_state -> - filter_state.prechecked_manager_op_count - = Operation_hash.Map.cardinal filter_state.prechecked_manager_ops - && filter_state.prechecked_manager_op_count - = ManagerOpWeightSet.cardinal filter_state.prechecked_op_weights - && ManagerOpWeightSet.for_all - (fun {operation_hash; weight} -> - match - Operation_hash.Map.find - operation_hash - filter_state.prechecked_manager_ops - with - | None -> false - | Some info -> Q.equal weight info.weight) - filter_state.prechecked_op_weights - && eq_op_weight_opt - (ManagerOpWeightSet.min_elt filter_state.prechecked_op_weights) - filter_state.min_prechecked_op_weight) - filter_state - () - -(** Test that [add_manager_op] adds the given operation to the filter - state, and removes the replaced operation if applicable. *) -let test_add_manager_op = - let open QCheck2 in - Test.make - ~count - ~name:"Add a manager operation" - (Gen.pair Generators.filter_state_with_two_operations_gen Gen.bool) - (fun ((filter_state, (oph, op_info), (oph_to_replace, _)), should_replace) - -> - (* Both [oph] and [oph_to_replace] have even odds of being - already present in [filter_state] or fresh. *) - let replacement = - if should_replace then ( - assume (not (Operation_hash.equal oph_to_replace oph)) ; - `Replace (oph_to_replace, ())) - else `No_replace - in - let filter_state = add_manager_op filter_state oph op_info replacement in - check_filter_state_invariants filter_state - && qcheck_cond - ~pp:(fun fmt set -> - Format.fprintf - fmt - "%a was not found in prechecked_manager_ops: %a" - Operation_hash.pp - oph - pp_prechecked_manager_ops - set) - ~cond:(Operation_hash.Map.mem oph) - filter_state.prechecked_manager_ops - () - && - if should_replace then - qcheck_cond - ~pp:(fun fmt () -> - Format.fprintf - fmt - "%a should have been removed from prechecked_manager_ops." - Operation_hash.pp - oph_to_replace) - ~cond:(fun () -> - not - (Operation_hash.Map.mem - oph_to_replace - filter_state.prechecked_manager_ops)) - () - () - else true) - -(** Test that [remove] removes the operation from the filter state if - it was present, and that adding then removing is the same as doing - nothing but removing the replaced operation if there is one. *) -let test_remove_present = - let open QCheck2 in - Test.make - ~count - ~name:"Remove an existing operation hash" - (Gen.triple - Generators.filter_state_with_operation_gen - Generators.oph_and_info_gen - Gen.bool) - (fun ((initial_state, (oph_to_replace, _)), (oph, op_info), should_replace) - -> - (* Add a fresh operation [oph] to the state. *) - assume - (not (Operation_hash.Map.mem oph initial_state.prechecked_manager_ops)) ; - let replacement = - if should_replace then `Replace (oph_to_replace, ()) else `No_replace - in - let filter_state = add_manager_op initial_state oph op_info replacement in - (* Remove [oph] from the state, in which it was present. *) - let filter_state = remove_operation filter_state oph in - let (_ : bool) = - (* Check that the state invariants are preserved and that - [oph] has been removed. *) - check_filter_state_invariants filter_state - && qcheck_cond - ~pp:(fun fmt () -> - Format.fprintf - fmt - "%a should have been removed from prechecked_manager_ops." - Operation_hash.pp - oph) - ~cond:(fun () -> - not - (Operation_hash.Map.mem - oph - filter_state.prechecked_manager_ops)) - () - () - in - (* Check that adding a fresh operation then removing it is the - same as doing nothing except removing any replaced operation. *) - let initial_state_without_replaced_op = - if should_replace then remove_operation initial_state oph_to_replace - else initial_state - in - qcheck_eq - ~pp:pp_state - ~eq:eq_state - initial_state_without_replaced_op - filter_state) - -(** Test that [remove] leaves the filter state intact if the operation - hash is unknown. *) -let test_remove_unknown = - let open QCheck2 in - Test.make - ~count - ~name:"Remove an unknown operation hash" - (Gen.pair Generators.filter_state_gen Generators.operation_hash_gen) - (fun (initial_state, oph) -> - assume - (not (Operation_hash.Map.mem oph initial_state.prechecked_manager_ops)) ; - let filter_state = remove_operation initial_state oph in - qcheck_eq ~pp:pp_state ~eq:eq_state initial_state filter_state) - -let () = - Alcotest.run - ~__FILE__ - Protocol.name - [ - ("add_manager_op", qcheck_wrap [test_add_manager_op]); - ("remove", qcheck_wrap [test_remove_present; test_remove_unknown]); - ] diff --git a/src/proto_017_PtNairob/lib_plugin/test/test_plugin.ml b/src/proto_017_PtNairob/lib_plugin/test/test_plugin.ml deleted file mode 100644 index 3da31c78ee11d8ccf225df7df0ca90d4e780f960..0000000000000000000000000000000000000000 --- a/src/proto_017_PtNairob/lib_plugin/test/test_plugin.ml +++ /dev/null @@ -1,84 +0,0 @@ -(*****************************************************************************) -(* *) -(* Open Source License *) -(* Copyright (c) 2022 Nomadic Labs, *) -(* *) -(* Permission is hereby granted, free of charge, to any person obtaining a *) -(* copy of this software and associated documentation files (the "Software"),*) -(* to deal in the Software without restriction, including without limitation *) -(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) -(* and/or sell copies of the Software, and to permit persons to whom the *) -(* Software is furnished to do so, subject to the following conditions: *) -(* *) -(* The above copyright notice and this permission notice shall be included *) -(* in all copies or substantial portions of the Software. *) -(* *) -(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) -(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) -(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) -(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) -(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) -(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) -(* DEALINGS IN THE SOFTWARE. *) -(* *) -(*****************************************************************************) - -(** Testing - ------- - Component: Shell (Plugin) - Invocation: dune exec src/proto_017_PtNairob/lib_plugin/test/main.exe \ - -- --file test_plugin.ml - Subject: Unit tests the plugin -*) - -open Plugin.Mempool -open Test_utils - -let count = Some 1000 - -(** This test checks that [flush] empties the filter state. *) -let test_flush () = - let l = - QCheck2.Gen.generate - ~n:(Option.value ~default:1 count) - Generators.filter_state_with_operation_gen - in - List.iter_es - (fun (ops_state, (oph, (op_info : manager_op_info))) -> - let ops_state = add_manager_op ops_state oph op_info `No_replace in - let open Lwt_result_syntax in - let* block, _contract = Context.init1 () in - let* incremental = Incremental.begin_construction block in - let ctxt = Incremental.alpha_ctxt incremental in - let head = block.header.shell in - let round_durations = Alpha_context.Constants.round_durations ctxt in - let hard_gas_limit_per_block = - Alpha_context.Constants.hard_gas_limit_per_block ctxt - in - let* filter_state = - init_state ~head round_durations hard_gas_limit_per_block - in - let filter_state = {filter_state with ops_state} in - let* flushed_state = flush filter_state ~head in - if eq_state flushed_state.ops_state empty_ops_state then return_unit - else - failwith - "Flushed state is not empty : %a" - pp_state - flushed_state.ops_state) - l - -let () = - Alcotest_lwt.run - ~__FILE__ - Protocol.name - [ - ( "on_flush", - [ - Tztest.tztest - "[on_flush ~validation_state ...] yields an empty state " - `Quick - test_flush; - ] ); - ] - |> Lwt_main.run diff --git a/src/proto_017_PtNairob/lib_plugin/test/test_utils.ml b/src/proto_017_PtNairob/lib_plugin/test/test_utils.ml deleted file mode 100644 index 57402233a49e2a128e60f1bc831509bef25e12d0..0000000000000000000000000000000000000000 --- a/src/proto_017_PtNairob/lib_plugin/test/test_utils.ml +++ /dev/null @@ -1,116 +0,0 @@ -(*****************************************************************************) -(* *) -(* Open Source License *) -(* Copyright (c) 2022 Nomadic Labs, *) -(* *) -(* Permission is hereby granted, free of charge, to any person obtaining a *) -(* copy of this software and associated documentation files (the "Software"),*) -(* to deal in the Software without restriction, including without limitation *) -(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) -(* and/or sell copies of the Software, and to permit persons to whom the *) -(* Software is furnished to do so, subject to the following conditions: *) -(* *) -(* The above copyright notice and this permission notice shall be included *) -(* in all copies or substantial portions of the Software. *) -(* *) -(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) -(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) -(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) -(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) -(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) -(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) -(* DEALINGS IN THE SOFTWARE. *) -(* *) -(*****************************************************************************) - -open Plugin.Mempool -open Alpha_context - -let config drift_opt = - { - default_config with - clock_drift = - Option.map - (fun drift -> Period.of_seconds_exn (Int64.of_int drift)) - drift_opt; - replace_by_fee_factor = Q.make (Z.of_int 105) (Z.of_int 100); - } - -let pp_prechecked_manager_ops fmt set = - Format.fprintf - fmt - "[%a]" - (Format.pp_print_list (fun ppf (oph, (op_info : manager_op_info)) -> - Format.fprintf - ppf - "(%a -> {fee: %a; gas: %a; weight: %a})" - Operation_hash.pp - oph - Alpha_context.Tez.pp - op_info.fee - Alpha_context.Gas.Arith.pp - op_info.gas_limit - Q.pp_print - op_info.weight)) - (Operation_hash.Map.bindings set) - -let pp_manager_op_weight fmt weight = - Format.fprintf - fmt - "{oph: %a; weight: %a}" - Operation_hash.pp - weight.operation_hash - Q.pp_print - weight.weight - -let pp_prechecked_op_weights fmt set = - Format.fprintf - fmt - "[%a]" - (Format.pp_print_list (fun ppf manager_op_weight -> - pp_manager_op_weight ppf manager_op_weight)) - (ManagerOpWeightSet.elements set) - -let pp_op_weight_opt fmt = function - | None -> Format.fprintf fmt "None" - | Some op_weight -> - Format.fprintf fmt "Some %a" pp_manager_op_weight op_weight - -let pp_state fmt state = - Format.fprintf - fmt - "@[state:@,\ - {@,\ - prechecked_manager_op_count: %d@,\ - prechecked_manager_ops: %a;@,\ - prechecked_op_weights: %a;@,\ - min_prechecked_op_weight: %a@,\ - }@]" - state.prechecked_manager_op_count - pp_prechecked_manager_ops - state.prechecked_manager_ops - pp_prechecked_op_weights - state.prechecked_op_weights - pp_op_weight_opt - state.min_prechecked_op_weight - -let eq_prechecked_manager_ops = - Operation_hash.Map.equal - (fun - {manager_op = _; fee = fee1; gas_limit = gas1; weight = w1} - {manager_op = _; fee = fee2; gas_limit = gas2; weight = w2} - -> Tez.equal fee1 fee2 && Gas.Arith.equal gas1 gas2 && Q.equal w1 w2) - -let eq_op_weight_opt = - Option.equal (fun op_weight1 op_weight2 -> - Operation_hash.equal op_weight1.operation_hash op_weight2.operation_hash - && Q.equal op_weight1.weight op_weight2.weight) - -(* This function needs to be updated if the filter state is extended *) -let eq_state s1 s2 = - s1.prechecked_manager_op_count = s2.prechecked_manager_op_count - && eq_prechecked_manager_ops - s1.prechecked_manager_ops - s2.prechecked_manager_ops - && ManagerOpWeightSet.equal s1.prechecked_op_weights s2.prechecked_op_weights - && eq_op_weight_opt s1.min_prechecked_op_weight s2.min_prechecked_op_weight diff --git a/src/proto_alpha/lib_plugin/mempool.ml b/src/proto_alpha/lib_plugin/mempool.ml index 7347f2b19750986215f75b5fedc80eae09c80fcb..f078f3d64137af0956b8149b9a01b34de5f5bf6c 100644 --- a/src/proto_alpha/lib_plugin/mempool.ml +++ b/src/proto_alpha/lib_plugin/mempool.ml @@ -28,12 +28,6 @@ open Protocol open Alpha_context -type error_classification = - [ `Branch_delayed of tztrace - | `Branch_refused of tztrace - | `Refused of tztrace - | `Outdated of tztrace ] - type nanotez = Q.t let nanotez_enc : nanotez Data_encoding.t = @@ -62,22 +56,12 @@ type config = { minimal_fees : Tez.t; minimal_nanotez_per_gas_unit : nanotez; minimal_nanotez_per_byte : nanotez; - allow_script_failure : bool; - (** If [true], this makes [post_filter_manager] unconditionally return - [`Passed_postfilter filter_state], no matter the operation's - success. *) clock_drift : Period.t option; replace_by_fee_factor : Q.t; - (** This field determines the amount of additional fees (given as a - factor of the declared fees) a manager should add to an operation - in order to (eventually) replace an existing (prechecked) one - in the mempool. Note that other criteria, such as the gas ratio, - are also taken into account to decide whether to accept the - replacement or not. *) - max_prechecked_manager_operations : int; - (** Maximal number of prechecked operations to keep. The mempool only - keeps the [max_prechecked_manager_operations] operations with the - highest fee/gas and fee/size ratios. *) + (** Factor by which the fee and fee/gas ratio of an old operation in + the mempool are both multiplied to determine the values that a new + operation must exceed in order to replace the old operation. See + the [better_fees_and_ratio] function further below. *) } let default_minimal_fees = @@ -101,12 +85,10 @@ let default_config = minimal_fees = default_minimal_fees; minimal_nanotez_per_gas_unit = default_minimal_nanotez_per_gas_unit; minimal_nanotez_per_byte = default_minimal_nanotez_per_byte; - allow_script_failure = true; clock_drift = None; replace_by_fee_factor = Q.make (Z.of_int 105) (Z.of_int 100) (* Default value of [replace_by_fee_factor] is set to 5% *); - max_prechecked_manager_operations = 5_000; } let config_encoding : config Data_encoding.t = @@ -116,35 +98,27 @@ let config_encoding : config Data_encoding.t = minimal_fees; minimal_nanotez_per_gas_unit; minimal_nanotez_per_byte; - allow_script_failure; clock_drift; replace_by_fee_factor; - max_prechecked_manager_operations; } -> ( minimal_fees, minimal_nanotez_per_gas_unit, minimal_nanotez_per_byte, - allow_script_failure, clock_drift, - replace_by_fee_factor, - max_prechecked_manager_operations )) + replace_by_fee_factor )) (fun ( minimal_fees, minimal_nanotez_per_gas_unit, minimal_nanotez_per_byte, - allow_script_failure, clock_drift, - replace_by_fee_factor, - max_prechecked_manager_operations ) -> + replace_by_fee_factor ) -> { minimal_fees; minimal_nanotez_per_gas_unit; minimal_nanotez_per_byte; - allow_script_failure; clock_drift; replace_by_fee_factor; - max_prechecked_manager_operations; }) - (obj7 + (obj5 (dft "minimal_fees" Tez.encoding default_config.minimal_fees) (dft "minimal_nanotez_per_gas_unit" @@ -154,59 +128,14 @@ let config_encoding : config Data_encoding.t = "minimal_nanotez_per_byte" nanotez_enc default_config.minimal_nanotez_per_byte) - (dft "allow_script_failure" bool default_config.allow_script_failure) (opt "clock_drift" Period.encoding) (dft "replace_by_fee_factor" manager_op_replacement_factor_enc - default_config.replace_by_fee_factor) - (dft - "max_prechecked_manager_operations" - int31 - default_config.max_prechecked_manager_operations)) - -(** An Alpha_context manager operation, packed so that the type is not - parametrized by ['kind]. *) -type manager_op = Manager_op : 'kind Kind.manager operation -> manager_op - -(** Information stored for each prechecked manager operation. - - Note that this record does not include the operation hash because - it is instead used as key in the map that stores this information - in the [state] below. *) -type manager_op_info = { - manager_op : manager_op; - (** Used when we want to remove the operation with - {!Validate.remove_manager_operation}. *) - fee : Tez.t; - gas_limit : Fixed_point_repr.integral_tag Gas.Arith.t; - (** Both [fee] and [gas_limit] are used to determine whether a new - operation from the same manager should replace this one. *) - weight : Q.t; - (** Used to update [ops_prechecked] and [min_prechecked_op_weight] - in [state] when appropriate. *) -} - -type manager_op_weight = {operation_hash : Operation_hash.t; weight : Q.t} - -(** Build a {!manager_op_weight} from operation hash and {!manager_op_info}. *) -let mk_op_weight oph (info : manager_op_info) = - {operation_hash = oph; weight = info.weight} - -let compare_manager_op_weight op1 op2 = - let c = Q.compare op1.weight op2.weight in - if c <> 0 then c - else Operation_hash.compare op1.operation_hash op2.operation_hash - -module ManagerOpWeightSet = Set.Make (struct - type t = manager_op_weight - - (* Sort by weight *) - let compare = compare_manager_op_weight -end) + default_config.replace_by_fee_factor)) (** Static information to store in the filter state. *) -type state_info = { +type filter_info = { head : Block_header.shell_header; round_durations : Round.round_durations; hard_gas_limit_per_block : Gas.Arith.integral; @@ -215,36 +144,6 @@ type state_info = { grandparent_level_start : Timestamp.t; } -(** State that tracks validated manager operations. *) -type ops_state = { - prechecked_manager_op_count : int; - (** Number of prechecked manager operations. - Invariants: - - [prechecked_manager_op_count - = Operation_hash.Map.cardinal prechecked_manager_ops - = ManagerOpWeightSet.cardinal prechecked_op_weights] - - [prechecked_manager_op_count <= max_prechecked_manager_operations] *) - prechecked_manager_ops : manager_op_info Operation_hash.Map.t; - (** All prechecked manager operations. See {!manager_op_info}. *) - prechecked_op_weights : ManagerOpWeightSet.t; - (** The {!manager_op_weight} of all prechecked manager operations. *) - min_prechecked_op_weight : manager_op_weight option; - (** The prechecked operation in [op_prechecked_managers], if any, with - the minimal weight. - Invariant: - - [min_prechecked_op_weight = min { x | x \in prechecked_op_weights }] *) -} - -type state = {state_info : state_info; ops_state : ops_state} - -let empty_ops_state = - { - prechecked_manager_op_count = 0; - prechecked_manager_ops = Operation_hash.Map.empty; - prechecked_op_weights = ManagerOpWeightSet.empty; - min_prechecked_op_weight = None; - } - let init_state_prototzresult ~head round_durations hard_gas_limit_per_block = let open Lwt_result_syntax in let*? head_round = @@ -266,7 +165,7 @@ let init_state_prototzresult ~head round_durations hard_gas_limit_per_block = Period.(add proposal_level_offset proposal_round_offset) in let grandparent_level_start = Timestamp.(head.timestamp - proposal_offset) in - let state_info = + return { head; round_durations; @@ -275,8 +174,6 @@ let init_state_prototzresult ~head round_durations hard_gas_limit_per_block = round_zero_duration; grandparent_level_start; } - in - return {state_info; ops_state = empty_ops_state} let init_state ~head round_durations hard_gas_limit_per_block = Lwt.map @@ -299,7 +196,7 @@ let init context ~(head : Tezos_base.Block_header.shell_header) = let hard_gas_limit_per_block = Constants.hard_gas_limit_per_block ctxt in init_state ~head round_durations hard_gas_limit_per_block -let flush old_state ~head = +let flush old_filter_info ~head = (* To avoid the need to prepare a context as in [init], we retrieve the [round_durations] from the previous state. Indeed, they are only determined by the [minimal_block_delay] and @@ -310,8 +207,8 @@ let flush old_state ~head = constant. *) init_state ~head - old_state.state_info.round_durations - old_state.state_info.hard_gas_limit_per_block + old_filter_info.round_durations + old_filter_info.hard_gas_limit_per_block let manager_prio p = `Low p @@ -348,125 +245,6 @@ let () = (function Fees_too_low -> Some () | _ -> None) (fun () -> Fees_too_low) -type Environment.Error_monad.error += - | Manager_restriction of {oph : Operation_hash.t; fee : Tez.t} - -let () = - Environment.Error_monad.register_error_kind - `Temporary - ~id:"prefilter.manager_restriction" - ~title:"Only one manager operation per manager per block allowed" - ~description:"Only one manager operation per manager per block allowed" - ~pp:(fun ppf (oph, fee) -> - Format.fprintf - ppf - "Only one manager operation per manager per block allowed (found %a \ - with %atez fee. You may want to use --replace to provide adequate fee \ - and replace it)." - Operation_hash.pp - oph - Tez.pp - fee) - Data_encoding.( - obj2 - (req "operation_hash" Operation_hash.encoding) - (req "operation_fee" Tez.encoding)) - (function Manager_restriction {oph; fee} -> Some (oph, fee) | _ -> None) - (fun (oph, fee) -> Manager_restriction {oph; fee}) - -type Environment.Error_monad.error += - | Manager_operation_replaced of { - old_hash : Operation_hash.t; - new_hash : Operation_hash.t; - } - -let () = - Environment.Error_monad.register_error_kind - `Permanent - ~id:"plugin.manager_operation_replaced" - ~title:"Manager operation replaced" - ~description:"The manager operation has been replaced" - ~pp:(fun ppf (old_hash, new_hash) -> - Format.fprintf - ppf - "The manager operation %a has been replaced with %a" - Operation_hash.pp - old_hash - Operation_hash.pp - new_hash) - (Data_encoding.obj2 - (Data_encoding.req "old_hash" Operation_hash.encoding) - (Data_encoding.req "new_hash" Operation_hash.encoding)) - (function - | Manager_operation_replaced {old_hash; new_hash} -> - Some (old_hash, new_hash) - | _ -> None) - (fun (old_hash, new_hash) -> - Manager_operation_replaced {old_hash; new_hash}) - -type Environment.Error_monad.error += Fees_too_low_for_mempool of Tez.t - -let () = - Environment.Error_monad.register_error_kind - `Temporary - ~id:"prefilter.fees_too_low_for_mempool" - ~title:"Operation fees are too low to be considered in full mempool" - ~description:"Operation fees are too low to be considered in full mempool" - ~pp:(fun ppf required_fees -> - Format.fprintf - ppf - "The mempool is full, the number of prechecked manager operations has \ - reached the limit max_prechecked_manager_operations set by the \ - filter. Increase operation fees to at least %atz for the operation to \ - be considered and propagated by THIS node. Note that the operations \ - with the minimum fees in the mempool risk being removed if better \ - ones are received." - Tez.pp - required_fees) - Data_encoding.(obj1 (req "required_fees" Tez.encoding)) - (function - | Fees_too_low_for_mempool required_fees -> Some required_fees | _ -> None) - (fun required_fees -> Fees_too_low_for_mempool required_fees) - -type Environment.Error_monad.error += Removed_fees_too_low_for_mempool - -let () = - Environment.Error_monad.register_error_kind - `Temporary - ~id:"plugin.removed_fees_too_low_for_mempool" - ~title:"Operation removed because fees are too low for full mempool" - ~description:"Operation removed because fees are too low for full mempool" - ~pp:(fun ppf () -> - Format.fprintf - ppf - "The mempool is full, the number of prechecked manager operations has \ - reached the limit max_prechecked_manager_operations set by the \ - filter. Operation was removed because another operation with a better \ - fees/gas-size ratio was received and accepted by the mempool.") - Data_encoding.unit - (function Removed_fees_too_low_for_mempool -> Some () | _ -> None) - (fun () -> Removed_fees_too_low_for_mempool) - -(* TODO: https://gitlab.com/tezos/tezos/-/issues/2238 - Write unit tests for the feature 'replace-by-fee' and for other changes - introduced by other MRs in the plugin. *) -(* In order to decide if the new operation can replace an old one from the - same manager, we check if its fees (resp. fees/gas ratio) are greater than - (or equal to) the old operations's fees (resp. fees/gas ratio), bumped by - the factor [config.replace_by_fee_factor]. -*) -let better_fees_and_ratio = - let bump config q = Q.mul q config.replace_by_fee_factor in - fun config old_gas old_fee new_gas new_fee -> - let old_fee = Tez.to_mutez old_fee |> Z.of_int64 |> Q.of_bigint in - let old_gas = Gas.Arith.integral_to_z old_gas |> Q.of_bigint in - let new_fee = Tez.to_mutez new_fee |> Z.of_int64 |> Q.of_bigint in - let new_gas = Gas.Arith.integral_to_z new_gas |> Q.of_bigint in - let old_ratio = Q.div old_fee old_gas in - let new_ratio = Q.div new_fee new_gas in - Q.compare new_ratio (bump config old_ratio) >= 0 - && Q.compare new_fee (bump config old_fee) >= 0 - let size_of_operation op = (WithExceptions.Option.get ~loc:__LOC__ @@ Data_encoding.Binary.fixed_length @@ -493,54 +271,10 @@ let weight_and_resources_manager_operation ~hard_gas_limit_per_block ?size ~fee let resources = Q.max size_ratio gas_ratio in (Q.(fee_f / resources), resources) -(** Return fee for an operation that consumes [op_resources] for its weight to - be strictly greater than [min_weight]. *) -let required_fee_manager_operation_weight ~op_resources ~min_weight = - let req_mutez_q = Q.((min_weight * op_resources) + Q.one) in - Tez.of_mutez_exn @@ Q.to_int64 req_mutez_q - -(** Check if an operation as a weight (fees w.r.t gas and size) large enough to - be prechecked and return said weight. In the case where the prechecked - mempool is full, return an error if the weight is too small, or return the - operation to be replaced otherwise. *) -let check_minimal_weight config state ~fee ~gas_limit op = - let weight, op_resources = - weight_and_resources_manager_operation - ~hard_gas_limit_per_block:state.state_info.hard_gas_limit_per_block - ~fee - ~gas:gas_limit - op - in - if - state.ops_state.prechecked_manager_op_count - < config.max_prechecked_manager_operations - then - (* The precheck mempool is not full yet *) - `Weight_ok (`No_replace, [weight]) - else - match state.ops_state.min_prechecked_op_weight with - | None -> - (* The precheck mempool is empty *) - `Weight_ok (`No_replace, [weight]) - | Some {weight = min_weight; operation_hash = min_oph} -> - if Q.(weight > min_weight) then - (* The operation has a weight greater than the minimal - prechecked operation, replace the latest with the new one *) - `Weight_ok (`Replace min_oph, [weight]) - else - (* Otherwise fail and give indication as to what to fee should - be for the operation to be prechecked *) - let required_fee = - required_fee_manager_operation_weight ~op_resources ~min_weight - in - `Fail - (`Branch_delayed - [Environment.wrap_tzerror (Fees_too_low_for_mempool required_fee)]) - let pre_filter_manager : type t. + filter_info -> config -> - state -> Operation.packed_protocol_data -> t Kind.manager contents_list -> [ `Passed_prefilter of Q.t list @@ -548,7 +282,7 @@ let pre_filter_manager : | `Branch_delayed of tztrace | `Refused of tztrace | `Outdated of tztrace ] = - fun config filter_state packed_op op -> + fun filter_info config packed_op op -> let size = size_of_operation packed_op in let check_gas_and_fee fee gas_limit = let fees_in_nanotez = @@ -582,12 +316,15 @@ let pre_filter_manager : | Ok (fee, gas_limit) -> ( match check_gas_and_fee fee gas_limit with | `Refused _ as err -> err - | `Fees_ok -> ( - match - check_minimal_weight config filter_state ~fee ~gas_limit packed_op - with - | `Fail errs -> errs - | `Weight_ok (_, weight) -> `Passed_prefilter weight)) + | `Fees_ok -> + let weight, _op_resources = + weight_and_resources_manager_operation + ~hard_gas_limit_per_block:filter_info.hard_gas_limit_per_block + ~fee + ~gas:gas_limit + packed_op + in + `Passed_prefilter [weight]) type Environment.Error_monad.error += Wrong_operation @@ -753,23 +490,20 @@ let acceptable_op ~config ~round_durations ~round_zero_duration ~proposal_level acceptable *) acceptable ~drift ~op_earliest_ts ~now_timestamp -let pre_filter_far_future_consensus_ops config ~filter_state +let pre_filter_far_future_consensus_ops filter_info config ({level = op_level; round = op_round; _} : consensus_content) : bool Lwt.t = let res = let open Result_syntax in let now_timestamp = Time.System.now () |> Time.System.to_protocol in - let* proposal_level = - Raw_level.of_int32 filter_state.state_info.head.level - in + let* proposal_level = Raw_level.of_int32 filter_info.head.level in acceptable_op ~config - ~round_durations:filter_state.state_info.round_durations - ~round_zero_duration:filter_state.state_info.round_zero_duration + ~round_durations:filter_info.round_durations + ~round_zero_duration:filter_info.round_zero_duration ~proposal_level - ~proposal_round:filter_state.state_info.head_round - ~proposal_timestamp:filter_state.state_info.head.timestamp - ~proposal_predecessor_level_start: - filter_state.state_info.grandparent_level_start + ~proposal_round:filter_info.head_round + ~proposal_timestamp:filter_info.head.timestamp + ~proposal_predecessor_level_start:filter_info.grandparent_level_start ~op_level ~op_round ~now_timestamp @@ -785,13 +519,13 @@ let pre_filter_far_future_consensus_ops config ~filter_state We add [config.clock_drift] time as a safety margin. *) -let pre_filter config ~filter_state +let pre_filter filter_info config ({shell = _; protocol_data = Operation_data {contents; _} as op} : Main.operation) = let prefilter_manager_op manager_op = Lwt.return @@ - match pre_filter_manager config filter_state op manager_op with + match pre_filter_manager filter_info config op manager_op with | `Passed_prefilter prio -> `Passed_prefilter (manager_prio prio) | (`Branch_refused _ | `Branch_delayed _ | `Refused _ | `Outdated _) as err -> @@ -802,7 +536,7 @@ let pre_filter config ~filter_state Lwt.return (`Refused [Environment.wrap_tzerror Wrong_operation]) | Single (Preendorsement consensus_content) | Single (Endorsement consensus_content) -> - pre_filter_far_future_consensus_ops ~filter_state config consensus_content + pre_filter_far_future_consensus_ops filter_info config consensus_content >>= fun keep -> if keep then Lwt.return @@ `Passed_prefilter consensus_prio else @@ -823,171 +557,28 @@ let pre_filter config ~filter_state | Single (Manager_operation _) as op -> prefilter_manager_op op | Cons (Manager_operation _, _) as op -> prefilter_manager_op op -(** Remove a manager operation hash from the ops_state. - Do nothing if the operation was not in the state. *) -let remove_operation ops_state oph = - match Operation_hash.Map.find oph ops_state.prechecked_manager_ops with - | None -> - (* Not present in the ops_state: nothing to do. *) - ops_state - | Some info -> - let prechecked_manager_ops = - Operation_hash.Map.remove oph ops_state.prechecked_manager_ops - in - let prechecked_manager_op_count = - ops_state.prechecked_manager_op_count - 1 - in - let prechecked_op_weights = - ManagerOpWeightSet.remove - (mk_op_weight oph info) - ops_state.prechecked_op_weights - in - let min_prechecked_op_weight = - match ops_state.min_prechecked_op_weight with - | None -> None - | Some min_op_weight -> - if Operation_hash.equal min_op_weight.operation_hash oph then - ManagerOpWeightSet.min_elt prechecked_op_weights - else Some min_op_weight - in - { - prechecked_manager_op_count; - prechecked_manager_ops; - prechecked_op_weights; - min_prechecked_op_weight; - } - -(** Remove a manager operation hash from the ops_state. - Do nothing if the operation was not in the state. *) -let remove ~filter_state oph = - {filter_state with ops_state = remove_operation filter_state.ops_state oph} - -(** Add a manager operation hash and information to the filter state. - Do nothing if the operation is already present in the state. *) -let add_manager_op ops_state oph info replacement = - let ops_state = - match replacement with - | `No_replace -> ops_state - | `Replace (oph, _classification) -> remove_operation ops_state oph - in - if Operation_hash.Map.mem oph ops_state.prechecked_manager_ops then - (* Already present in the ops_state: nothing to do. *) - ops_state - else - let prechecked_manager_op_count = - ops_state.prechecked_manager_op_count + 1 - in - let prechecked_manager_ops = - Operation_hash.Map.add oph info ops_state.prechecked_manager_ops - in - let op_weight = mk_op_weight oph info in - let prechecked_op_weights = - ManagerOpWeightSet.add op_weight ops_state.prechecked_op_weights - in - let min_prechecked_op_weight = - match ops_state.min_prechecked_op_weight with - | Some old_min when compare_manager_op_weight old_min op_weight <= 0 -> - Some old_min - | _ -> Some op_weight - in - { - prechecked_manager_op_count; - prechecked_manager_ops; - prechecked_op_weights; - min_prechecked_op_weight; - } - -let add_manager_op_and_enforce_mempool_bound config filter_state oph - (op : 'manager_kind Kind.manager operation) = - let open Lwt_result_syntax in - let*? fee, gas_limit = - Result.map_error - (fun err -> `Refused (Environment.wrap_tztrace err)) - (get_manager_operation_gas_and_fee op.protocol_data.contents) - in - let* replacement, weight = - match - check_minimal_weight - config - filter_state - ~fee - ~gas_limit - (Operation_data op.protocol_data) - with - | `Weight_ok (`No_replace, weight) -> - (* The mempool is not full: no need to replace any operation. *) - return (`No_replace, weight) - | `Weight_ok (`Replace min_weight_oph, weight) -> - (* The mempool is full yet the new operation has enough weight - to be included: the old operation with the lowest weight is - reclassified as [Branch_delayed]. *) - (* TODO: https://gitlab.com/tezos/tezos/-/issues/2347 The - branch_delayed ring is bounded to 1000, so we may loose - operations. We can probably do better. *) - let replace_err = - Environment.wrap_tzerror Removed_fees_too_low_for_mempool - in - let replacement = - `Replace (min_weight_oph, `Branch_delayed [replace_err]) - in - return (replacement, weight) - | `Fail err -> - (* The mempool is full and the weight of the new operation is - too low: raise the error returned by {!check_minimal_weight}. *) - fail err - in - let weight = match weight with [x] -> x | _ -> assert false in - let info = {manager_op = Manager_op op; gas_limit; fee; weight} in - let ops_state = add_manager_op filter_state.ops_state oph info replacement in - return ({filter_state with ops_state}, replacement) - -(** If the provided operation is a manager operation, add it to the - filter_state. If the mempool is full, either return an error if the - operation does not have enough weight to be included, or return the - operation with minimal weight that gets removed to make room. - - Do nothing on non-manager operations. - - If [replace] is provided, then it is removed from [filter_state] - before processing [op]. (If [replace] is a non-manager operation, - this does nothing since it was never in [filter_state] to begin with.) - Note that when this happens, the mempool can no longer be full after - the operation has been removed, so this function always returns - [`No_replace]. - - This function is designed to be called by the shell each time a - new operation has been validated by the protocol. It will be - removed in the future once the shell is able to bound the number of - operations in the mempool by itself. *) -let add_operation_and_enforce_mempool_bound ?replace config filter_state - (oph, op) = - let filter_state = - match replace with - | Some replace_oph -> - (* If the operation to replace is not a manager operation, then - it cannot be present in the [filter_state]. But then, - [remove] does nothing anyway. *) - remove ~filter_state replace_oph - | None -> filter_state - in - let {protocol_data = Operation_data protocol_data; _} = op in - let call_manager protocol_data = - add_manager_op_and_enforce_mempool_bound - config - filter_state - oph - {shell = op.shell; protocol_data} - in - match protocol_data.contents with - | Single (Manager_operation _) -> call_manager protocol_data - | Cons (Manager_operation _, _) -> call_manager protocol_data - | Single _ -> return (filter_state, `No_replace) - let is_manager_operation op = match Operation.acceptable_pass op with | Some pass -> Compare.Int.equal pass Operation_repr.manager_pass | None -> false +(** Determine whether the new manager operation is sufficiently better + than the old manager operation to replace it. Sufficiently better + means that the new operation's fee and fee/gas ratio are both + greater than or equal to the old operation's same metrics bumped by + the factor [config.replace_by_fee_factor]. *) +let better_fees_and_ratio = + let bump config q = Q.mul q config.replace_by_fee_factor in + fun config old_gas old_fee new_gas new_fee -> + let old_fee = Tez.to_mutez old_fee |> Z.of_int64 |> Q.of_bigint in + let old_gas = Gas.Arith.integral_to_z old_gas |> Q.of_bigint in + let new_fee = Tez.to_mutez new_fee |> Z.of_int64 |> Q.of_bigint in + let new_gas = Gas.Arith.integral_to_z new_gas |> Q.of_bigint in + let old_ratio = Q.div old_fee old_gas in + let new_ratio = Q.div new_fee new_gas in + Q.compare new_ratio (bump config old_ratio) >= 0 + && Q.compare new_fee (bump config old_fee) >= 0 + (** [conflict_handler config] returns a conflict handler for {!Mempool.add_operation} (see {!Mempool.conflict_handler}). @@ -1028,3 +619,15 @@ let conflict_handler config : Mempool.conflict_handler = | Ok _ | Error _ -> `Keep else if Operation.compare existing_operation new_operation < 0 then `Replace else `Keep + +module Internal_for_tests = struct + let default_config_with_clock_drift clock_drift = + {default_config with clock_drift} + + let default_config_with_replace_factor replace_by_fee_factor = + {default_config with replace_by_fee_factor} + + let get_clock_drift {clock_drift; _} = clock_drift + + let acceptable_op = acceptable_op +end diff --git a/src/proto_alpha/lib_plugin/mempool.mli b/src/proto_alpha/lib_plugin/mempool.mli new file mode 100644 index 0000000000000000000000000000000000000000..7562f4aedc0527430cbe973419eda5b47b14fb9d --- /dev/null +++ b/src/proto_alpha/lib_plugin/mempool.mli @@ -0,0 +1,150 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2018 Nomadic Development. *) +(* Copyright (c) 2021 Nomadic Labs, *) +(* Copyright (c) 2022 TriliTech *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(** Plugin for the shell mempool. It must include the signature + [FILTER.Mempool] from [lib_shell/shell_plugin.mli]. *) + +(** Settings for the {!pre_filter}: + - minimal fees to accept an operation (absolute, relative to the + gas limit, and relative to the byte size) + - clock drift for the prefiltering of consensus operations + + and for the {!conflict_handler}: + - replacement factor, that is, how much better a new manager + operation needs to be, in terms of both absolute fees and fee/gas + ratios, in order to replace an old conflicting manager operation. *) +type config + +(** Default parameters. *) +val default_config : config + +(** Encoding for {!config}. *) +val config_encoding : config Data_encoding.t + +(** Static information needed by {!pre_filter}. + + It depends on the [head] block upon which a mempool is built. *) +type filter_info + +(** Create a {!filter_info} based on the [head] block and the current + context. *) +val init : + Environment.Context.t -> + head:Block_header.shell_header -> + (filter_info, tztrace) result Lwt.t + +(** Create a new {!filter_info} based on the [head] block. + + Parts of the old {!filter_info} (which may have been built on + a different block) are recycled, so that this function is more + efficient than {!init} and does not need an + {!Environment.Context.t} argument. *) +val flush : + filter_info -> head:Block_header.shell_header -> filter_info tzresult Lwt.t + +(** Perform some preliminary checks on an operation. + + For manager operations, check that its fee, fee/gas ratio, and + fee/size ratio all meet the minimal requirements specified in the + {!config}. + + For consensus operations, check that it is possible for the + operation to have been produced before now (plus additional time + equal to the [clock_drift] from {!config}, as a safety margin). + Indeed, without this check, a baker could flood the network with + consensus operations for any future rounds or levels. The ml file + contains more detailled explanations with diagrams. *) +val pre_filter : + filter_info -> + config -> + Protocol.Alpha_context.packed_operation -> + [ `Passed_prefilter of [`High | `Medium | `Low of Q.t list] + | `Branch_delayed of tztrace + | `Branch_refused of tztrace + | `Refused of tztrace + | `Outdated of tztrace ] + Lwt.t + +(** Return a conflict handler for {!Protocol.Mempool.add_operation} + (see {!Protocol.Mempool.conflict_handler}). + + For non-manager operations, select the greater operation according + to {!Protocol.Alpha_context.Operation.compare}. + + A manager operation is replaced only when the new operation's fee + and fee/gas ratio both exceed the old operation's by at least a + factor specified in the {!config}. + + Precondition: both operations must be individually valid (to be + able to call {!Protocol.Alpha_context.Operation.compare}). *) +val conflict_handler : config -> Protocol.Mempool.conflict_handler + +(** The following type, encoding, and default values are exported for + [bin_sc_rollup_node/configuration.ml]. *) + +(** An amount of fees in nanotez. *) +type nanotez = Q.t + +(** Encoding for {!nanotez}. *) +val nanotez_enc : nanotez Data_encoding.t + +(** Minimal absolute fees in {!default_config}. *) +val default_minimal_fees : Protocol.Alpha_context.Tez.t + +(** Minimal fee over gas_limit ratio in {!default_config}. *) +val default_minimal_nanotez_per_gas_unit : nanotez + +(** Minimal fee over byte size ratio in {!default_config}. *) +val default_minimal_nanotez_per_byte : nanotez + +module Internal_for_tests : sig + open Protocol.Alpha_context + + (** {!default_config} with a custom value for the [clock_drift] field. *) + val default_config_with_clock_drift : Period.t option -> config + + (** {!default_config} with a custom [replace_by_fee_factor]. *) + val default_config_with_replace_factor : nanotez -> config + + (** Return the [clock_drift] field of the given {!config}. *) + val get_clock_drift : config -> Period.t option + + (** The main auxiliary function for {!pre_filter} regarding + consensus operations. *) + val acceptable_op : + config:config -> + round_durations:Round.round_durations -> + round_zero_duration:Period.t -> + proposal_level:Raw_level.t -> + proposal_round:Round.t -> + proposal_timestamp:Timestamp.time -> + proposal_predecessor_level_start:Timestamp.time -> + op_level:Raw_level.t -> + op_round:Round.t -> + now_timestamp:Timestamp.time -> + bool Environment.Error_monad.tzresult +end diff --git a/src/proto_alpha/lib_plugin/test/dune b/src/proto_alpha/lib_plugin/test/dune index 1d8398ee0a44c8e5b0bf3ed371657e8551d264e0..16c62fa130db18e639e363208f5a19483766e759 100644 --- a/src/proto_alpha/lib_plugin/test/dune +++ b/src/proto_alpha/lib_plugin/test/dune @@ -1,11 +1,9 @@ ; This file was automatically generated, do not edit. ; Edit file manifest/main.ml instead. -(library - (name src_proto_alpha_lib_plugin_test_tezt_lib) - (instrumentation (backend bisect_ppx)) +(executables + (names test_conflict_handler test_consensus_filter) (libraries - tezt.core tezos-base tezos-base-test-helpers tezos-base.unix @@ -18,11 +16,11 @@ tezos-protocol-alpha tezos-protocol-alpha.parameters tezos-alpha-test-helpers) - (library_flags (:standard -linkall)) + (link_flags + (:standard) + (:include %{workspace_root}/macos-link-flags.sexp)) (flags (:standard) - -open Tezt_core - -open Tezt_core.Base -open Tezos_base.TzPervasives -open Tezos_base.TzPervasives.Error_monad.Legacy_monad_globals -open Tezos_base_test_helpers @@ -33,32 +31,14 @@ -open Tezos_protocol_alpha -open Tezos_protocol_alpha.Protocol -open Tezos_protocol_alpha_parameters - -open Tezos_alpha_test_helpers) - (modules - test_consensus_filter - test_filter_state - test_plugin - test_conflict_handler - test_utils - generators)) - -(executable - (name main) - (instrumentation (backend bisect_ppx --bisect-sigterm)) - (libraries - src_proto_alpha_lib_plugin_test_tezt_lib - tezt) - (link_flags - (:standard) - (:include %{workspace_root}/macos-link-flags.sexp)) - (modules main)) + -open Tezos_alpha_test_helpers)) (rule (alias runtest) (package tezos-protocol-plugin-alpha-tests) - (enabled_if (<> false %{env:RUNTEZTALIAS=true})) - (action (run %{dep:./main.exe}))) + (action (run %{dep:./test_conflict_handler.exe}))) (rule - (targets main.ml) - (action (with-stdout-to %{targets} (echo "let () = Tezt.Test.run ()")))) + (alias runtest) + (package tezos-protocol-plugin-alpha-tests) + (action (run %{dep:./test_consensus_filter.exe}))) diff --git a/src/proto_alpha/lib_plugin/test/generators.ml b/src/proto_alpha/lib_plugin/test/generators.ml deleted file mode 100644 index 2987fab6c3ca28554e002b1f78dd7259421c7e39..0000000000000000000000000000000000000000 --- a/src/proto_alpha/lib_plugin/test/generators.ml +++ /dev/null @@ -1,136 +0,0 @@ -(*****************************************************************************) -(* *) -(* Open Source License *) -(* Copyright (c) 2022 Nomadic Labs, *) -(* *) -(* Permission is hereby granted, free of charge, to any person obtaining a *) -(* copy of this software and associated documentation files (the "Software"),*) -(* to deal in the Software without restriction, including without limitation *) -(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) -(* and/or sell copies of the Software, and to permit persons to whom the *) -(* Software is furnished to do so, subject to the following conditions: *) -(* *) -(* The above copyright notice and this permission notice shall be included *) -(* in all copies or substantial portions of the Software. *) -(* *) -(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) -(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) -(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) -(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) -(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) -(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) -(* DEALINGS IN THE SOFTWARE. *) -(* *) -(*****************************************************************************) - -module Mempool = Plugin.Mempool - -let string_gen = QCheck2.Gen.small_string ?gen:None - -let public_key_hash_gen : - (Signature.public_key_hash * Signature.public_key * Signature.secret_key) - QCheck2.Gen.t = - let open QCheck2.Gen in - let+ seed = string_size (32 -- 64) in - let seed = Bytes.of_string seed in - Signature.generate_key ~seed () - -(* TODO: https://gitlab.com/tezos/tezos/-/issues/2407 - move this function to an helper file? *) -let operation_hash_gen : Operation_hash.t QCheck2.Gen.t = - let open QCheck2.Gen in - let+ s = QCheck2.Gen.string_size (return 32) in - Operation_hash.of_string_exn s - -let dummy_manager_op_info = - let fee = Alpha_context.Tez.zero in - let gas_limit = Alpha_context.Gas.Arith.zero in - let manager_op = - let open Alpha_context in - let source = Signature.Public_key_hash.zero in - let counter = Manager_counter.Internal_for_tests.of_int 0 in - let storage_limit = Z.zero in - let operation = Set_deposits_limit None in - let contents = - Manager_operation - {source; fee; counter; operation; gas_limit; storage_limit} - in - let contents = Single contents in - let protocol_data = {contents; signature = Some Signature.zero} in - let branch = Block_hash.zero in - Mempool.Manager_op {shell = {branch}; protocol_data} - in - Mempool.{manager_op; fee; gas_limit; weight = Q.zero} - -let oph_and_info_gen = - let open QCheck2.Gen in - let+ oph = operation_hash_gen in - (oph, dummy_manager_op_info) - -let filter_state_gen : Plugin.Mempool.ops_state QCheck2.Gen.t = - let open QCheck2.Gen in - let open Plugin.Mempool in - let+ ops = small_list oph_and_info_gen in - List.fold_left - (fun state (oph, info) -> - if Operation_hash.Map.mem oph state.prechecked_manager_ops then state - else - let prechecked_manager_op_count = - state.prechecked_manager_op_count + 1 - in - let op_weight = mk_op_weight oph info in - let min_prechecked_op_weight = - match state.min_prechecked_op_weight with - | Some old_min - when Mempool.compare_manager_op_weight old_min op_weight <= 0 -> - Some old_min - | Some _ | None -> Some op_weight - in - { - prechecked_manager_op_count; - prechecked_manager_ops = - Operation_hash.Map.add oph info state.prechecked_manager_ops; - prechecked_op_weights = - ManagerOpWeightSet.add op_weight state.prechecked_op_weights; - min_prechecked_op_weight; - }) - Plugin.Mempool.empty_ops_state - ops - -(** Generate a pair of operation hash and manager_op_info, that has - even odds of belonging to the given filter_state or being fresh. *) -let with_filter_state_operation_gen : - Plugin.Mempool.ops_state -> - (Operation_hash.t * Plugin.Mempool.manager_op_info) QCheck2.Gen.t = - fun state -> - let open QCheck2.Gen in - let* use_fresh = bool in - if use_fresh || Operation_hash.Map.is_empty state.prechecked_manager_ops then - oph_and_info_gen - else oneofl (Operation_hash.Map.bindings state.prechecked_manager_ops) - -(** Generate both a filter_state, and a pair of operation hash and - manager_op_info. The pair has even odds of belonging to the - filter_state or being fresh. *) -let filter_state_with_operation_gen : - (Plugin.Mempool.ops_state - * (Operation_hash.t * Plugin.Mempool.manager_op_info)) - QCheck2.Gen.t = - let open QCheck2.Gen in - filter_state_gen >>= fun state -> - pair (return state) (with_filter_state_operation_gen state) - -(** Generate a filter_state, and two pairs of operation hash and - manager_op_info. The pairs have indepedent, even odds of belonging - to the filter_state or being fresh. *) -let filter_state_with_two_operations_gen : - (Plugin.Mempool.ops_state - * (Operation_hash.t * Plugin.Mempool.manager_op_info) - * (Operation_hash.t * Plugin.Mempool.manager_op_info)) - QCheck2.Gen.t = - let open QCheck2.Gen in - let* filter_state = filter_state_gen in - triple - (return filter_state) - (with_filter_state_operation_gen filter_state) - (with_filter_state_operation_gen filter_state) diff --git a/src/proto_alpha/lib_plugin/test/test_conflict_handler.ml b/src/proto_alpha/lib_plugin/test/test_conflict_handler.ml index 02d337f40e6c4b153f7e81bd3c25a99b0d0a0566..7593af3d8dc7b28ba86d154af08c88aa8d0b7b0f 100644 --- a/src/proto_alpha/lib_plugin/test/test_conflict_handler.ml +++ b/src/proto_alpha/lib_plugin/test/test_conflict_handler.ml @@ -175,18 +175,17 @@ let test_manager_ops () = (* Config that requires 10% better fee and ratio to replace. *) let config10 = - { - Plugin.Mempool.default_config with - replace_by_fee_factor = Q.make (Z.of_int 11) (Z.of_int 10); - } + Plugin.Mempool.Internal_for_tests.default_config_with_replace_factor + (Q.make (Z.of_int 11) (Z.of_int 10)) in + check_conflict_handler ~__LOC__ config10 ~old ~nw:op_same `Keep ; check_conflict_handler ~__LOC__ config10 ~old ~nw:op_fee5 `Keep ; check_conflict_handler ~__LOC__ config10 ~old ~nw:op_ratio5 `Keep ; check_conflict_handler ~__LOC__ config10 ~old ~nw:op_both5 `Keep ; (* Config that replaces when the new op has at least as much fee and ratio. *) let config0 = - {Plugin.Mempool.default_config with replace_by_fee_factor = Q.one} + Plugin.Mempool.Internal_for_tests.default_config_with_replace_factor Q.one in check_conflict_handler ~__LOC__ config0 ~old ~nw:op_same `Replace ; check_conflict_handler ~__LOC__ config0 ~old ~nw:op_fee5 `Replace ; diff --git a/src/proto_alpha/lib_plugin/test/test_consensus_filter.ml b/src/proto_alpha/lib_plugin/test/test_consensus_filter.ml index 36750245d8d4168a86b9b56caee17f137fc85269..beb03ac1352f598fee9a41f4cddd0a5cbfce438e 100644 --- a/src/proto_alpha/lib_plugin/test/test_consensus_filter.ml +++ b/src/proto_alpha/lib_plugin/test/test_consensus_filter.ml @@ -32,9 +32,8 @@ *) open Qcheck2_helpers -open Plugin.Mempool open Alpha_context -open Test_utils +open Plugin.Mempool.Internal_for_tests (** {2. Conversion helpers} *) @@ -48,15 +47,20 @@ module Generator = struct prefix ^ printer d ^ suffix let config = - let+ x = opt small_nat in - config x + let* drift_int_opt = opt small_nat in + let clock_drift = + Option.map + (fun drift_int -> Period.of_seconds_exn (Int64.of_int drift_int)) + drift_int_opt + in + return (default_config_with_clock_drift clock_drift) let print_config = decorate ~prefix:"clock_drift " (fun config -> Option.fold ~none:"round_0 duration" ~some:(fun drift -> Int64.to_string @@ Period.to_seconds drift) - config.clock_drift) + (get_clock_drift config)) let of_result = Result.value_f ~default:(fun _ -> assert false) @@ -282,7 +286,7 @@ let test_acceptable_current_level = ~proposal_round ~round:op_round >>? fun expected_time -> - max_ts config.clock_drift proposal_timestamp now_timestamp + max_ts (get_clock_drift config) proposal_timestamp now_timestamp >>? fun max_timestamp -> ok Timestamp.(expected_time <= max_timestamp) ) ==> no_error @@ -321,7 +325,7 @@ let test_not_acceptable_current_level = ~proposal_round ~round:op_round >>? fun expected_time -> - max_ts config.clock_drift proposal_timestamp now_timestamp + max_ts (get_clock_drift config) proposal_timestamp now_timestamp >>? fun max_timestamp -> ok Timestamp.( @@ -370,7 +374,7 @@ let test_acceptable_next_level = ~predecessor_round:Round.zero ~round:op_round >>? fun expected_time -> - max_ts config.clock_drift proposal_timestamp now_timestamp + max_ts (get_clock_drift config) proposal_timestamp now_timestamp >>? fun max_timestamp -> ok Timestamp.(expected_time <= max_timestamp) ) ==> no_error @@ -421,7 +425,7 @@ let test_not_acceptable_next_level = proposal_timestamp +? Round.round_duration round_durations proposal_round) >>? fun next_level_ts -> - max_ts config.clock_drift next_level_ts now_timestamp + max_ts (get_clock_drift config) next_level_ts now_timestamp >>? fun max_timestamp -> ok Timestamp.(expected_time > max_timestamp) ) ; no_error diff --git a/src/proto_alpha/lib_plugin/test/test_filter_state.ml b/src/proto_alpha/lib_plugin/test/test_filter_state.ml deleted file mode 100644 index 4c46ac273838013b6c9d2ccf0b33e7a09190c271..0000000000000000000000000000000000000000 --- a/src/proto_alpha/lib_plugin/test/test_filter_state.ml +++ /dev/null @@ -1,194 +0,0 @@ -(*****************************************************************************) -(* *) -(* Open Source License *) -(* Copyright (c) 2022 Nomadic Labs, *) -(* *) -(* Permission is hereby granted, free of charge, to any person obtaining a *) -(* copy of this software and associated documentation files (the "Software"),*) -(* to deal in the Software without restriction, including without limitation *) -(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) -(* and/or sell copies of the Software, and to permit persons to whom the *) -(* Software is furnished to do so, subject to the following conditions: *) -(* *) -(* The above copyright notice and this permission notice shall be included *) -(* in all copies or substantial portions of the Software. *) -(* *) -(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) -(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) -(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) -(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) -(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) -(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) -(* DEALINGS IN THE SOFTWARE. *) -(* *) -(*****************************************************************************) - -(** Testing - ------- - Component: Shell (Plugin) - Invocation: dune exec src/proto_alpha/lib_plugin/test/main.exe \ - -- --file test_filter_state.ml - Subject: Unit tests the filter state functions of the plugin -*) - -open Qcheck2_helpers -open Plugin.Mempool -open Test_utils - -let count = 1000 - -let check_filter_state_invariants filter_state = - qcheck_cond - ~pp:(fun fmt filter_state -> - Format.fprintf - fmt - "The following filter_state breaks invariants:@.%a" - pp_state - filter_state) - ~cond:(fun filter_state -> - filter_state.prechecked_manager_op_count - = Operation_hash.Map.cardinal filter_state.prechecked_manager_ops - && filter_state.prechecked_manager_op_count - = ManagerOpWeightSet.cardinal filter_state.prechecked_op_weights - && ManagerOpWeightSet.for_all - (fun {operation_hash; weight} -> - match - Operation_hash.Map.find - operation_hash - filter_state.prechecked_manager_ops - with - | None -> false - | Some info -> Q.equal weight info.weight) - filter_state.prechecked_op_weights - && eq_op_weight_opt - (ManagerOpWeightSet.min_elt filter_state.prechecked_op_weights) - filter_state.min_prechecked_op_weight) - filter_state - () - -(** Test that [add_manager_op] adds the given operation to the filter - state, and removes the replaced operation if applicable. *) -let test_add_manager_op = - let open QCheck2 in - Test.make - ~count - ~name:"Add a manager operation" - (Gen.pair Generators.filter_state_with_two_operations_gen Gen.bool) - (fun ((filter_state, (oph, op_info), (oph_to_replace, _)), should_replace) - -> - (* Both [oph] and [oph_to_replace] have even odds of being - already present in [filter_state] or fresh. *) - let replacement = - if should_replace then ( - assume (not (Operation_hash.equal oph_to_replace oph)) ; - `Replace (oph_to_replace, ())) - else `No_replace - in - let filter_state = add_manager_op filter_state oph op_info replacement in - check_filter_state_invariants filter_state - && qcheck_cond - ~pp:(fun fmt set -> - Format.fprintf - fmt - "%a was not found in prechecked_manager_ops: %a" - Operation_hash.pp - oph - pp_prechecked_manager_ops - set) - ~cond:(Operation_hash.Map.mem oph) - filter_state.prechecked_manager_ops - () - && - if should_replace then - qcheck_cond - ~pp:(fun fmt () -> - Format.fprintf - fmt - "%a should have been removed from prechecked_manager_ops." - Operation_hash.pp - oph_to_replace) - ~cond:(fun () -> - not - (Operation_hash.Map.mem - oph_to_replace - filter_state.prechecked_manager_ops)) - () - () - else true) - -(** Test that [remove] removes the operation from the filter state if - it was present, and that adding then removing is the same as doing - nothing but removing the replaced operation if there is one. *) -let test_remove_present = - let open QCheck2 in - Test.make - ~count - ~name:"Remove an existing operation hash" - (Gen.triple - Generators.filter_state_with_operation_gen - Generators.oph_and_info_gen - Gen.bool) - (fun ((initial_state, (oph_to_replace, _)), (oph, op_info), should_replace) - -> - (* Add a fresh operation [oph] to the state. *) - assume - (not (Operation_hash.Map.mem oph initial_state.prechecked_manager_ops)) ; - let replacement = - if should_replace then `Replace (oph_to_replace, ()) else `No_replace - in - let filter_state = add_manager_op initial_state oph op_info replacement in - (* Remove [oph] from the state, in which it was present. *) - let filter_state = remove_operation filter_state oph in - let (_ : bool) = - (* Check that the state invariants are preserved and that - [oph] has been removed. *) - check_filter_state_invariants filter_state - && qcheck_cond - ~pp:(fun fmt () -> - Format.fprintf - fmt - "%a should have been removed from prechecked_manager_ops." - Operation_hash.pp - oph) - ~cond:(fun () -> - not - (Operation_hash.Map.mem - oph - filter_state.prechecked_manager_ops)) - () - () - in - (* Check that adding a fresh operation then removing it is the - same as doing nothing except removing any replaced operation. *) - let initial_state_without_replaced_op = - if should_replace then remove_operation initial_state oph_to_replace - else initial_state - in - qcheck_eq - ~pp:pp_state - ~eq:eq_state - initial_state_without_replaced_op - filter_state) - -(** Test that [remove] leaves the filter state intact if the operation - hash is unknown. *) -let test_remove_unknown = - let open QCheck2 in - Test.make - ~count - ~name:"Remove an unknown operation hash" - (Gen.pair Generators.filter_state_gen Generators.operation_hash_gen) - (fun (initial_state, oph) -> - assume - (not (Operation_hash.Map.mem oph initial_state.prechecked_manager_ops)) ; - let filter_state = remove_operation initial_state oph in - qcheck_eq ~pp:pp_state ~eq:eq_state initial_state filter_state) - -let () = - Alcotest.run - ~__FILE__ - Protocol.name - [ - ("add_manager_op", qcheck_wrap [test_add_manager_op]); - ("remove", qcheck_wrap [test_remove_present; test_remove_unknown]); - ] diff --git a/src/proto_alpha/lib_plugin/test/test_plugin.ml b/src/proto_alpha/lib_plugin/test/test_plugin.ml deleted file mode 100644 index 6d0f89e43f85930879714fed44729b1dafe25b08..0000000000000000000000000000000000000000 --- a/src/proto_alpha/lib_plugin/test/test_plugin.ml +++ /dev/null @@ -1,84 +0,0 @@ -(*****************************************************************************) -(* *) -(* Open Source License *) -(* Copyright (c) 2022 Nomadic Labs, *) -(* *) -(* Permission is hereby granted, free of charge, to any person obtaining a *) -(* copy of this software and associated documentation files (the "Software"),*) -(* to deal in the Software without restriction, including without limitation *) -(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) -(* and/or sell copies of the Software, and to permit persons to whom the *) -(* Software is furnished to do so, subject to the following conditions: *) -(* *) -(* The above copyright notice and this permission notice shall be included *) -(* in all copies or substantial portions of the Software. *) -(* *) -(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) -(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) -(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) -(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) -(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) -(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) -(* DEALINGS IN THE SOFTWARE. *) -(* *) -(*****************************************************************************) - -(** Testing - ------- - Component: Shell (Plugin) - Invocation: dune exec src/proto_alpha/lib_plugin/test/main.exe \ - -- --file test_plugin.ml - Subject: Unit tests the plugin -*) - -open Plugin.Mempool -open Test_utils - -let count = Some 1000 - -(** This test checks that [flush] empties the filter state. *) -let test_flush () = - let l = - QCheck2.Gen.generate - ~n:(Option.value ~default:1 count) - Generators.filter_state_with_operation_gen - in - List.iter_es - (fun (ops_state, (oph, (op_info : manager_op_info))) -> - let ops_state = add_manager_op ops_state oph op_info `No_replace in - let open Lwt_result_syntax in - let* block, _contract = Context.init1 () in - let* incremental = Incremental.begin_construction block in - let ctxt = Incremental.alpha_ctxt incremental in - let head = block.header.shell in - let round_durations = Alpha_context.Constants.round_durations ctxt in - let hard_gas_limit_per_block = - Alpha_context.Constants.hard_gas_limit_per_block ctxt - in - let* filter_state = - init_state ~head round_durations hard_gas_limit_per_block - in - let filter_state = {filter_state with ops_state} in - let* flushed_state = flush filter_state ~head in - if eq_state flushed_state.ops_state empty_ops_state then return_unit - else - failwith - "Flushed state is not empty : %a" - pp_state - flushed_state.ops_state) - l - -let () = - Alcotest_lwt.run - ~__FILE__ - Protocol.name - [ - ( "on_flush", - [ - Tztest.tztest - "[on_flush ~validation_state ...] yields an empty state " - `Quick - test_flush; - ] ); - ] - |> Lwt_main.run diff --git a/src/proto_alpha/lib_plugin/test/test_utils.ml b/src/proto_alpha/lib_plugin/test/test_utils.ml deleted file mode 100644 index 57402233a49e2a128e60f1bc831509bef25e12d0..0000000000000000000000000000000000000000 --- a/src/proto_alpha/lib_plugin/test/test_utils.ml +++ /dev/null @@ -1,116 +0,0 @@ -(*****************************************************************************) -(* *) -(* Open Source License *) -(* Copyright (c) 2022 Nomadic Labs, *) -(* *) -(* Permission is hereby granted, free of charge, to any person obtaining a *) -(* copy of this software and associated documentation files (the "Software"),*) -(* to deal in the Software without restriction, including without limitation *) -(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) -(* and/or sell copies of the Software, and to permit persons to whom the *) -(* Software is furnished to do so, subject to the following conditions: *) -(* *) -(* The above copyright notice and this permission notice shall be included *) -(* in all copies or substantial portions of the Software. *) -(* *) -(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) -(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) -(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) -(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) -(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) -(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) -(* DEALINGS IN THE SOFTWARE. *) -(* *) -(*****************************************************************************) - -open Plugin.Mempool -open Alpha_context - -let config drift_opt = - { - default_config with - clock_drift = - Option.map - (fun drift -> Period.of_seconds_exn (Int64.of_int drift)) - drift_opt; - replace_by_fee_factor = Q.make (Z.of_int 105) (Z.of_int 100); - } - -let pp_prechecked_manager_ops fmt set = - Format.fprintf - fmt - "[%a]" - (Format.pp_print_list (fun ppf (oph, (op_info : manager_op_info)) -> - Format.fprintf - ppf - "(%a -> {fee: %a; gas: %a; weight: %a})" - Operation_hash.pp - oph - Alpha_context.Tez.pp - op_info.fee - Alpha_context.Gas.Arith.pp - op_info.gas_limit - Q.pp_print - op_info.weight)) - (Operation_hash.Map.bindings set) - -let pp_manager_op_weight fmt weight = - Format.fprintf - fmt - "{oph: %a; weight: %a}" - Operation_hash.pp - weight.operation_hash - Q.pp_print - weight.weight - -let pp_prechecked_op_weights fmt set = - Format.fprintf - fmt - "[%a]" - (Format.pp_print_list (fun ppf manager_op_weight -> - pp_manager_op_weight ppf manager_op_weight)) - (ManagerOpWeightSet.elements set) - -let pp_op_weight_opt fmt = function - | None -> Format.fprintf fmt "None" - | Some op_weight -> - Format.fprintf fmt "Some %a" pp_manager_op_weight op_weight - -let pp_state fmt state = - Format.fprintf - fmt - "@[state:@,\ - {@,\ - prechecked_manager_op_count: %d@,\ - prechecked_manager_ops: %a;@,\ - prechecked_op_weights: %a;@,\ - min_prechecked_op_weight: %a@,\ - }@]" - state.prechecked_manager_op_count - pp_prechecked_manager_ops - state.prechecked_manager_ops - pp_prechecked_op_weights - state.prechecked_op_weights - pp_op_weight_opt - state.min_prechecked_op_weight - -let eq_prechecked_manager_ops = - Operation_hash.Map.equal - (fun - {manager_op = _; fee = fee1; gas_limit = gas1; weight = w1} - {manager_op = _; fee = fee2; gas_limit = gas2; weight = w2} - -> Tez.equal fee1 fee2 && Gas.Arith.equal gas1 gas2 && Q.equal w1 w2) - -let eq_op_weight_opt = - Option.equal (fun op_weight1 op_weight2 -> - Operation_hash.equal op_weight1.operation_hash op_weight2.operation_hash - && Q.equal op_weight1.weight op_weight2.weight) - -(* This function needs to be updated if the filter state is extended *) -let eq_state s1 s2 = - s1.prechecked_manager_op_count = s2.prechecked_manager_op_count - && eq_prechecked_manager_ops - s1.prechecked_manager_ops - s2.prechecked_manager_ops - && ManagerOpWeightSet.equal s1.prechecked_op_weights s2.prechecked_op_weights - && eq_op_weight_opt s1.min_prechecked_op_weight s2.min_prechecked_op_weight diff --git a/tezt/lib_tezos/constant.ml b/tezt/lib_tezos/constant.ml index 1e339abe1f837facaf529531465eca1a8c4edca4..0cb83e054c8b20e003c1731846df82f6ccdbbb7d 100644 --- a/tezt/lib_tezos/constant.ml +++ b/tezt/lib_tezos/constant.ml @@ -129,4 +129,9 @@ module Error_msg = struct rex "Gas limit exceeded during typechecking or execution.\n\ Try again with a higher gas limit." + + let rejected_by_full_mempool = + rex + "The mempool is full and operation (.*) does not have enough fees to \ + replace existing operations" end diff --git a/tezt/lib_tezos/mempool.ml b/tezt/lib_tezos/mempool.ml index f02c1f5d4a5e5f2a08986b1c9abf5258c4b2bf12..207d0ae01055c7f75af958f2fe5c81480ddb9e4a 100644 --- a/tezt/lib_tezos/mempool.ml +++ b/tezt/lib_tezos/mempool.ml @@ -128,3 +128,227 @@ let check_mempool ?(applied = []) ?(branch_delayed = []) ?(branch_refused = []) (expected_mempool = mempool) classified_typ ~error_msg:"Expected mempool %L, got %R") + +module Config = struct + type t = { + minimal_fees : int option; + minimal_nanotez_per_gas_unit : (int * int) option; + minimal_nanotez_per_byte : (int * int) option; + replace_by_fee_factor : (int * int) option; + max_operations : int option; + max_total_bytes : int option; + } + + let make ?minimal_fees ?minimal_nanotez_per_gas_unit ?minimal_nanotez_per_byte + ?replace_by_fee_factor ?max_operations ?max_total_bytes () = + { + minimal_fees; + minimal_nanotez_per_gas_unit; + minimal_nanotez_per_byte; + replace_by_fee_factor; + max_operations; + max_total_bytes; + } + + let eq_int_pair (f1, s1) (f2, s2) = Int.equal f1 f2 && Int.equal s1 s2 + + let equal x1 x2 = + Option.equal Int.equal x1.minimal_fees x2.minimal_fees + && Option.equal + eq_int_pair + x1.minimal_nanotez_per_gas_unit + x2.minimal_nanotez_per_gas_unit + && Option.equal + eq_int_pair + x1.minimal_nanotez_per_byte + x2.minimal_nanotez_per_byte + && Option.equal + eq_int_pair + x1.replace_by_fee_factor + x2.replace_by_fee_factor + && Option.equal Int.equal x1.max_operations x2.max_operations + && Option.equal Int.equal x1.max_total_bytes x2.max_total_bytes + + let to_json_u config : JSON.u = + let str_value n = `String (string_of_int n) in + let int_str_field name = Option.map (fun n -> (name, str_value n)) in + let pair_field name = + Option.map (fun (n1, n2) -> (name, `A [str_value n1; str_value n2])) + in + let int_field name = + Option.map (fun n -> (name, `Float (float_of_int n))) + in + `O + (List.filter_map + Fun.id + [ + int_str_field "minimal_fees" config.minimal_fees; + pair_field + "minimal_nanotez_per_gas_unit" + config.minimal_nanotez_per_gas_unit; + pair_field "minimal_nanotez_per_byte" config.minimal_nanotez_per_byte; + pair_field "replace_by_fee_factor" config.replace_by_fee_factor; + int_field "max_operations" config.max_operations; + int_field "max_total_bytes" config.max_total_bytes; + ]) + + let to_string config = Ezjsonm.value_to_string (to_json_u config) + + let pp fmt config = Format.fprintf fmt "%s" (to_string config) + + let check_equal expected actual = + Check.( + (expected = actual) + (equalable pp equal) + ~error_msg:"Wrong filter configuration: %R.\nExpected: %L.") + + let of_json json = + let open JSON in + let as_int_pair_opt t = + match as_list_opt t with + | Some [x; y] -> Some (as_int x, as_int y) + (* A missing field is interpreted as [`Null], from which [as_list_opt] + produces [Some []]. *) + | Some [] -> None + | Some _ | None -> + Test.fail + "Constructing a filter_config from json: %s. Expected a list of \ + length 2, found: %s." + (encode json) + (encode t) + in + { + minimal_fees = json |-> "minimal_fees" |> as_int_opt; + minimal_nanotez_per_gas_unit = + json |-> "minimal_nanotez_per_gas_unit" |> as_int_pair_opt; + minimal_nanotez_per_byte = + json |-> "minimal_nanotez_per_byte" |> as_int_pair_opt; + replace_by_fee_factor = + json |-> "replace_by_fee_factor" |> as_int_pair_opt; + max_operations = json |-> "max_operations" |> as_int_opt; + max_total_bytes = json |-> "max_total_bytes" |> as_int_opt; + } + + (** Default filter configuration for protocol alpha + (see src/proto_alpha/lib_plugin/plugin.ml + and src/lib_shell/prevalidator_bounding.ml). *) + + let default_minimal_fees = 100 + + let default_minimal_nanotez_per_gas_unit = (100, 1) + + let default_minimal_nanotez_per_byte = (1000, 1) + + let default_replace_by_fee_factor = (21, 20) + + let default_max_operations = 10_000 + + let default_max_total_bytes = 10_000_000 + + let default = + { + minimal_fees = Some default_minimal_fees; + minimal_nanotez_per_gas_unit = Some default_minimal_nanotez_per_gas_unit; + minimal_nanotez_per_byte = Some default_minimal_nanotez_per_byte; + replace_by_fee_factor = Some default_replace_by_fee_factor; + max_operations = Some default_max_operations; + max_total_bytes = Some default_max_total_bytes; + } + + let fill_with_default config = + let aux default v = Some (Option.value v ~default) in + { + minimal_fees = aux default_minimal_fees config.minimal_fees; + minimal_nanotez_per_gas_unit = + aux + default_minimal_nanotez_per_gas_unit + config.minimal_nanotez_per_gas_unit; + minimal_nanotez_per_byte = + aux default_minimal_nanotez_per_byte config.minimal_nanotez_per_byte; + replace_by_fee_factor = + aux default_replace_by_fee_factor config.replace_by_fee_factor; + max_operations = aux default_max_operations config.max_operations; + max_total_bytes = aux default_max_total_bytes config.max_total_bytes; + } + + (** Return a copy of the given filter config, where fields equal + to their default value have been removed (i.e. set to [None]). *) + let clear_default config = + let clear_if_default eq_fun default = function + | Some x when eq_fun default x -> None + | x -> x + in + let aux_int = clear_if_default Int.equal in + let aux_pair = clear_if_default eq_int_pair in + { + minimal_fees = aux_int default_minimal_fees config.minimal_fees; + minimal_nanotez_per_gas_unit = + aux_pair + default_minimal_nanotez_per_gas_unit + config.minimal_nanotez_per_gas_unit; + minimal_nanotez_per_byte = + aux_pair + default_minimal_nanotez_per_byte + config.minimal_nanotez_per_byte; + replace_by_fee_factor = + aux_pair default_replace_by_fee_factor config.replace_by_fee_factor; + max_operations = aux_int default_max_operations config.max_operations; + max_total_bytes = aux_int default_max_total_bytes config.max_total_bytes; + } + + let check_get_filter_all_variations ?(log = false) expected_config client = + let expected_full = fill_with_default expected_config in + let* json = RPC.Client.call client @@ RPC.get_chain_mempool_filter () in + check_equal expected_full (of_json json) ; + let* json = + RPC.Client.call client + @@ RPC.get_chain_mempool_filter ~include_default:true () + in + check_equal expected_full (of_json json) ; + let expected_partial = clear_default expected_config in + let* json = + RPC.Client.call client + @@ RPC.get_chain_mempool_filter ~include_default:false () + in + check_equal expected_partial (of_json json) ; + if log then + Log.info + "GET /chains/main/mempool/filter returned expected configurations \ + (respectively including/excluding default fields): %s and %s." + (to_string expected_full) + (to_string expected_partial) ; + unit + + let call_post_filter json_u client = + RPC.Client.call client + @@ RPC.post_chain_mempool_filter ~data:(Data json_u) () + + let post_filter ?(log = false) config client = + let json_u = to_json_u config in + if log then + Log.info + "Set mempool filter config to: %s." + (Ezjsonm.value_to_string json_u) ; + call_post_filter json_u client + + let post_filter_str ?(log = false) config_str client = + if log then Log.info "Set mempool filter config to: %s." config_str ; + call_post_filter (Ezjsonm.from_string config_str) client + + let set_filter ?log ?minimal_fees ?minimal_nanotez_per_gas_unit + ?minimal_nanotez_per_byte ?replace_by_fee_factor ?max_operations + ?max_total_bytes client = + let config = + make + ?minimal_fees + ?minimal_nanotez_per_gas_unit + ?minimal_nanotez_per_byte + ?replace_by_fee_factor + ?max_operations + ?max_total_bytes + () + in + let* output = post_filter ?log config client in + check_equal (fill_with_default config) (of_json output) ; + unit +end diff --git a/tezt/lib_tezos/mempool.mli b/tezt/lib_tezos/mempool.mli index 0e8e4fe0405c91ae84d07217e66266a755f23548..cf949701ce72698eb37ea2d0d31ca63dc07a488d 100644 --- a/tezt/lib_tezos/mempool.mli +++ b/tezt/lib_tezos/mempool.mli @@ -77,3 +77,95 @@ val check_mempool : ?unprocessed:string list -> t -> unit + +(** Mempool filter configuration. *) +module Config : sig + (** Representation of the mempool configuration. [None] means that + the field has been or will be omitted in the json encoding, in + which case they are equal to the default values given below. *) + type t = { + minimal_fees : int option; + minimal_nanotez_per_gas_unit : (int * int) option; + minimal_nanotez_per_byte : (int * int) option; + replace_by_fee_factor : (int * int) option; + max_operations : int option; + max_total_bytes : int option; + } + + (** Build the Ezjsonm representation then encode it as a string. *) + val to_string : t -> string + + (** Call [Check.( = )] on two configurations. *) + val check_equal : t -> t -> unit + + (** Return the config corresponding to the given json. If any field + is missing, it is set to [None]. *) + val of_json : JSON.t -> t + + (** Default value for the [minimal_fees] field + (see src/proto_alpha/lib_plugin/plugin.ml) *) + val default_minimal_fees : int + + (** Default value for the [minimal_nanotez_per_gas_unit] field + (see src/proto_alpha/lib_plugin/plugin.ml) *) + val default_minimal_nanotez_per_gas_unit : int * int + + (** Default value for the [minimal_nanotez_per_byte] field + (see src/proto_alpha/lib_plugin/plugin.ml) *) + val default_minimal_nanotez_per_byte : int * int + + (** Default value for the [replace_by_fee_factor] field + (see src/proto_alpha/lib_plugin/plugin.ml) *) + val default_replace_by_fee_factor : int * int + + (** Default value for the [max_operations] field + (see src/lib_shell/prevalidator_bounding.ml) *) + val default_max_operations : int + + (** Default value for the [max_total_bytes] field + (see src/lib_shell/prevalidator_bounding.ml) *) + val default_max_total_bytes : int + + (** Configuration where each field is explicitely set to its default value. *) + val default : t + + (** Returns a copy of the given filter config, where missing fields + (i.e. containing [None]) have been set to their default value. *) + val fill_with_default : t -> t + + (** Check that the RPC [GET /chains/main/mempool/filter] returns the json + corresponding to the provided {!t}, testing all possibilities for + the optional argument [include_default] (omitted/[true]/[false]). *) + val check_get_filter_all_variations : ?log:bool -> t -> Client.t -> unit Lwt.t + + (** Call the RPC [POST /chains/main/mempool/filter] with data set to + the json representation of the given config {!t}. + + @param log When true, log the input config at the info + level. Defaults to false. *) + val post_filter : ?log:bool -> t -> Client.t -> JSON.t Lwt.t + + (** Same as {!post_filter} but takes the data config as a string. *) + val post_filter_str : ?log:bool -> string -> Client.t -> JSON.t Lwt.t + + (** Set the mempool filter config to the provided parameters. Note + that every omitted parameter will be set back to its default + value, even if it previously held a different value. + + More precisely, call {!post_filter} on the config corresponding + to the arguments named after config fields. Then, check that the + config returned by the RPC is the same as the input one (where + each omitted field has been filled with its default value). + + @param log is passed on to {!post_filter}. *) + val set_filter : + ?log:bool -> + ?minimal_fees:int -> + ?minimal_nanotez_per_gas_unit:int * int -> + ?minimal_nanotez_per_byte:int * int -> + ?replace_by_fee_factor:int * int -> + ?max_operations:int -> + ?max_total_bytes:int -> + Client.t -> + unit Lwt.t +end diff --git a/tezt/tests/RPC_test.ml b/tezt/tests/RPC_test.ml index cf261c95f818791510caaa4fb97ec58d4f77e2c9..cfc015511d175753a359171b5f264b6314f6a5fc 100644 --- a/tezt/tests/RPC_test.ml +++ b/tezt/tests/RPC_test.ml @@ -972,16 +972,28 @@ let test_mempool _test_mode_tag protocol ?endpoint client = in get_filter_variations () in - let* _ = get_filter_variations () in - let* _ = + let* () = get_filter_variations () in + let* () = + (* valid configuration *) post_and_get_filter - {|{ "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false }|} + {|{ "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], + "minimal_nanotez_per_byte": [ "56", "3" ], + "replace_by_fee_factor": ["21", "20"], "max_operations": 0, + "max_total_bytes": 100_000_000 }|} in - let* _ = + let* () = + (* valid configuration with omitted fields *) post_and_get_filter - {|{ "minimal_fees": "200", "allow_script_failure": true }|} + {|{ "minimal_fees": "200", "replace_by_fee_factor": ["1", "1"] }|} + in + let* () = + (* invalid field name *) + post_and_get_filter {|{ "max_operations": 100, "invalid_field_name": 100 }|} + in + let* () = + (* ill-typed data *) post_and_get_filter {|{ "minimal_fees": "toto" }|} in - let* _ = post_and_get_filter "{}" in + let* () = (* back to default config *) post_and_get_filter "{}" in unit let start_with_acl address acl = diff --git a/tezt/tests/dune b/tezt/tests/dune index 5c3f437ce5f58bc87138805a892bf74e82c79a16..2f3d4a73043b823b8fa19fc38bfbe322e39b67d5 100644 --- a/tezt/tests/dune +++ b/tezt/tests/dune @@ -27,7 +27,6 @@ src_proto_alpha_lib_protocol_test_integration_gas_tezt_lib src_proto_alpha_lib_protocol_test_integration_consensus_tezt_lib src_proto_alpha_lib_protocol_test_integration_tezt_lib - src_proto_alpha_lib_plugin_test_tezt_lib src_proto_alpha_lib_delegate_test_tezt_lib src_proto_alpha_lib_dal_test_tezt_lib src_proto_alpha_lib_dac_plugin_test_tezt_lib @@ -42,7 +41,6 @@ src_proto_017_PtNairob_lib_protocol_test_integration_gas_tezt_lib src_proto_017_PtNairob_lib_protocol_test_integration_consensus_tezt_lib src_proto_017_PtNairob_lib_protocol_test_integration_tezt_lib - src_proto_017_PtNairob_lib_plugin_test_tezt_lib src_proto_017_PtNairob_lib_delegate_test_tezt_lib src_proto_017_PtNairob_lib_dal_test_tezt_lib src_proto_017_PtNairob_lib_dac_plugin_test_tezt_lib @@ -57,7 +55,6 @@ src_proto_016_PtMumbai_lib_protocol_test_integration_gas_tezt_lib src_proto_016_PtMumbai_lib_protocol_test_integration_consensus_tezt_lib src_proto_016_PtMumbai_lib_protocol_test_integration_tezt_lib - src_proto_016_PtMumbai_lib_plugin_test_tezt_lib src_proto_016_PtMumbai_lib_delegate_test_tezt_lib src_proto_016_PtMumbai_lib_dal_test_tezt_lib src_proto_016_PtMumbai_lib_client_test_tezt_lib diff --git a/tezt/tests/expected/RPC_test.ml/Alpha- (mode client) RPC regression tests- mempool.out b/tezt/tests/expected/RPC_test.ml/Alpha- (mode client) RPC regression tests- mempool.out index b4e657f61c30f919d1de67ee08d5a8b8133bdd16..f609b2dbc4c73aa45306d91b3723a283bad812a6 100644 --- a/tezt/tests/expected/RPC_test.ml/Alpha- (mode client) RPC regression tests- mempool.out +++ b/tezt/tests/expected/RPC_test.ml/Alpha- (mode client) RPC regression tests- mempool.out @@ -288,15 +288,15 @@ curl -s 'http://localhost:[PORT]/chains/main/mempool/monitor_operations?applied= ./octez-client rpc get /chains/main/mempool/filter { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=false' {} @@ -311,70 +311,126 @@ curl -s 'http://localhost:[PORT]/chains/main/mempool/monitor_operations?applied= "56", "3" ], - "allow_script_failure": false + "replace_by_fee_factor": [ + "21", + "20" + ], + "max_operations": 0, + "max_total_bytes": 100000000 }' { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "56", "3" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client rpc get /chains/main/mempool/filter { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "56", "3" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "56", "3" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=false' { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false } + "minimal_nanotez_per_byte": [ "56", "3" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client rpc post /chains/main/mempool/filter with '{ "minimal_fees": "200", - "allow_script_failure": true + "replace_by_fee_factor": [ + "1", + "1" + ] +}' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client rpc get /chains/main/mempool/filter +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client rpc get '/chains/main/mempool/filter?include_default=true' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client rpc get '/chains/main/mempool/filter?include_default=false' +{ "minimal_fees": "200", "replace_by_fee_factor": [ "1", "1" ] } + +./octez-client rpc post /chains/main/mempool/filter with '{ + "max_operations": 100, + "invalid_field_name": 100 +}' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client rpc get /chains/main/mempool/filter +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client rpc get '/chains/main/mempool/filter?include_default=true' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client rpc get '/chains/main/mempool/filter?include_default=false' +{ "minimal_fees": "200", "replace_by_fee_factor": [ "1", "1" ] } + +./octez-client rpc post /chains/main/mempool/filter with '{ + "minimal_fees": "toto" }' { "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get /chains/main/mempool/filter { "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=false' -{ "minimal_fees": "200" } +{ "minimal_fees": "200", "replace_by_fee_factor": [ "1", "1" ] } ./octez-client rpc post /chains/main/mempool/filter with '{}' { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get /chains/main/mempool/filter { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=false' {} diff --git a/tezt/tests/expected/RPC_test.ml/Alpha- (mode proxy) RPC regression tests- mempool.out b/tezt/tests/expected/RPC_test.ml/Alpha- (mode proxy) RPC regression tests- mempool.out index 8e7d92dd98a829448c7f5e8c09a6c92521cd3db5..e2537102d1ebc3cd55db95e71ebc5ad3f420f777 100644 --- a/tezt/tests/expected/RPC_test.ml/Alpha- (mode proxy) RPC regression tests- mempool.out +++ b/tezt/tests/expected/RPC_test.ml/Alpha- (mode proxy) RPC regression tests- mempool.out @@ -288,15 +288,15 @@ curl -s 'http://localhost:[PORT]/chains/main/mempool/monitor_operations?applied= ./octez-client --mode proxy rpc get /chains/main/mempool/filter { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=false' {} @@ -311,70 +311,126 @@ curl -s 'http://localhost:[PORT]/chains/main/mempool/monitor_operations?applied= "56", "3" ], - "allow_script_failure": false + "replace_by_fee_factor": [ + "21", + "20" + ], + "max_operations": 0, + "max_total_bytes": 100000000 }' { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "56", "3" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client --mode proxy rpc get /chains/main/mempool/filter { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "56", "3" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "56", "3" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=false' { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false } + "minimal_nanotez_per_byte": [ "56", "3" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client --mode proxy rpc post /chains/main/mempool/filter with '{ "minimal_fees": "200", - "allow_script_failure": true + "replace_by_fee_factor": [ + "1", + "1" + ] +}' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client --mode proxy rpc get /chains/main/mempool/filter +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=true' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=false' +{ "minimal_fees": "200", "replace_by_fee_factor": [ "1", "1" ] } + +./octez-client --mode proxy rpc post /chains/main/mempool/filter with '{ + "max_operations": 100, + "invalid_field_name": 100 +}' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client --mode proxy rpc get /chains/main/mempool/filter +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=true' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=false' +{ "minimal_fees": "200", "replace_by_fee_factor": [ "1", "1" ] } + +./octez-client --mode proxy rpc post /chains/main/mempool/filter with '{ + "minimal_fees": "toto" }' { "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get /chains/main/mempool/filter { "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=false' -{ "minimal_fees": "200" } +{ "minimal_fees": "200", "replace_by_fee_factor": [ "1", "1" ] } ./octez-client --mode proxy rpc post /chains/main/mempool/filter with '{}' { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get /chains/main/mempool/filter { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=false' {} diff --git a/tezt/tests/expected/RPC_test.ml/Mumbai- (mode client) RPC regression tests- mempool.out b/tezt/tests/expected/RPC_test.ml/Mumbai- (mode client) RPC regression tests- mempool.out index d5652cefa436d337f9f71d2fa0a39a9fa062fc3f..82844fd01638be2f0c284a264162ee04feeac60b 100644 --- a/tezt/tests/expected/RPC_test.ml/Mumbai- (mode client) RPC regression tests- mempool.out +++ b/tezt/tests/expected/RPC_test.ml/Mumbai- (mode client) RPC regression tests- mempool.out @@ -288,15 +288,15 @@ curl -s 'http://localhost:[PORT]/chains/main/mempool/monitor_operations?applied= ./octez-client rpc get /chains/main/mempool/filter { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=false' {} @@ -311,70 +311,126 @@ curl -s 'http://localhost:[PORT]/chains/main/mempool/monitor_operations?applied= "56", "3" ], - "allow_script_failure": false + "replace_by_fee_factor": [ + "21", + "20" + ], + "max_operations": 0, + "max_total_bytes": 100000000 }' { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "56", "3" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client rpc get /chains/main/mempool/filter { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "56", "3" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "56", "3" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=false' { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false } + "minimal_nanotez_per_byte": [ "56", "3" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client rpc post /chains/main/mempool/filter with '{ "minimal_fees": "200", - "allow_script_failure": true + "replace_by_fee_factor": [ + "1", + "1" + ] +}' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client rpc get /chains/main/mempool/filter +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client rpc get '/chains/main/mempool/filter?include_default=true' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client rpc get '/chains/main/mempool/filter?include_default=false' +{ "minimal_fees": "200", "replace_by_fee_factor": [ "1", "1" ] } + +./octez-client rpc post /chains/main/mempool/filter with '{ + "max_operations": 100, + "invalid_field_name": 100 +}' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client rpc get /chains/main/mempool/filter +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client rpc get '/chains/main/mempool/filter?include_default=true' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client rpc get '/chains/main/mempool/filter?include_default=false' +{ "minimal_fees": "200", "replace_by_fee_factor": [ "1", "1" ] } + +./octez-client rpc post /chains/main/mempool/filter with '{ + "minimal_fees": "toto" }' { "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get /chains/main/mempool/filter { "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=false' -{ "minimal_fees": "200" } +{ "minimal_fees": "200", "replace_by_fee_factor": [ "1", "1" ] } ./octez-client rpc post /chains/main/mempool/filter with '{}' { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get /chains/main/mempool/filter { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=false' {} diff --git a/tezt/tests/expected/RPC_test.ml/Mumbai- (mode proxy) RPC regression tests- mempool.out b/tezt/tests/expected/RPC_test.ml/Mumbai- (mode proxy) RPC regression tests- mempool.out index 3eb9909c471fec5ac8ee6d1f0e30906542992c79..20534cf2e5fb0dbcb686ef19f7dd0ccb88724d49 100644 --- a/tezt/tests/expected/RPC_test.ml/Mumbai- (mode proxy) RPC regression tests- mempool.out +++ b/tezt/tests/expected/RPC_test.ml/Mumbai- (mode proxy) RPC regression tests- mempool.out @@ -288,15 +288,15 @@ curl -s 'http://localhost:[PORT]/chains/main/mempool/monitor_operations?applied= ./octez-client --mode proxy rpc get /chains/main/mempool/filter { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=false' {} @@ -311,70 +311,126 @@ curl -s 'http://localhost:[PORT]/chains/main/mempool/monitor_operations?applied= "56", "3" ], - "allow_script_failure": false + "replace_by_fee_factor": [ + "21", + "20" + ], + "max_operations": 0, + "max_total_bytes": 100000000 }' { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "56", "3" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client --mode proxy rpc get /chains/main/mempool/filter { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "56", "3" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "56", "3" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=false' { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false } + "minimal_nanotez_per_byte": [ "56", "3" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client --mode proxy rpc post /chains/main/mempool/filter with '{ "minimal_fees": "200", - "allow_script_failure": true + "replace_by_fee_factor": [ + "1", + "1" + ] +}' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client --mode proxy rpc get /chains/main/mempool/filter +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=true' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=false' +{ "minimal_fees": "200", "replace_by_fee_factor": [ "1", "1" ] } + +./octez-client --mode proxy rpc post /chains/main/mempool/filter with '{ + "max_operations": 100, + "invalid_field_name": 100 +}' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client --mode proxy rpc get /chains/main/mempool/filter +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=true' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=false' +{ "minimal_fees": "200", "replace_by_fee_factor": [ "1", "1" ] } + +./octez-client --mode proxy rpc post /chains/main/mempool/filter with '{ + "minimal_fees": "toto" }' { "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get /chains/main/mempool/filter { "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=false' -{ "minimal_fees": "200" } +{ "minimal_fees": "200", "replace_by_fee_factor": [ "1", "1" ] } ./octez-client --mode proxy rpc post /chains/main/mempool/filter with '{}' { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get /chains/main/mempool/filter { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=false' {} diff --git a/tezt/tests/expected/RPC_test.ml/Nairobi- (mode client) RPC regression tests- mempool.out b/tezt/tests/expected/RPC_test.ml/Nairobi- (mode client) RPC regression tests- mempool.out index 0c14b652f9daee00f718425bcb813402fcbe2709..d373c8d9a5a985b7bedd043fce44e3c0f04545bb 100644 --- a/tezt/tests/expected/RPC_test.ml/Nairobi- (mode client) RPC regression tests- mempool.out +++ b/tezt/tests/expected/RPC_test.ml/Nairobi- (mode client) RPC regression tests- mempool.out @@ -288,15 +288,15 @@ curl -s 'http://localhost:[PORT]/chains/main/mempool/monitor_operations?applied= ./octez-client rpc get /chains/main/mempool/filter { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=false' {} @@ -311,70 +311,126 @@ curl -s 'http://localhost:[PORT]/chains/main/mempool/monitor_operations?applied= "56", "3" ], - "allow_script_failure": false + "replace_by_fee_factor": [ + "21", + "20" + ], + "max_operations": 0, + "max_total_bytes": 100000000 }' { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "56", "3" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client rpc get /chains/main/mempool/filter { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "56", "3" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "56", "3" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=false' { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false } + "minimal_nanotez_per_byte": [ "56", "3" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client rpc post /chains/main/mempool/filter with '{ "minimal_fees": "200", - "allow_script_failure": true + "replace_by_fee_factor": [ + "1", + "1" + ] +}' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client rpc get /chains/main/mempool/filter +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client rpc get '/chains/main/mempool/filter?include_default=true' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client rpc get '/chains/main/mempool/filter?include_default=false' +{ "minimal_fees": "200", "replace_by_fee_factor": [ "1", "1" ] } + +./octez-client rpc post /chains/main/mempool/filter with '{ + "max_operations": 100, + "invalid_field_name": 100 +}' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client rpc get /chains/main/mempool/filter +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client rpc get '/chains/main/mempool/filter?include_default=true' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client rpc get '/chains/main/mempool/filter?include_default=false' +{ "minimal_fees": "200", "replace_by_fee_factor": [ "1", "1" ] } + +./octez-client rpc post /chains/main/mempool/filter with '{ + "minimal_fees": "toto" }' { "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get /chains/main/mempool/filter { "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=false' -{ "minimal_fees": "200" } +{ "minimal_fees": "200", "replace_by_fee_factor": [ "1", "1" ] } ./octez-client rpc post /chains/main/mempool/filter with '{}' { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get /chains/main/mempool/filter { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client rpc get '/chains/main/mempool/filter?include_default=false' {} diff --git a/tezt/tests/expected/RPC_test.ml/Nairobi- (mode proxy) RPC regression tests- mempool.out b/tezt/tests/expected/RPC_test.ml/Nairobi- (mode proxy) RPC regression tests- mempool.out index bf056f59a0d36cac46dcd26976dd5f21e73791bf..e476804a158aba3397ebe4e8f2ef1a707789e824 100644 --- a/tezt/tests/expected/RPC_test.ml/Nairobi- (mode proxy) RPC regression tests- mempool.out +++ b/tezt/tests/expected/RPC_test.ml/Nairobi- (mode proxy) RPC regression tests- mempool.out @@ -288,15 +288,15 @@ curl -s 'http://localhost:[PORT]/chains/main/mempool/monitor_operations?applied= ./octez-client --mode proxy rpc get /chains/main/mempool/filter { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=false' {} @@ -311,70 +311,126 @@ curl -s 'http://localhost:[PORT]/chains/main/mempool/monitor_operations?applied= "56", "3" ], - "allow_script_failure": false + "replace_by_fee_factor": [ + "21", + "20" + ], + "max_operations": 0, + "max_total_bytes": 100000000 }' { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "56", "3" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client --mode proxy rpc get /chains/main/mempool/filter { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "56", "3" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "56", "3" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=false' { "minimal_fees": "50", "minimal_nanotez_per_gas_unit": [ "201", "5" ], - "minimal_nanotez_per_byte": [ "56", "3" ], "allow_script_failure": false } + "minimal_nanotez_per_byte": [ "56", "3" ], "max_operations": 0, + "max_total_bytes": 100000000 } ./octez-client --mode proxy rpc post /chains/main/mempool/filter with '{ "minimal_fees": "200", - "allow_script_failure": true + "replace_by_fee_factor": [ + "1", + "1" + ] +}' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client --mode proxy rpc get /chains/main/mempool/filter +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=true' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=false' +{ "minimal_fees": "200", "replace_by_fee_factor": [ "1", "1" ] } + +./octez-client --mode proxy rpc post /chains/main/mempool/filter with '{ + "max_operations": 100, + "invalid_field_name": 100 +}' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client --mode proxy rpc get /chains/main/mempool/filter +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=true' +{ "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } + +./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=false' +{ "minimal_fees": "200", "replace_by_fee_factor": [ "1", "1" ] } + +./octez-client --mode proxy rpc post /chains/main/mempool/filter with '{ + "minimal_fees": "toto" }' { "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get /chains/main/mempool/filter { "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "200", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "1", "1" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=false' -{ "minimal_fees": "200" } +{ "minimal_fees": "200", "replace_by_fee_factor": [ "1", "1" ] } ./octez-client --mode proxy rpc post /chains/main/mempool/filter with '{}' { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get /chains/main/mempool/filter { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=true' { "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "100", "1" ], - "minimal_nanotez_per_byte": [ "1000", "1" ], "allow_script_failure": true, - "replace_by_fee_factor": [ "21", "20" ], - "max_prechecked_manager_operations": 5000 } + "minimal_nanotez_per_byte": [ "1000", "1" ], + "replace_by_fee_factor": [ "21", "20" ], "max_operations": 10000, + "max_total_bytes": 10000000 } ./octez-client --mode proxy rpc get '/chains/main/mempool/filter?include_default=false' {} diff --git a/tezt/tests/prevalidator.ml b/tezt/tests/prevalidator.ml index 2967a52bbffc174b78a23ff574746d30f521b0b8..bd1b57ee30c5c88a15c6d5de929c53fd58ee5724 100644 --- a/tezt/tests/prevalidator.ml +++ b/tezt/tests/prevalidator.ml @@ -72,19 +72,6 @@ module Revamped = struct let* () = flush_waiter in Node.wait_for_level node (level + 1) - (** Calls RPC [POST /chains/main/mempool/filter] from [client], with [data] - formatted from string [config_str]. If [log] is [true], also logs this - string. *) - let set_filter ?(log = false) config_str client = - let* res = - RPC.Client.call client - @@ RPC.post_chain_mempool_filter - ~data:(Data (Ezjsonm.from_string config_str)) - () - in - if log then Log.info "Updated filter config with: %s." config_str ; - return res - (* Wait for the [operation_to_reclassify] event from the prevalidator and return the number of operations that were set to be reclassified. *) let wait_for_operations_not_flushed_event node = @@ -1553,19 +1540,21 @@ module Revamped = struct let* () = check_mempool ~applied:[oph4; oph3; oph2; oph1] client1 in check_mempool ~applied:[oph4; oph3; oph2; oph1] client2 - let test_prefiltered_limit = + let test_full_mempool = Protocol.register_test ~__FILE__ - ~title:"Test prefiltered limits of mempool" - ~tags:["mempool"; "gc"; "limit"] + ~title:"test full mempool" + ~tags:["mempool"; "gc"; "limit"; "bounding"; "full"] @@ fun protocol -> - log_step 0 "Connect and initialise two nodes." ; - (* We configure the filter with a limit of 4, in order to easily be able to - inject more with our 5 bootstrap accounts *) - let max_prechecked_manager_operations = 4 in + Log.info "Test the bound on operation count in the mempool." ; + (* We configure the filter with a limit of 4 operations, so that + we can easily inject more with our 5 bootstrap accounts. *) + let max_operations = 4 in (* Control fees and gas limits to easily influence weight (i.e. ratio) *) let fee = 1000 in let gas_limit = 1500 in + + log_step 0 "Initialize and connect two nodes." ; let* node1 = Node.init ~event_sections_levels:[("prevalidator", `Debug)] @@ -1581,28 +1570,12 @@ module Revamped = struct log_step 1 - "Update the nodes filter to allow only %d prechecked manager operations." - max_prechecked_manager_operations ; - let* _ = - set_filter - ~log:true - (sf - {|{ "max_prechecked_manager_operations" : %d }|} - max_prechecked_manager_operations) - client1 - and* _ = - set_filter - ~log:true - (sf - {|{ "max_prechecked_manager_operations" : %d }|} - max_prechecked_manager_operations) - client2 - in + "Update the mempool filter to allow at most %d valid operations." + max_operations ; + let* () = Mempool.Config.set_filter ~log:true ~max_operations client1 + and* () = Mempool.Config.set_filter ~log:true ~max_operations client2 in - log_step - 2 - "Inject max_prechecked_manager_operations %d operations." - max_prechecked_manager_operations ; + log_step 2 "Inject max_operations = %d operations." max_operations ; let* ops = Lwt.all @@ List.mapi @@ -1628,31 +1601,19 @@ module Revamped = struct 4 "The client should report when the mempool is full and not enough fees \ are provided." ; - let transfer_should_fail = - Client.spawn_transfer - ~giver:Constant.bootstrap5.alias - ~receiver:Constant.bootstrap2.alias - ~amount:(Tez.of_int 55) - ~fee:(Tez.of_mutez_int fee) - ~gas_limit - client1 - in - let* std_err = - Process.check_and_read_stderr ~expect_failure:true transfer_should_fail - in - (match std_err =~* rex "Increase operation fees to at least (.*)tz" with - | None -> - Test.fail - ~__LOC__ - "The client should fail when the mempool is full and not enough fees \ - are provided." - | Some required -> - Check.( - (required = "0.001001") - string - ~error_msg:"The required fees are %L but expected %R")) ; - - log_step 5 "Inject an extra operation with same fees (but mempool is full)." ; + let* () = + Process.check_error + ~msg:Constant.Error_msg.rejected_by_full_mempool + (Client.spawn_transfer + ~giver:Constant.bootstrap5.alias + ~receiver:Constant.bootstrap2.alias + ~amount:(Tez.of_int 55) + ~fee:(Tez.of_mutez_int (fee - 1)) + ~gas_limit + client1) + in + + log_step 5 "Force inject an extra operation with not enough fees." ; let* (`OpHash oph5) = Operation.inject_transfer ~force:true @@ -1660,7 +1621,7 @@ module Revamped = struct ~dest:Constant.bootstrap2 ~wait_for_injection:node1 ~amount:1 - ~fee + ~fee:(fee - 1) ~gas_limit client1 in @@ -1736,18 +1697,21 @@ module Revamped = struct log_step 12 "Check mempool after flush." ; check_mempool ~branch_refused:[oph5] client1 - let test_prefiltered_limit_remove = + let test_full_mempool_and_replace_same_manager = Protocol.register_test ~__FILE__ - ~title:"Test prefiltered limits of mempool after a remove/replace" - ~tags:["mempool"; "gc"; "limit"; "replace"; "remove"] + ~title:"test full mempool and replace same manager" + ~tags:["mempool"; "gc"; "limit"; "bounding"; "full"; "replace"; "remove"] @@ fun protocol -> - log_step 0 "Connect and initialise two nodes." ; + Log.info + "Test the interaction between the mempool operation bound and \ + same-manager replace-by-fees." ; (* We configure the filter with a limit of 1 *) - let max_prechecked_manager_operations = 1 in + let max_operations = 1 in (* Control fees and gas limits to easily influence weight (i.e. ratio) *) let fee = 1000 in let gas_limit = 1500 in + log_step 0 "Initialize and connect two nodes." ; let* node1 = Node.init ~event_sections_levels:[("prevalidator", `Debug)] @@ -1763,23 +1727,10 @@ module Revamped = struct log_step 1 - "Update the nodes filter to allow only %d prechecked manager operations." - max_prechecked_manager_operations ; - let* _ = - set_filter - ~log:true - (sf - {|{ "max_prechecked_manager_operations" : %d }|} - max_prechecked_manager_operations) - client1 - and* _ = - set_filter - ~log:true - (sf - {|{ "max_prechecked_manager_operations" : %d }|} - max_prechecked_manager_operations) - client2 - in + "Update the mempool filter to allow at most %d valid operations." + max_operations ; + let* () = Mempool.Config.set_filter ~log:true ~max_operations client1 + and* () = Mempool.Config.set_filter ~log:true ~max_operations client2 in log_step 2 "Inject an operation" ; let* (`OpHash oph1) = @@ -2939,8 +2890,10 @@ let wait_for_arrival_of_ophash ophash node = (** [set_filter_no_fee_requirement client] sets all fields [minimal_*] to 0 in the filter configuration of [client]'s mempool. *) let set_filter_no_fee_requirement = - Revamped.set_filter - {|{ "minimal_fees": "0", "minimal_nanotez_per_gas_unit": [ "0", "1" ], "minimal_nanotez_per_byte": [ "0", "1" ] }|} + Mempool.Config.set_filter + ~minimal_fees:0 + ~minimal_nanotez_per_gas_unit:(0, 1) + ~minimal_nanotez_per_byte:(0, 1) (** Checks that arguments [applied] and [refused] are the number of operations in the mempool of [client] with the corresponding classification, @@ -3455,181 +3408,6 @@ let injecting_old_operation_fails = in Process.check_error ~msg:injection_error_rex process -(** Mempool filter configuration. *) -module Filter_config = struct - type t = { - minimal_fees : int option; - minimal_nanotez_per_gas_unit : (int * int) option; - minimal_nanotez_per_byte : (int * int) option; - allow_script_failure : bool option; - } - - let eq_int_pair (f1, s1) (f2, s2) = Int.equal f1 f2 && Int.equal s1 s2 - - let equal - { - minimal_fees = mf1; - minimal_nanotez_per_gas_unit = mng1; - minimal_nanotez_per_byte = mnb1; - allow_script_failure = asf1; - } - { - minimal_fees = mf2; - minimal_nanotez_per_gas_unit = mng2; - minimal_nanotez_per_byte = mnb2; - allow_script_failure = asf2; - } = - Option.equal Int.equal mf1 mf2 - && Option.equal eq_int_pair mng1 mng2 - && Option.equal eq_int_pair mnb1 mnb2 - && Option.equal Bool.equal asf1 asf2 - - let pp fmt - { - minimal_fees = mf; - minimal_nanotez_per_gas_unit = mng; - minimal_nanotez_per_byte = mnb; - allow_script_failure = asf; - } = - [ - Option.map (sf {|"minimal_fees": "%d"|}) mf; - Option.map - (fun (n1, n2) -> - sf {|"minimal_nanotez_per_gas_unit": [ "%d", "%d" ]|} n1 n2) - mng; - Option.map - (fun (n1, n2) -> - sf {|"minimal_nanotez_per_byte": [ "%d", "%d" ]|} n1 n2) - mnb; - Option.map (sf {|"allow_script_failure": %b|}) asf; - ] - |> List.map Option.to_list |> List.flatten |> String.concat ", " - |> Format.fprintf fmt {|{ %s }|} - - let show : t -> string = Format.asprintf "%a" pp - - let check_equal expected actual = - Check.( - (expected = actual) - (equalable pp equal) - ~error_msg:"Wrong filter configuration: %R.@.Expected: %L.") - - (** Returns the filter configuration corresponding to [json]. If any field - of {!filter_config} is missing from [json], it is set to the default - value (i.e. the corresponding value in {!default_config}. *) - let of_json json = - let open JSON in - let as_int_pair_opt t = - match as_list_opt t with - | Some [x; y] -> Some (as_int x, as_int y) - (* A missing field is interpreted as [`Null], from which [as_list_opt] - produces [Some []]. *) - | Some [] -> None - | Some _ | None -> - Test.fail - "Constructing a filter_config from json: %s. Expected a list of \ - length 2, found: %s." - (encode json) - (encode t) - in - { - minimal_fees = json |-> "minimal_fees" |> as_int_opt; - minimal_nanotez_per_gas_unit = - json |-> "minimal_nanotez_per_gas_unit" |> as_int_pair_opt; - minimal_nanotez_per_byte = - json |-> "minimal_nanotez_per_byte" |> as_int_pair_opt; - allow_script_failure = json |-> "allow_script_failure" |> as_bool_opt; - } - - (** Default filter configuration for protocol alpha - (in proto_alpha/lib_plugin/plugin.ml). *) - - let default_minimal_fees = 100 - - let default_minimal_nanotez_per_gas_unit = (100, 1) - - let default_minimal_nanotez_per_byte = (1000, 1) - - let default_allow_script_failure = true - - let default = - { - minimal_fees = Some default_minimal_fees; - minimal_nanotez_per_gas_unit = Some default_minimal_nanotez_per_gas_unit; - minimal_nanotez_per_byte = Some default_minimal_nanotez_per_byte; - allow_script_failure = Some default_allow_script_failure; - } - - (** Returns a copy of the given filter config, where missing fields - (i.e. containing [None]) have been set to their default value. *) - let fill_with_default - { - minimal_fees = mf; - minimal_nanotez_per_gas_unit = mng; - minimal_nanotez_per_byte = mnb; - allow_script_failure = asf; - } = - Option. - { - minimal_fees = Some (value mf ~default:default_minimal_fees); - minimal_nanotez_per_gas_unit = - Some (value mng ~default:default_minimal_nanotez_per_gas_unit); - minimal_nanotez_per_byte = - Some (value mnb ~default:default_minimal_nanotez_per_byte); - allow_script_failure = - Some (value asf ~default:default_allow_script_failure); - } - - (** Returns a copy of the given filter config, where fields equal - to their default value have been removed (i.e. set to [None]). *) - let clear_default - { - minimal_fees = mf; - minimal_nanotez_per_gas_unit = mng; - minimal_nanotez_per_byte = mnb; - allow_script_failure = asf; - } = - let clear_if_default eq_fun default = function - | Some x when eq_fun default x -> None - | x -> x - in - { - minimal_fees = clear_if_default Int.equal default_minimal_fees mf; - minimal_nanotez_per_gas_unit = - clear_if_default eq_int_pair default_minimal_nanotez_per_gas_unit mng; - minimal_nanotez_per_byte = - clear_if_default eq_int_pair default_minimal_nanotez_per_byte mnb; - allow_script_failure = - clear_if_default Bool.equal default_allow_script_failure asf; - } - - (** Checks that RPC [GET /chains/main/mempool/filter] returns the - appropriate result for [expected_config], testing all possibilities - for optional argument [include_default] (omitted/[true]/[false]). *) - let check_RPC_GET_all_variations ?(log = false) expected_config client = - let expected_full = fill_with_default expected_config in - let* json = RPC.Client.call client @@ RPC.get_chain_mempool_filter () in - check_equal expected_full (of_json json) ; - let* json = - RPC.Client.call client - @@ RPC.get_chain_mempool_filter ~include_default:true () - in - check_equal expected_full (of_json json) ; - let expected_partial = clear_default expected_config in - let* json = - RPC.Client.call client - @@ RPC.get_chain_mempool_filter ~include_default:false () - in - check_equal expected_partial (of_json json) ; - if log then - Log.info - "GET /chains/main/mempool/filter returned expected configurations \ - (respectively including/excluding default fields): %s and %s." - (show expected_full) - (show expected_partial) ; - unit -end - (* Probably to be replaced during upcoming mempool tests refactoring *) let init_single_node_and_activate_protocol ?(arguments = Node.[Synchronisation_threshold 0; Connections 0]) @@ -3685,7 +3463,7 @@ let test_get_post_mempool_filter = value." in Protocol.register_test ~__FILE__ ~title ~tags @@ fun protocol -> - let open Filter_config in + let open Mempool.Config in log_step 1 step1_msg ; let* node1, client1 = (* We need event level [debug] for event @@ -3695,13 +3473,13 @@ let test_get_post_mempool_filter = protocol in log_step 2 step2_msg ; - let* () = check_RPC_GET_all_variations ~log:true default client1 in + let* () = check_get_filter_all_variations ~log:true default client1 in log_step 3 step3_msg ; let set_config_and_check msg config = Log.info "%s" msg ; - let* output = Revamped.set_filter ~log:true (show config) client1 in + let* output = post_filter ~log:true config client1 in check_equal (fill_with_default config) (of_json output) ; - check_RPC_GET_all_variations ~log:true config client1 + check_get_filter_all_variations ~log:true config client1 in let* () = set_config_and_check @@ -3711,7 +3489,9 @@ let test_get_post_mempool_filter = minimal_fees = Some 25; minimal_nanotez_per_gas_unit = None; minimal_nanotez_per_byte = Some (1050, 1); - allow_script_failure = Some false; + replace_by_fee_factor = None; + max_operations = Some 0; + max_total_bytes = Some 11; } in let* () = @@ -3721,7 +3501,9 @@ let test_get_post_mempool_filter = minimal_fees = Some 1; minimal_nanotez_per_gas_unit = Some (2, 3); minimal_nanotez_per_byte = Some (4, 5); - allow_script_failure = Some false; + replace_by_fee_factor = Some (6, 7); + max_operations = Some 8; + max_total_bytes = Some 9; } in let config3 = @@ -3729,7 +3511,9 @@ let test_get_post_mempool_filter = minimal_fees = None; minimal_nanotez_per_gas_unit = Some default_minimal_nanotez_per_gas_unit; minimal_nanotez_per_byte = Some (4, 2); - allow_script_failure = Some default_allow_script_failure; + replace_by_fee_factor = None; + max_operations = Some default_max_operations; + max_total_bytes = Some 10_000_000; } in let* () = @@ -3747,7 +3531,7 @@ let test_get_post_mempool_filter = "invalid_mempool_filter_configuration.v0" (Fun.const (Some ())) in - let* output = Revamped.set_filter invalid_config_str client1 in + let* output = post_filter_str invalid_config_str client1 in check_equal config3_full (of_json output) ; let* () = waiter in let* output = RPC.Client.call client1 @@ RPC.get_chain_mempool_filter () in @@ -3759,16 +3543,20 @@ let test_get_post_mempool_filter = Tezos_base__TzPervasives.List.iter_s test_invalid_config [ - {|{ "minimal_fees": "100", "minimal_nanotez_per_byte": [ "1050", "1" ], "allow_script_failure": false, "invalid_field_name": 0 }|}; + (* invalid field name *) + {|{ "minimal_fees": "100", "minimal_nanotez_per_gas_unit": [ "1050", "1" ], "minimal_nanotez_per_byte": [ "7", "5" ], "replace_by_fee_factor": ["21", "20"], "max_operations": 0, "max_total_bytes": 10_000_000, "invalid_field_name": 0 }|}; + (* wrong type *) {|{ "minimal_fees": true}|}; + (* not enough elements in a pair *) {|{ "minimal_nanotez_per_gas_unit": [ "100" ]}|}; + (* too many elements in a pair *) {|{ "minimal_nanotez_per_gas_unit": [ "100", "1", "10" ]}|}; ] in log_step 5 step5_msg ; - let* output = Revamped.set_filter ~log:true "{}" client1 in + let* output = post_filter_str ~log:true "{}" client1 in check_equal default (of_json output) ; - check_RPC_GET_all_variations ~log:true default client1 + check_get_filter_all_variations ~log:true default client1 (** Similar to [Node_event_level.transfer_and_wait_for_injection] but more general. Should be merged with it during upcoming mempool tests refactoring. *) @@ -3969,9 +3757,10 @@ let test_mempool_filter_operation_arrival = in log_step 5 step5 ; let* _ = - Revamped.set_filter + Mempool.Config.set_filter ~log:true - {|{ "minimal_nanotez_per_gas_unit": [ "0", "1" ], "minimal_nanotez_per_byte": [ "0", "1" ] }|} + ~minimal_nanotez_per_gas_unit:(0, 1) + ~minimal_nanotez_per_byte:(0, 1) client1 in let waiterB = wait_for_arrival node1 in @@ -3990,8 +3779,10 @@ let test_mempool_filter_operation_arrival = in log_step 7 step7 ; let* _ = - Revamped.set_filter - {|{ "minimal_fees": "10", "minimal_nanotez_per_gas_unit": [ "0", "1" ], "minimal_nanotez_per_byte": [ "0", "1" ] }|} + Mempool.Config.set_filter + ~minimal_fees:10 + ~minimal_nanotez_per_gas_unit:(0, 1) + ~minimal_nanotez_per_byte:(0, 1) client1 in let waiterC = wait_for_arrival node1 in @@ -4085,8 +3876,8 @@ let register ~protocols = Revamped.ban_operation protocols ; Revamped.unban_operation_and_reinject protocols ; Revamped.unban_all_operations protocols ; - Revamped.test_prefiltered_limit protocols ; - Revamped.test_prefiltered_limit_remove protocols ; + Revamped.test_full_mempool protocols ; + Revamped.test_full_mempool_and_replace_same_manager protocols ; Revamped.precheck_with_empty_balance protocols ; Revamped.inject_operations protocols ; Revamped.test_inject_manager_batch protocols ;