From 4d545cddb13897aad70161a96d534d64da7239b5 Mon Sep 17 00:00:00 2001 From: Victor Dumitrescu Date: Fri, 7 Jul 2023 11:53:34 +0200 Subject: [PATCH 01/10] Manifest: rename octez_injector to octez_injector_lib --- manifest/main.ml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/manifest/main.ml b/manifest/main.ml index 1f10929c67f8..36e591330358 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -4315,7 +4315,7 @@ let octez_crawler = octez_shell; ] -let octez_injector = +let octez_injector_lib = public_lib "octez-injector" ~path:"src/lib_injector" @@ -4369,7 +4369,7 @@ let octez_smart_rollup_node_lib = octez_dal_node_lib |> open_; octez_dac_lib |> open_; octez_dac_client_lib |> open_; - octez_injector |> open_; + octez_injector_lib |> open_; octez_version_value |> open_; octez_layer2_store |> open_; octez_crawler |> open_; @@ -6346,7 +6346,7 @@ let hash = Protocol.hash [ octez_base |> open_ ~m:"TzPervasives"; main |> open_; - octez_injector |> open_; + octez_injector_lib |> open_; octez_smart_rollup_lib |> open_; ] ~inline_tests:ppx_expect @@ -6396,7 +6396,7 @@ let hash = Protocol.hash irmin; aches; aches_lwt; - octez_injector |> open_; + octez_injector_lib |> open_; octez_smart_rollup_node_lib |> open_; octez_scoru_wasm; octez_scoru_wasm_fast; -- GitLab From c6ce610413eb21d4a25957e9135d5e428752cca8 Mon Sep 17 00:00:00 2001 From: Victor Dumitrescu Date: Fri, 7 Jul 2023 12:00:53 +0200 Subject: [PATCH 02/10] Lib_injector: instantiate injector server --- src/lib_injector/injector_server.ml | 108 ++++++++++++++++++ src/lib_injector/injector_server.mli | 44 +++++++ src/lib_injector/injector_server_operation.ml | 64 +++++++++++ .../injector_server_operation.mli | 26 +++++ 4 files changed, 242 insertions(+) create mode 100644 src/lib_injector/injector_server.ml create mode 100644 src/lib_injector/injector_server.mli create mode 100644 src/lib_injector/injector_server_operation.ml create mode 100644 src/lib_injector/injector_server_operation.mli diff --git a/src/lib_injector/injector_server.ml b/src/lib_injector/injector_server.ml new file mode 100644 index 000000000000..190417407b14 --- /dev/null +++ b/src/lib_injector/injector_server.ml @@ -0,0 +1,108 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Functori, *) +(* Copyright (c) 2023 Nomadic Labs *) +(* *) +(*****************************************************************************) + +module Configuration = struct + type tag = Transaction + + type fee_parameters = Injector_sigs.fee_parameter + + let tags = [Transaction] + + let string_of_purpose = function Transaction -> "Transaction" + + let default_fee_parameters = + { + Injector_sigs.minimal_fees = {Injector_sigs.mutez = 100L}; + minimal_nanotez_per_byte = Q.of_int 1000; + minimal_nanotez_per_gas_unit = Q.of_int 100; + force_low_fee = false; + fee_cap = {Injector_sigs.mutez = 1_000_000L}; + burn_cap = {Injector_sigs.mutez = 1_000_000L}; + } +end + +(* The rest of this module is adapted from + [lib_smart_rollup_node/injector.ml] *) + +type state = { + cctxt : Client_context.full; + fee_parameters : Configuration.fee_parameters; + minimal_block_delay : int64; + delay_increment_per_round : int64; +} + +open Injector_sigs + +module Parameters : + PARAMETERS + with type state = state + and type Tag.t = Configuration.tag + and type Operation.t = Injector_server_operation.t = struct + type nonrec state = state + + let events_section = ["injector"; "server"] + + module Tag : TAG with type t = Configuration.tag = struct + type t = Configuration.tag + + let compare = Stdlib.compare + + let equal = Stdlib.( = ) + + let hash = Hashtbl.hash + + let string_of_tag = Configuration.string_of_purpose + + let pp ppf t = Format.pp_print_string ppf (string_of_tag t) + + let encoding : t Data_encoding.t = + let open Data_encoding in + match Configuration.tags with + (* first case can be removed once we have multiple tags *) + | [tag] -> conv string_of_tag (fun _ -> tag) string + | tags -> string_enum (List.map (fun t -> (string_of_tag t, t)) tags) + end + + module Operation = Injector_server_operation + + (* Very coarse approximation for the number of operation we + expect for each block *) + let table_estimated_size : Tag.t -> int = function Transaction -> 100 + + let operation_tag : Operation.t -> Tag.t = function + | Transaction _ -> Transaction + + (* TODO: https://gitlab.com/tezos/tezos/-/issues/6281 + revise if multiple operation kinds have different fee parameter + structures *) + let fee_parameter {fee_parameters; _} _ = fee_parameters + + (* TODO: https://gitlab.com/tezos/tezos/-/issues/3459 + Decide if some batches must have all the operations succeed. See + {!Injector_sigs.Parameter.batch_must_succeed}. *) + let batch_must_succeed _ = `At_least_one + + let retry_unsuccessful_operation _node_ctxt (_op : Operation.t) status = + let open Lwt_syntax in + match status with + | Backtracked | Skipped | Other_branch -> + (* Always retry backtracked or skipped operations, or operations that + are on another branch because of a reorg. *) + return Retry + | Failed error -> ( + (* TODO: https://gitlab.com/tezos/tezos/-/issues/4071 + Think about which operations should be retried and when. *) + match classify_trace error with + | Permanent | Outdated -> return Forget + | Branch | Temporary -> return Retry) + | Never_included -> + (* Forget operations that are never included *) + return Forget +end + +include Injector_functor.Make (Parameters) diff --git a/src/lib_injector/injector_server.mli b/src/lib_injector/injector_server.mli new file mode 100644 index 000000000000..6835cafef8dc --- /dev/null +++ b/src/lib_injector/injector_server.mli @@ -0,0 +1,44 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Nomadic Labs *) +(* *) +(*****************************************************************************) + +(** This module defines an instance of the injector functor used in the + standalone injector server binary in [bin_injector_server], which is + independent of the rollup node and is meant to inject manager operations + (only transactions are currently supported). + + Other modules in this library which are only used for the injector binary + are prefixed with [injector_server*]. The injector server supports + multiple protocols. Each protocol is supported using a plugin module + [proto_*/lib_injector/injector_plugin.ml] *) + +module Configuration : sig + type tag = Transaction + + type fee_parameters = Injector_sigs.fee_parameter + + val tags : tag trace + + val string_of_purpose : tag -> string + + val default_fee_parameters : Injector_sigs.fee_parameter +end + +type state = { + cctxt : Client_context.full; + fee_parameters : Configuration.fee_parameters; + (** [minimal_block_delay] and [delay_increment_per_round] are protocol + constants required for the injector plugin to compute the time remaining + until the following block *) + minimal_block_delay : int64; + delay_increment_per_round : int64; +} + +include + Injector_sigs.S + with type state := state + and type tag := Configuration.tag + and type operation := Injector_server_operation.t diff --git a/src/lib_injector/injector_server_operation.ml b/src/lib_injector/injector_server_operation.ml new file mode 100644 index 000000000000..33160896fd67 --- /dev/null +++ b/src/lib_injector/injector_server_operation.ml @@ -0,0 +1,64 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Nomadic Labs *) +(* *) +(*****************************************************************************) + +type parameters = {entrypoint : string; value : string} + +type t = + | Transaction of { + amount : int64; + destination : string; + parameters : parameters option; + } + +(* adapted from Operation_repr.Encoding.Manager_operations *) +let encoding : t Data_encoding.t = + let open Data_encoding in + let case tag kind encoding proj inj = + case + ~title:kind + (Tag tag) + (merge_objs (obj1 (req "kind" (constant kind))) encoding) + (fun o -> Option.map (fun p -> ((), p)) (proj o)) + (fun ((), p) -> inj p) + in + def "injector_operation" + @@ union + [ + case + 0 + "transaction" + (obj3 + (req "amount" string) + (req "destination" string) + (opt + "parameters" + (obj2 (req "entrypoint" string) (req "value" string)))) + (function + | Transaction {amount; destination; parameters} -> + Some + ( Int64.to_string amount, + destination, + Option.map + (fun {entrypoint; value} -> (entrypoint, value)) + parameters )) + (fun (amount, destination, parameters) -> + Transaction + { + amount = Int64.of_string amount; + destination; + parameters = + Option.map + (fun (entrypoint, value) -> {entrypoint; value}) + parameters; + }); + ] + +let pp ppf = function + | Transaction {amount; destination = _; parameters = _} -> + Format.fprintf ppf "Transaction of %Ld tez" amount + +let unique = function Transaction _ -> true diff --git a/src/lib_injector/injector_server_operation.mli b/src/lib_injector/injector_server_operation.mli new file mode 100644 index 000000000000..ccaa602d4b24 --- /dev/null +++ b/src/lib_injector/injector_server_operation.mli @@ -0,0 +1,26 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Nomadic Labs *) +(* *) +(*****************************************************************************) + +(** Parameters for smart contract calls *) +type parameters = {entrypoint : string; value : string} + +(** Operations which can be processed by the injector *) +type t = + | Transaction of { + amount : int64; + destination : string; + parameters : parameters option; + } + +(** Encoding for operations (used by injector for on-disk persistence) *) +val encoding : t Data_encoding.t + +(** Pretty printer (human readable) for operations *) +val pp : Format.formatter -> t -> unit + +(** [false] if the injector will accept duplicate such operations. *) +val unique : t -> bool -- GitLab From 5999913853967101767c8d9591c5a934c73a79f3 Mon Sep 17 00:00:00 2001 From: Victor Dumitrescu Date: Fri, 7 Jul 2023 11:56:47 +0200 Subject: [PATCH 03/10] Manifest: add octez-injector-alpha --- .gitlab/ci/jobs/packaging/opam_package.yml | 2 ++ dune-project | 1 + manifest/main.ml | 30 ++++++++++++++++++++-- opam/tezos-injector-alpha.opam | 24 +++++++++++++++++ src/proto_alpha/lib_injector/dune | 23 +++++++++++++++++ 5 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 opam/tezos-injector-alpha.opam create mode 100644 src/proto_alpha/lib_injector/dune diff --git a/.gitlab/ci/jobs/packaging/opam_package.yml b/.gitlab/ci/jobs/packaging/opam_package.yml index 0f1b1ebc23aa..e385632b337a 100644 --- a/.gitlab/ci/jobs/packaging/opam_package.yml +++ b/.gitlab/ci/jobs/packaging/opam_package.yml @@ -876,6 +876,8 @@ opam:tezos-dal-node-services: variables: package: tezos-dal-node-services +# Ignoring unreleased package tezos-injector-alpha. + # Ignoring unreleased package tezos-lazy-containers-tests. # Ignoring unreleased package tezos-micheline-rewriting. diff --git a/dune-project b/dune-project index 3f5663380212..acd4397ba4db 100644 --- a/dune-project +++ b/dune-project @@ -94,6 +94,7 @@ (package (name tezos-dac-node-lib-test)(allow_empty)) (package (name tezos-dal-node-lib)) (package (name tezos-dal-node-services)) +(package (name tezos-injector-alpha)(allow_empty)) (package (name tezos-lazy-containers-tests)(allow_empty)) (package (name tezos-micheline-rewriting)) (package (name tezos-openapi)) diff --git a/manifest/main.ml b/manifest/main.ml index 36e591330358..14cd06626673 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -4594,6 +4594,8 @@ module Protocol : sig val octez_sc_rollup_node : t -> target option + val octez_injector : t -> target option + val baking_exn : t -> target val genesis : t @@ -4712,13 +4714,14 @@ end = struct octez_sc_rollup : target option; octez_sc_rollup_layer2 : target option; octez_sc_rollup_node : target option; + octez_injector : target option; } let make ?client ?client_commands ?client_commands_registration ?baking_commands_registration ?plugin ?plugin_registerer ?dal ?dac ?test_helpers ?parameters ?benchmarks_proto ?octez_sc_rollup - ?octez_sc_rollup_layer2 ?octez_sc_rollup_node ?baking ~status ~name ~main - ~embedded () = + ?octez_sc_rollup_layer2 ?octez_sc_rollup_node ?octez_injector ?baking + ~status ~name ~main ~embedded () = { status; name; @@ -4739,6 +4742,7 @@ end = struct octez_sc_rollup; octez_sc_rollup_layer2; octez_sc_rollup_node; + octez_injector; } let all_rev : t list ref = ref [] @@ -4803,6 +4807,8 @@ end = struct let octez_sc_rollup_node p = p.octez_sc_rollup_node + let octez_injector p = p.octez_injector + (* N as in "protocol number in the Alpha family". *) module N = struct (* This function is asymmetrical on purpose: we don't want to compare @@ -6335,6 +6341,25 @@ let hash = Protocol.hash alcotezt; ] in + let octez_injector = + only_if N.(number >= 019) @@ fun () -> + private_lib + (sf "octez_injector_%s" short_hash) + ~path:(path // "lib_injector") + ~synopsis: + "Tezos/Protocol: protocol-specific library for the injector binary" + ~opam:(sf "tezos-injector-%s" name_dash) + ~deps: + [ + octez_base |> open_ ~m:"TzPervasives"; + main |> open_; + octez_injector_lib |> open_; + client |> if_some |> open_; + octez_client_base |> open_; + plugin |> if_some |> open_; + ] + ~linkall:true + in let octez_sc_rollup_layer2 = only_if N.(number >= 016) @@ fun () -> octez_protocol_lib @@ -6781,6 +6806,7 @@ let hash = Protocol.hash ?octez_sc_rollup ?octez_sc_rollup_layer2 ?octez_sc_rollup_node + ?octez_injector () let active = register_alpha_family Active diff --git a/opam/tezos-injector-alpha.opam b/opam/tezos-injector-alpha.opam new file mode 100644 index 000000000000..cf83521dddb8 --- /dev/null +++ b/opam/tezos-injector-alpha.opam @@ -0,0 +1,24 @@ +# This file was automatically generated, do not edit. +# Edit file manifest/main.ml instead. +opam-version: "2.0" +maintainer: "contact@tezos.com" +authors: ["Tezos devteam"] +homepage: "https://www.tezos.com/" +bug-reports: "https://gitlab.com/tezos/tezos/issues" +dev-repo: "git+https://gitlab.com/tezos/tezos.git" +license: "MIT" +depends: [ + "dune" { >= "3.0" } + "ocaml" { >= "4.14" } + "octez-libs" + "tezos-protocol-alpha" + "octez-injector" + "octez-protocol-alpha-libs" + "octez-shell-libs" +] +build: [ + ["rm" "-r" "vendors" "contrib"] + ["dune" "build" "-p" name "-j" jobs] + ["dune" "runtest" "-p" name "-j" jobs] {with-test} +] +synopsis: "Tezos/Protocol: protocol-specific library for the injector binary" diff --git a/src/proto_alpha/lib_injector/dune b/src/proto_alpha/lib_injector/dune new file mode 100644 index 000000000000..a820d13070b2 --- /dev/null +++ b/src/proto_alpha/lib_injector/dune @@ -0,0 +1,23 @@ +; This file was automatically generated, do not edit. +; Edit file manifest/main.ml instead. + +(library + (name octez_injector_alpha) + (package tezos-injector-alpha) + (instrumentation (backend bisect_ppx)) + (libraries + octez-libs.tezos-base + tezos-protocol-alpha.protocol + octez-injector + octez-protocol-alpha-libs.tezos-client + octez-shell-libs.tezos-client-base + octez-protocol-alpha-libs.tezos-protocol-plugin) + (library_flags (:standard -linkall)) + (flags + (:standard) + -open Tezos_base.TzPervasives + -open Tezos_protocol_alpha + -open Octez_injector + -open Tezos_client_alpha + -open Tezos_client_base + -open Tezos_protocol_plugin_alpha)) -- GitLab From b083dc87e455be29dafce54dd396c1c4705fb27d Mon Sep 17 00:00:00 2001 From: Victor Dumitrescu Date: Fri, 7 Jul 2023 11:58:05 +0200 Subject: [PATCH 04/10] Proto/alpha: injector library --- .../lib_injector/injector_plugin.ml | 446 ++++++++++++++++++ .../lib_sc_rollup_node/sc_rollup_injector.ml | 2 + 2 files changed, 448 insertions(+) create mode 100644 src/proto_alpha/lib_injector/injector_plugin.ml diff --git a/src/proto_alpha/lib_injector/injector_plugin.ml b/src/proto_alpha/lib_injector/injector_plugin.ml new file mode 100644 index 000000000000..a4fa4cf85a9f --- /dev/null +++ b/src/proto_alpha/lib_injector/injector_plugin.ml @@ -0,0 +1,446 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Nomadic Labs, *) +(* Copyright (c) 2023 Functori, *) +(* *) +(*****************************************************************************) + +open Protocol +open Alpha_context +open Protocol_client_context +open Injector_sigs +open Injector_server +open Injector_server_operation +module Block_cache = + Aches_lwt.Lache.Make_result + (Aches.Rache.Transfer (Aches.Rache.LRU) (Block_hash)) + +module Proto_client = struct + open Tezos_micheline + + type operation = Injector_server_operation.t + + type state = Injector_server.state + + type unsigned_operation = + Tezos_base.Operation.shell_header * packed_contents_list + + let max_operation_data_length = Constants.max_operation_data_length + + let manager_pass = Operation_repr.manager_pass + + let to_manager_operation : t -> packed_manager_operation = function + | Transaction {amount; destination; parameters} -> + let destination = + Contract.of_b58check destination + |> WithExceptions.Result.to_exn_f ~error:(fun _trace -> + Stdlib.failwith + "Injector_plugin.to_manager_operation: invalid destination") + in + let entrypoint, parameters = + match parameters with + | Some {entrypoint; value} -> + let entrypoint = + Entrypoint.of_string_lax entrypoint + |> WithExceptions.Result.to_exn_f ~error:(fun _trace -> + Stdlib.failwith + "Injector_plugin.to_manager_operation: invalid \ + entrypoint") + in + let expr = + Michelson_v1_parser.parse_expression value + |> Micheline_parser.no_parsing_error + |> WithExceptions.Result.to_exn_f ~error:(fun _trace -> + Stdlib.failwith + "Injector_plugin.to_manager_operation: invalid \ + parameters") + in + (entrypoint, Script.lazy_expr expr.expanded) + | None -> (Entrypoint.default, Script.unit_parameter) + in + Manager + (Transaction + { + amount = Tez.of_mutez_exn amount; + destination; + parameters; + entrypoint; + }) + + let of_manager_operation : type kind. kind manager_operation -> t option = + function + | Transaction {amount; parameters; entrypoint; destination} -> + Option.bind (Data_encoding.force_decode parameters) (fun parameters -> + Some + (Transaction + { + amount = Tez.to_mutez amount; + destination = Contract.to_b58check destination; + parameters = + Some + { + value = + Michelson_v1_printer.micheline_string_of_expression + ~zero_loc:true + parameters; + entrypoint = Entrypoint.to_string entrypoint; + }; + })) + | _ -> None + + let manager_operation_size (Manager operation) = + let contents = + Manager_operation + { + source = Signature.Public_key_hash.zero; + operation; + fee = Tez.zero; + counter = Manager_counter.Internal_for_tests.of_int 0; + gas_limit = Gas.Arith.zero; + storage_limit = Z.zero; + } + in + Data_encoding.Binary.length + Operation.contents_encoding_with_legacy_attestation_name + (Contents contents) + + let operation_size op = manager_operation_size (to_manager_operation op) + + (* The operation size overhead is an upper bound (in practice) of the overhead + that will be added to a manager operation. To compute it we can use any + manager operation (here a revelation), add an overhead with upper bounds as + values (for the fees, limits, counters, etc.) and compare the encoded + operations with respect to their size. + NOTE: This information is only used to pre-select operations from the + injector queue as a candidate batch. *) + let operation_size_overhead = + let dummy_operation = + Reveal + (Signature.Public_key.of_b58check_exn + "edpkuBknW28nW72KG6RoHtYW7p12T6GKc7nAbwYX5m8Wd9sDVC9yav") + in + let dummy_contents = + Manager_operation + { + source = Signature.Public_key_hash.zero; + operation = dummy_operation; + fee = Tez.of_mutez_exn 3_000_000L; + counter = Manager_counter.Internal_for_tests.of_int 500_000; + gas_limit = Gas.Arith.integral_of_int_exn 500_000; + storage_limit = Z.of_int 500_000; + } + in + let dummy_size = + Data_encoding.Binary.length + Operation.contents_encoding_with_legacy_attestation_name + (Contents dummy_contents) + in + dummy_size - manager_operation_size (Manager dummy_operation) + + let manager_operation_result_status (type kind) + (op_result : kind Apply_results.manager_operation_result) : + operation_status = + match op_result with + | Applied _ -> Successful + | Backtracked (_, None) -> Unsuccessful Backtracked + | Skipped _ -> Unsuccessful Skipped + | Backtracked (_, Some err) + (* Backtracked because internal operation failed *) + | Failed (_, err) -> + Unsuccessful (Failed (Environment.wrap_tztrace err)) + + let operation_result_status (type kind) + (op_result : kind Apply_results.contents_result) : operation_status = + match op_result with + | Preattestation_result _ -> Successful + | Attestation_result _ -> Successful + | Dal_attestation_result _ -> Successful + | Seed_nonce_revelation_result _ -> Successful + | Vdf_revelation_result _ -> Successful + | Double_attestation_evidence_result _ -> Successful + | Double_preattestation_evidence_result _ -> Successful + | Double_baking_evidence_result _ -> Successful + | Activate_account_result _ -> Successful + | Proposals_result -> Successful + | Ballot_result -> Successful + | Drain_delegate_result _ -> Successful + | Manager_operation_result {operation_result; _} -> + manager_operation_result_status operation_result + + let operation_contents_status (type kind) + (contents : kind Apply_results.contents_result_list) ~index : + operation_status tzresult = + let rec rec_status : + type kind. int -> kind Apply_results.contents_result_list -> _ = + fun n -> function + | Apply_results.Single_result _ when n <> 0 -> + error_with "No operation with index %d" index + | Single_result result -> Ok (operation_result_status result) + | Cons_result (result, _rest) when n = 0 -> + Ok (operation_result_status result) + | Cons_result (_result, rest) -> rec_status (n - 1) rest + in + rec_status index contents + + let operation_status_of_receipt (operation : Protocol.operation_receipt) + ~index : operation_status tzresult = + match (operation : _) with + | No_operation_metadata -> + error_with "Cannot find operation status because metadata is missing" + | Operation_metadata {contents} -> operation_contents_status contents ~index + + (* TODO: https://gitlab.com/tezos/tezos/-/issues/6339 *) + (* Don't make multiple calls to [operations_in_pass] RPC *) + let get_block_operations = + let ops_cache = Block_cache.create 32 in + fun cctxt block_hash -> + Block_cache.bind_or_put + ops_cache + block_hash + (fun block_hash -> + let open Lwt_result_syntax in + let+ operations = + Alpha_block_services.Operations.operations_in_pass + cctxt + ~chain:cctxt#chain + ~block:(`Hash (block_hash, 0)) + ~metadata:`Always + manager_pass + in + List.fold_left + (fun acc (op : Alpha_block_services.operation) -> + Operation_hash.Map.add op.hash op acc) + Operation_hash.Map.empty + operations) + Lwt.return + + let operation_status (node_ctxt : state) block_hash operation_hash ~index = + let open Lwt_result_syntax in + let* operations = get_block_operations node_ctxt.cctxt block_hash in + match Operation_hash.Map.find_opt operation_hash operations with + | None -> return_none + | Some operation -> ( + match operation.receipt with + | Empty -> + failwith "Cannot find operation status because metadata is empty" + | Too_large -> + failwith + "Cannot find operation status because metadata is too large" + | Receipt receipt -> + let*? status = operation_status_of_receipt receipt ~index in + return_some status) + + let dummy_sk_uri = + WithExceptions.Result.get_ok ~loc:__LOC__ + @@ Tezos_signer_backends.Unencrypted.make_sk + @@ Signature.Secret_key.of_b58check_exn + "edsk3UqeiQWXX7NFEY1wUs6J1t2ez5aQ3hEWdqX5Jr5edZiGLW8nZr" + + let simulate_operations cctxt ~force ~source ~src_pk ~successor_level + ~fee_parameter operations = + let open Lwt_result_syntax in + let fee_parameter : Injection.fee_parameter = + { + minimal_fees = Tez.of_mutez_exn fee_parameter.minimal_fees.mutez; + minimal_nanotez_per_byte = fee_parameter.minimal_nanotez_per_byte; + minimal_nanotez_per_gas_unit = + fee_parameter.minimal_nanotez_per_gas_unit; + force_low_fee = fee_parameter.force_low_fee; + fee_cap = Tez.of_mutez_exn fee_parameter.fee_cap.mutez; + burn_cap = Tez.of_mutez_exn fee_parameter.burn_cap.mutez; + } + in + let open Annotated_manager_operation in + let annotated_operations = + List.map + (fun operation -> + let (Manager operation) = to_manager_operation operation in + Annotated_manager_operation + (Injection.prepare_manager_operation + ~fee:Limit.unknown + ~gas_limit:Limit.unknown + ~storage_limit:Limit.unknown + operation)) + operations + in + let (Manager_list annot_op) = + Annotated_manager_operation.manager_of_list annotated_operations + in + let cctxt = + new Protocol_client_context.wrap_full (cctxt :> Client_context.full) + in + let*! simulation_result = + Injection.inject_manager_operation + cctxt + ~simulation:true (* Only simulation here *) + ~force + ~chain:cctxt#chain + ~block:(`Head 0) + ~source + ~src_pk + ~src_sk:dummy_sk_uri + (* Use dummy secret key as it is not used by simulation *) + ~successor_level + ~fee:Limit.unknown + ~gas_limit:Limit.unknown + ~storage_limit:Limit.unknown + ~fee_parameter + annot_op + in + match simulation_result with + | Error trace -> + let exceeds_quota = + TzTrace.fold + (fun exceeds -> function + | Environment.Ecoproto_error + (Gas.Block_quota_exceeded | Gas.Operation_quota_exceeded) -> + true + | _ -> exceeds) + false + trace + in + fail (if exceeds_quota then `Exceeds_quotas trace else `TzError trace) + | Ok (_oph, packed_op, _contents, results) -> + let nb_ops = List.length operations in + let results = Apply_results.to_list (Contents_result_list results) in + (* packed_op can have reveal operations added automatically. *) + let start_index = List.length results - nb_ops in + (* remove extra reveal operations *) + let operations_statuses = + List.fold_left_i + (fun index_in_batch acc (Apply_results.Contents_result result) -> + if index_in_batch < start_index then acc + else + {index_in_batch; status = operation_result_status result} :: acc) + [] + results + |> List.rev + in + let unsigned_operation = + let {shell; protocol_data = Operation_data {contents; signature = _}} + = + packed_op + in + (shell, Contents_list contents) + in + return {operations_statuses; unsigned_operation} + + let sign_operation cctxt src_sk + ((shell, Contents_list contents) as unsigned_op) = + let open Lwt_result_syntax in + let unsigned_bytes = + Data_encoding.Binary.to_bytes_exn + Operation.unsigned_encoding_with_legacy_attestation_name + unsigned_op + in + let cctxt = + new Protocol_client_context.wrap_full (cctxt :> Client_context.full) + in + let+ signature = + Client_keys.sign + cctxt + ~watermark:Signature.Generic_operation + src_sk + unsigned_bytes + in + let op : packed_operation = + { + shell; + protocol_data = Operation_data {contents; signature = Some signature}; + } + in + Data_encoding.Binary.to_bytes_exn + Operation.encoding_with_legacy_attestation_name + op + + let time_until_next_block {minimal_block_delay; delay_increment_per_round; _} + (header : Tezos_base.Block_header.shell_header option) = + let open Result_syntax in + match header with + | None -> minimal_block_delay |> Int64.to_int |> Ptime.Span.of_int_s + | Some header -> + let minimal_block_delay = Period.of_seconds_exn minimal_block_delay in + let delay_increment_per_round = + Period.of_seconds_exn delay_increment_per_round + in + let next_level_timestamp = + let* durations = + Round.Durations.create + ~first_round_duration:minimal_block_delay + ~delay_increment_per_round + in + let* predecessor_round = Fitness.round_from_raw header.fitness in + Round.timestamp_of_round + durations + ~predecessor_timestamp:header.timestamp + ~predecessor_round + ~round:Round.zero + in + let next_level_timestamp = + Result.value + next_level_timestamp + ~default: + (WithExceptions.Result.get_ok + ~loc:__LOC__ + Timestamp.(header.timestamp +? minimal_block_delay)) + in + Ptime.diff + (Time.System.of_protocol_exn next_level_timestamp) + (Time.System.now ()) + + let check_fee_parameters {fee_parameters; _} = + let check_value purpose name compare to_string mempool_default value = + if compare mempool_default value > 0 then + error_with + "Bad configuration fee_parameter.%s for %s. It must be at least %s \ + for operations of the injector to be propagated." + name + (Configuration.string_of_purpose purpose) + (to_string mempool_default) + else Ok () + in + let check purpose + { + Injector_sigs.minimal_fees; + minimal_nanotez_per_byte; + minimal_nanotez_per_gas_unit; + force_low_fee = _; + fee_cap = _; + burn_cap = _; + } = + let open Result_syntax in + let+ () = + check_value + purpose + "minimal_fees" + Int64.compare + Int64.to_string + (Protocol.Alpha_context.Tez.to_mutez + Plugin.Mempool.default_minimal_fees) + minimal_fees.mutez + and+ () = + check_value + purpose + "minimal_nanotez_per_byte" + Q.compare + Q.to_string + Plugin.Mempool.default_minimal_nanotez_per_byte + minimal_nanotez_per_byte + and+ () = + check_value + purpose + "minimal_nanotez_per_gas_unit" + Q.compare + Q.to_string + Plugin.Mempool.default_minimal_nanotez_per_gas_unit + minimal_nanotez_per_gas_unit + in + () + in + check Transaction fee_parameters + + let checks state = check_fee_parameters state +end + +let () = register_proto_client Protocol.hash (module Proto_client) diff --git a/src/proto_alpha/lib_sc_rollup_node/sc_rollup_injector.ml b/src/proto_alpha/lib_sc_rollup_node/sc_rollup_injector.ml index 00883fc02d21..6a57c4062347 100644 --- a/src/proto_alpha/lib_sc_rollup_node/sc_rollup_injector.ml +++ b/src/proto_alpha/lib_sc_rollup_node/sc_rollup_injector.ml @@ -195,6 +195,8 @@ module Proto_client = struct error_with "Cannot find operation status because metadata is missing" | Operation_metadata {contents} -> operation_contents_status contents ~index + (* TODO: https://gitlab.com/tezos/tezos/-/issues/6339 *) + (* Don't make multiple calls to [operations_in_pass] RPC *) let get_block_operations = let ops_cache = Block_cache.create 32 in fun cctxt block_hash -> -- GitLab From 346a2ac0d27a81a978f26d795b686cb03caf9a81 Mon Sep 17 00:00:00 2001 From: Victor Dumitrescu Date: Fri, 7 Jul 2023 12:00:04 +0200 Subject: [PATCH 05/10] Manifest: add contrib/octez_injector_server --- .dockerignore | 5 ++-- .gitignore | 5 ++-- .gitlab/ci/jobs/packaging/opam_package.yml | 2 ++ contrib/octez_injector_server/dune | 32 ++++++++++++++++++++++ dune-project | 1 + manifest/main.ml | 24 ++++++++++++++++ opam/octez-injector-server.opam | 24 ++++++++++++++++ script-inputs/experimental-executables | 1 + 8 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 contrib/octez_injector_server/dune create mode 100644 opam/octez-injector-server.opam diff --git a/.dockerignore b/.dockerignore index 9715e2da21ed..bd0b7718f6a6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -37,6 +37,7 @@ octez-proxy-server octez-signer octez-smart-rollup-node* octez-smart-rollup-client-* +octez-injector-server octogram scripts/opam-test-all.sh.DONE @@ -96,8 +97,8 @@ crash.log crash.*.log # Exclude all .tfvars files, which are likely to contain sensitive data, such as -# password, private keys, and other secrets. These should not be part of version -# control as they are data points which are potentially sensitive and subject +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject # to change depending on the environment. *.tfvars *.tfvars.json diff --git a/.gitignore b/.gitignore index 234c75026193..7ee1132f7fc3 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ __pycache__ /octez-dal-node /octez-dac-* /octez-binaries +/octez-injector-server /octogram /simulation-scenario @@ -119,8 +120,8 @@ crash.log crash.*.log # Exclude all .tfvars files, which are likely to contain sensitive data, such as -# password, private keys, and other secrets. These should not be part of version -# control as they are data points which are potentially sensitive and subject +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject # to change depending on the environment. *.tfvars *.tfvars.json diff --git a/.gitlab/ci/jobs/packaging/opam_package.yml b/.gitlab/ci/jobs/packaging/opam_package.yml index e385632b337a..087f2687ce11 100644 --- a/.gitlab/ci/jobs/packaging/opam_package.yml +++ b/.gitlab/ci/jobs/packaging/opam_package.yml @@ -527,6 +527,8 @@ opam:octez-injector: variables: package: octez-injector +# Ignoring unreleased package octez-injector-server. + opam:octez-l2-libs: extends: - .opam_template diff --git a/contrib/octez_injector_server/dune b/contrib/octez_injector_server/dune new file mode 100644 index 000000000000..a25652bd5dde --- /dev/null +++ b/contrib/octez_injector_server/dune @@ -0,0 +1,32 @@ +; This file was automatically generated, do not edit. +; Edit file manifest/main.ml instead. + +(executable + (name injector_main) + (public_name octez-injector-server) + (package octez-injector-server) + (instrumentation (backend bisect_ppx)) + (libraries + octez-libs.tezos-base + octez-injector + octez-libs.tezos-stdlib-unix + octez-libs.tezos-rpc-http-server + octez-libs.tezos-rpc-http + octez-shell-libs.tezos-client-base + octez-shell-libs.tezos-client-base-unix + data-encoding + octez_injector_alpha) + (link_flags + (:standard) + (:include %{workspace_root}/static-link-flags.sexp) + (:include %{workspace_root}/macos-link-flags.sexp) + (-linkall)) + (flags + (:standard) + -open Tezos_base.TzPervasives + -open Octez_injector + -open Tezos_stdlib_unix + -open Tezos_rpc_http_server + -open Tezos_rpc_http + -open Tezos_client_base + -open Tezos_client_base_unix)) diff --git a/dune-project b/dune-project index acd4397ba4db..0245b6e0a5e6 100644 --- a/dune-project +++ b/dune-project @@ -27,6 +27,7 @@ (package (name octez-evm-proxy-lib-prod)(allow_empty)) (package (name octez-evm-proxy-tests)(allow_empty)) (package (name octez-injector)) +(package (name octez-injector-server)) (package (name octez-l2-libs)) (package (name octez-libs)) (package (name octez-node)) diff --git a/manifest/main.ml b/manifest/main.ml index 14cd06626673..492aca7455a8 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -7580,6 +7580,30 @@ let _octez_snoop = :: [S "package" :: [S "octez-snoop"]]; ] +let _octez_injector_server = + public_exe + "octez-injector-server" + ~internal_name:"injector_main" + ~path:"contrib/octez_injector_server" + ~synopsis:"Octez injector" + ~release_status:Experimental + ~with_macos_security_framework:true + ~linkall:true + ~deps: + [ + octez_base |> open_ ~m:"TzPervasives"; + octez_injector_lib |> open_; + octez_stdlib_unix |> open_; + octez_rpc_http_server |> open_; + octez_rpc_http |> open_; + octez_client_base |> open_; + octez_client_base_unix |> open_; + data_encoding; + (* No code from octez_injector_alpha is used, but it's imported in order to *) + (* run the protocol registration code *) + Protocol.(octez_injector alpha |> if_some); + ] + (* We use Dune's select statement and keep uTop optional *) (* Keeping uTop optional lets `make build` succeed, *) (* which uses tezos/opam-repository to resolve dependencies, *) diff --git a/opam/octez-injector-server.opam b/opam/octez-injector-server.opam new file mode 100644 index 000000000000..56479409d7b4 --- /dev/null +++ b/opam/octez-injector-server.opam @@ -0,0 +1,24 @@ +# This file was automatically generated, do not edit. +# Edit file manifest/main.ml instead. +opam-version: "2.0" +maintainer: "contact@tezos.com" +authors: ["Tezos devteam"] +homepage: "https://www.tezos.com/" +bug-reports: "https://gitlab.com/tezos/tezos/issues" +dev-repo: "git+https://gitlab.com/tezos/tezos.git" +license: "MIT" +depends: [ + "dune" { >= "3.0" } + "ocaml" { >= "4.14" } + "octez-libs" + "octez-injector" + "octez-shell-libs" + "data-encoding" { >= "0.7.1" & < "1.0.0" } + "tezos-injector-alpha" +] +build: [ + ["rm" "-r" "vendors"] + ["dune" "build" "-p" name "-j" jobs] + ["dune" "runtest" "-p" name "-j" jobs] {with-test} +] +synopsis: "Octez injector" diff --git a/script-inputs/experimental-executables b/script-inputs/experimental-executables index b7271fd0a691..388de2b5df06 100644 --- a/script-inputs/experimental-executables +++ b/script-inputs/experimental-executables @@ -2,6 +2,7 @@ octez-evm-proxy-server octez-smart-rollup-sequencer-node octez-smart-rollup-node octez-dal-node +octez-injector-server octez-smart-rollup-node-alpha octez-smart-rollup-client-alpha octez-accuser-alpha -- GitLab From 550729c0b6fafe7399a4e6e0207094ab8b91d475 Mon Sep 17 00:00:00 2001 From: Victor Dumitrescu Date: Fri, 7 Jul 2023 12:01:27 +0200 Subject: [PATCH 06/10] Octez_injector_server: initialize --- .../octez_injector_server/configuration.ml | 107 +++++++++++++ contrib/octez_injector_server/event.ml | 36 +++++ .../injector_daemon_http.ml | 144 ++++++++++++++++++ .../octez_injector_server/injector_main.ml | 133 ++++++++++++++++ .../injector_services.ml | 110 +++++++++++++ 5 files changed, 530 insertions(+) create mode 100644 contrib/octez_injector_server/configuration.ml create mode 100644 contrib/octez_injector_server/event.ml create mode 100644 contrib/octez_injector_server/injector_daemon_http.ml create mode 100644 contrib/octez_injector_server/injector_main.ml create mode 100644 contrib/octez_injector_server/injector_services.ml diff --git a/contrib/octez_injector_server/configuration.ml b/contrib/octez_injector_server/configuration.ml new file mode 100644 index 000000000000..6a1757a1935a --- /dev/null +++ b/contrib/octez_injector_server/configuration.ml @@ -0,0 +1,107 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Nomadic Labs *) +(* *) +(*****************************************************************************) + +type t = { + data_dir : string; + rpc_address : string; + rpc_port : int; + block_delay : float; + signer : string; +} + +let default_data_dir = + Filename.concat (Sys.getenv "HOME") ".octez-injector-server" + +let relative_filename data_dir = Filename.concat data_dir "config.json" + +let default_rpc_address = "127.0.0.1" + +let default_rpc_port = 6732 + +let default_block_delay = 0.5 + +let make ~data_dir ~rpc_address ~rpc_port ~block_delay signer = + let rpc_port = int_of_string rpc_port in + let block_delay = float_of_string block_delay in + {data_dir; rpc_address; rpc_port; block_delay; signer} + +let data_dir_path config subpath = Filename.concat config.data_dir subpath + +let filename config = relative_filename config.data_dir + +let data_dir config = config.data_dir + +let signers config = config.signer + +let encoding : t Data_encoding.t = + let open Data_encoding in + conv + (fun {data_dir; rpc_address; rpc_port; block_delay; signer} -> + (data_dir, rpc_address, rpc_port, block_delay, signer)) + (fun (data_dir, rpc_address, rpc_port, block_delay, signer) -> + {data_dir; rpc_address; rpc_port; block_delay; signer}) + (obj5 + (dft + "data-dir" + ~description:"Location of the data dir" + string + default_data_dir) + (req "rpc-addr" ~description:"RPC address" string) + (req "rpc-port" ~description:"RPC port" uint16) + (req "block-delay" ~description:"Block delay" float) + (req + "signer" + ~description: + "Signer for injected operations (only one signer currently \ + supported)" + string)) + +type error += Injector_server_unable_to_write_configuration_file of string + +let () = + register_error_kind + ~id:"injector-server.unable_to_write_configuration_file" + ~title:"Unable to write configuration file" + ~description:"Unable to write configuration file" + ~pp:(fun ppf file -> + Format.fprintf ppf "Unable to write the configuration file %s" file) + `Permanent + Data_encoding.(obj1 (req "file" string)) + (function + | Injector_server_unable_to_write_configuration_file path -> Some path + | _ -> None) + (fun path -> Injector_server_unable_to_write_configuration_file path) + +let save config = + let open Lwt_syntax in + let file = filename config in + protect @@ fun () -> + let* v = + let* () = Lwt_utils_unix.create_dir @@ data_dir config in + Lwt_utils_unix.with_atomic_open_out file @@ fun chan -> + let json = Data_encoding.Json.construct encoding config in + let content = Data_encoding.Json.to_string json in + Lwt_utils_unix.write_string chan content + in + Lwt.return + (Result.map_error + (fun _ -> [Injector_server_unable_to_write_configuration_file file]) + v) + +let load ~data_dir = + let open Lwt_result_syntax in + let+ json = + let*! json = Lwt_utils_unix.Json.read_file (relative_filename data_dir) in + match json with + | Ok json -> return json + | Error (Exn _ :: _ as e) -> + let*! () = Event.(emit data_dir_not_found data_dir) in + fail e + | Error e -> fail e + in + let config = Data_encoding.Json.destruct encoding json in + {config with data_dir} diff --git a/contrib/octez_injector_server/event.ml b/contrib/octez_injector_server/event.ml new file mode 100644 index 000000000000..67f62413b2a6 --- /dev/null +++ b/contrib/octez_injector_server/event.ml @@ -0,0 +1,36 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Nomadic Labs *) +(* *) +(*****************************************************************************) +include Internal_event.Simple + +let section = ["injector"; "daemon"] + +let data_dir_not_found = + declare_1 + ~section + ~name:"injector_server_no_data_dir" + ~msg: + "The injector server data directory {path} doesn't exist. Create using: \ + init-config --data-dir={path} " + ~level:Error + ("path", Data_encoding.(string)) + +let listening = + declare_1 + ~section + ~level:Notice + ~name:"injector_server_listening" + ~msg:"listening on address: {address}" + ("address", P2p_addr.encoding) + +let accepting_requests = + declare_2 + ~section + ~level:Notice + ~name:"injector_server_accepting_requests" + ~msg:"accepting {transport_protocol} requests on port {port}" + ("transport_protocol", Data_encoding.string) + ("port", Data_encoding.int31) diff --git a/contrib/octez_injector_server/injector_daemon_http.ml b/contrib/octez_injector_server/injector_daemon_http.ml new file mode 100644 index 000000000000..4ad7bea77751 --- /dev/null +++ b/contrib/octez_injector_server/injector_daemon_http.ml @@ -0,0 +1,144 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Nomadic Labs *) +(* *) +(*****************************************************************************) + +type error += Injector_server_unsupported_configuration of string + +let () = + register_error_kind + ~id:"injector-server.unsupported_configuration" + ~title:"Configuration not supported" + ~description:"Configuration not supported" + ~pp:(fun ppf reason -> + Format.fprintf ppf "Configuration not supported: %s" reason) + `Permanent + Data_encoding.(obj1 (req "reason" string)) + (function + | Injector_server_unsupported_configuration reason -> Some reason + | _ -> None) + (fun reason -> Injector_server_unsupported_configuration reason) + +type error += Injector_server_cannot_deserialize_contract_hash of string + +let () = + register_error_kind + ~id:"injector-server.cannot_deserialize_contract_hash" + ~title:"Contract hash could not be deserialized" + ~description:"Contract hash could not be deserialized" + ~pp:(fun ppf hash -> + Format.fprintf ppf "Contract hash %s could not be deserialized" hash) + `Permanent + Data_encoding.(obj1 (req "hash" string)) + (function + | Injector_server_cannot_deserialize_contract_hash hash -> Some hash + | _ -> None) + (fun hash -> Injector_server_cannot_deserialize_contract_hash hash) + +let make_signers_for_transactions signer block_delay = + (* Multiple or no signers currently not supported. The encoding also + sets a maximum length of 1 for the [signers] list. *) + let open Result_syntax in + let source = Signature.Public_key_hash.of_b58check_exn signer in + return + ( source, + [ + ( source, + `Delay_block block_delay, + [Injector_server.Configuration.Transaction] ); + ] ) + +let register_dir signer = + let open Lwt_result_syntax in + let open Injector_services in + let dir = + Tezos_rpc.Directory.register0 + Tezos_rpc.Directory.empty + add_pending_transaction + (fun () op -> + let*! inj_operation_hash = + Injector_server.add_pending_operation ~source:signer op + in + let*! () = Injector_server.inject () in + Lwt.return inj_operation_hash) + in + let dir = + Tezos_rpc.Directory.register0 dir operation_status (fun {op_hash} () -> + let op_hash = + Injector_server.Inj_operation.Hash.of_b58check_exn op_hash + in + let status = Injector_server.operation_status op_hash in + return + @@ Option.map + (fun (status : Injector_server.status) -> + match status with + | Pending _ -> Pending + | Injected info -> + Injected + { + injected_oph = info.oph; + injected_op_index = info.op_index; + } + | Included info -> + Included + { + included_oph = info.oph; + included_op_index = info.op_index; + block = info.l1_block; + level = info.l1_level; + }) + status) + in + + let dir = + Tezos_rpc.Directory.register0 dir inject (fun () () -> + let*! () = Injector_server.inject () in + return_unit) + in + dir + +let start ~rpc_address ~rpc_port ~source () = + let open Lwt_result_syntax in + let rpc_address = P2p_addr.of_string_exn rpc_address in + let mode = `TCP (`Port rpc_port) in + let acl = RPC_server.Acl.allow_all in + let dir = register_dir source in + let server = + RPC_server.init_server dir ~acl ~media_types:Media_type.all_media_types + in + Lwt.catch + (fun () -> + let*! () = Event.(emit listening) rpc_address in + let*! () = + RPC_server.launch + ~host:(Ipaddr.V6.to_string rpc_address) + server + ~callback:(RPC_server.resto_callback server) + mode + in + Lwt_utils.never_ending ()) + (function + | Unix.Unix_error (Unix.EADDRINUSE, "bind", "") -> + failwith "Port already in use." + | exn -> fail_with_exn exn) + +let run ~data_dir (cctxt : Client_context.full) = + let open Lwt_result_syntax in + let state : Injector_server.state = + { + cctxt; + fee_parameters = Injector_server.Configuration.default_fee_parameters; + minimal_block_delay = 15L; + delay_increment_per_round = 8L; + } + in + let* (Configuration.{rpc_address; rpc_port; data_dir = _; block_delay; signer} + as config) = + Configuration.load ~data_dir + in + let*? source, signers = make_signers_for_transactions signer block_delay in + let* () = Injector_server.init cctxt ~data_dir state ~signers in + let*! () = Event.(emit accepting_requests) ("HTTP", config.rpc_port) in + start ~rpc_address ~rpc_port ~source () diff --git a/contrib/octez_injector_server/injector_main.ml b/contrib/octez_injector_server/injector_main.ml new file mode 100644 index 000000000000..045430c4a0c2 --- /dev/null +++ b/contrib/octez_injector_server/injector_main.ml @@ -0,0 +1,133 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Nomadic Labs *) +(* *) +(*****************************************************************************) + +let group = + { + Tezos_clic.name = "octez-injector"; + title = "Commands related to the injector server"; + } + +let data_dir_arg = + let default = Configuration.default_data_dir in + Tezos_clic.default_arg + ~long:"data-dir" + ~placeholder:"data-dir" + ~doc:"The path to the injector server data directory" + ~default + (Client_config.string_parameter ()) + +let run_command = + let open Tezos_clic in + command + ~group + ~desc:"Run the injector server" + (args1 data_dir_arg) + (prefixes ["run"] @@ stop) + (fun data_dir cctxt -> Injector_daemon_http.run ~data_dir cctxt) + +module Config_init = struct + open Lwt_result_syntax + open Tezos_clic + + let create_configuration ~data_dir ~rpc_address ~rpc_port ~block_delay ~signer + (cctxt : Client_context.full) = + let config = + Configuration.make ~data_dir ~rpc_address ~rpc_port ~block_delay signer + in + let* () = Configuration.save config in + let*! _ = + cctxt#message + "Injector server configuration written in %s" + (Configuration.filename config) + in + return () + + let address_parameter = + param + ~name:"signer-address" + ~desc:"A Tezos address" + (parameter (fun _cctxt s -> + let open Lwt_result_syntax in + return s)) + + let command = + command + ~group + ~desc:"Initialize injector server configuration" + (args4 + (default_arg + ~doc:"listening address or host name" + ~short:'a' + ~long:"address" + ~placeholder:"host|address" + ~default:Configuration.default_rpc_address + (parameter (fun _ s -> return s))) + (default_arg + ~doc:"listening HTTP port" + ~short:'p' + ~long:"port" + ~placeholder:"port number" + ~default:(Configuration.default_rpc_port |> string_of_int) + (parameter (fun _ s -> return s))) + (default_arg + ~doc:"block delay" + ~short:'b' + ~long:"block-delay" + ~placeholder:"block delay" + ~default:(Configuration.default_block_delay |> string_of_float) + (parameter (fun _ s -> return s))) + (default_arg + ~doc:"data directory" + ~short:'d' + ~long:"data-dir" + ~placeholder:"path" + ~default:Configuration.default_data_dir + (parameter (fun _ x -> Lwt.return_ok x)))) + (prefix "init-config" @@ address_parameter @@ stop) + (fun (rpc_address, rpc_port, block_delay, data_dir) signer cctxt -> + create_configuration + ~data_dir + ~rpc_address + ~rpc_port + ~block_delay + ~signer + cctxt) +end + +let commands () = [Config_init.command; run_command] + +let select_commands _ _ = + let open Lwt_result_syntax in + return (commands ()) + +module Daemon_config = struct + type t = unit + + let global_options () = Tezos_clic.no_options + + let parse_config_args = Client_config.parse_config_args + + let default_chain = Client_config.default_chain + + let default_block = Client_config.default_block + + let default_base_dir = Client_config.default_base_dir + + let default_daily_logs_path = None + + let default_media_type = Daemon_config.default_media_type + + let other_registrations = None + + let clic_commands ~base_dir:_ ~config_commands:_ ~builtin_commands:_ + ~other_commands ~require_auth:_ = + other_commands + + let logger = None +end + +let () = Client_main_run.run (module Daemon_config) ~select_commands diff --git a/contrib/octez_injector_server/injector_services.ml b/contrib/octez_injector_server/injector_services.ml new file mode 100644 index 000000000000..1144bba37700 --- /dev/null +++ b/contrib/octez_injector_server/injector_services.ml @@ -0,0 +1,110 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Nomadic Labs *) +(* *) +(*****************************************************************************) + +open Injector_server + +let add_pending_transaction : + ( [`POST], + unit, + unit, + unit, + Injector_server_operation.t, + Inj_operation.hash ) + Tezos_rpc.Service.t = + Tezos_rpc.Service.post_service + ~description:"Add a pending operation to the injector queue" + ~query:Tezos_rpc.Query.empty + ~input:Injector_server_operation.encoding + ~output:Inj_operation.Hash.encoding + Tezos_rpc.Path.(root / "add_pending_transaction") + +type op_query = {op_hash : string} + +let injector_op_query : op_query Tezos_rpc.Query.t = + let open Tezos_rpc.Query in + query (fun op_hash -> {op_hash}) + |+ field "op_hash" Tezos_rpc.Arg.string "" (fun t -> t.op_hash) + |> seal + +(* Simplified version of [Injector.status] *) +type status = + | Pending + | Injected of {injected_oph : Operation_hash.t; injected_op_index : int} + | Included of { + included_oph : Operation_hash.t; + included_op_index : int; + block : Block_hash.t; + level : int32; + } + +let status_encoding : status Data_encoding.encoding = + Data_encoding.( + union + ~tag_size:`Uint8 + [ + case + ~title:"Pending" + (Tag 0) + (obj1 (req "pending" unit)) + (function Pending -> Some () | _ -> None) + (fun _s -> Pending); + case + ~title:"Injected" + (Tag 1) + (obj2 (req "injected_oph" string) (req "injected_op_index" int32)) + (function + | Injected {injected_oph; injected_op_index} -> + Some + ( Operation_hash.to_b58check injected_oph, + Int32.of_int injected_op_index ) + | _ -> None) + (fun (oph, op_index) -> + Injected + { + injected_oph = Operation_hash.of_b58check_exn oph; + injected_op_index = Int32.to_int op_index; + }); + case + ~title:"Included" + (Tag 2) + (obj4 + (req "included_oph" string) + (req "included_op_index" int32) + (req "block" string) + (req "level" int32)) + (function + | Included {included_oph; included_op_index; block; level} -> + Some + ( Operation_hash.to_b58check included_oph, + Int32.of_int included_op_index, + Block_hash.to_b58check block, + level ) + | _ -> None) + (fun (oph, op_index, block, level) -> + Included + { + included_oph = Operation_hash.of_b58check_exn oph; + included_op_index = Int32.to_int op_index; + block = Block_hash.of_b58check_exn block; + level; + }); + ]) + +let operation_status : + ([`GET], unit, unit, op_query, unit, status option) Tezos_rpc.Service.t = + Tezos_rpc.Service.get_service + ~description:"Query the status of an injector operation" + ~query:injector_op_query + ~output:(Data_encoding.option status_encoding) + Tezos_rpc.Path.(root / "operation_status") + +let inject : ([`GET], unit, unit, unit, unit, unit) Tezos_rpc.Service.t = + Tezos_rpc.Service.get_service + ~description:"Inject operations" + ~query:Tezos_rpc.Query.empty + ~output:Data_encoding.unit + Tezos_rpc.Path.(root / "inject") -- GitLab From e620868d3635b4f64141bb8cd48a449a822c9faf Mon Sep 17 00:00:00 2001 From: Victor Dumitrescu Date: Tue, 18 Jul 2023 16:42:56 +0200 Subject: [PATCH 07/10] Backport !9194 to Oxford --- manifest/main.ml | 2 +- opam/tezos-injector-018-Proxford.opam | 24 + src/proto_018_Proxford/lib_injector/dune | 23 + .../lib_injector/injector_plugin.ml | 438 ++++++++++++++++++ tezt/tests/main.ml | 1 + 5 files changed, 487 insertions(+), 1 deletion(-) create mode 100644 opam/tezos-injector-018-Proxford.opam create mode 100644 src/proto_018_Proxford/lib_injector/dune create mode 100644 src/proto_018_Proxford/lib_injector/injector_plugin.ml diff --git a/manifest/main.ml b/manifest/main.ml index 492aca7455a8..67d142f9f695 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -6342,7 +6342,7 @@ let hash = Protocol.hash ] in let octez_injector = - only_if N.(number >= 019) @@ fun () -> + only_if N.(number >= 018) @@ fun () -> private_lib (sf "octez_injector_%s" short_hash) ~path:(path // "lib_injector") diff --git a/opam/tezos-injector-018-Proxford.opam b/opam/tezos-injector-018-Proxford.opam new file mode 100644 index 000000000000..511da0761035 --- /dev/null +++ b/opam/tezos-injector-018-Proxford.opam @@ -0,0 +1,24 @@ +# This file was automatically generated, do not edit. +# Edit file manifest/main.ml instead. +opam-version: "2.0" +maintainer: "contact@tezos.com" +authors: ["Tezos devteam"] +homepage: "https://www.tezos.com/" +bug-reports: "https://gitlab.com/tezos/tezos/issues" +dev-repo: "git+https://gitlab.com/tezos/tezos.git" +license: "MIT" +depends: [ + "dune" { >= "3.0" } + "ocaml" { >= "4.14" } + "octez-libs" + "tezos-protocol-018-Proxford" + "octez-injector" + "octez-protocol-018-Proxford-libs" + "octez-shell-libs" +] +build: [ + ["rm" "-r" "vendors" "contrib"] + ["dune" "build" "-p" name "-j" jobs] + ["dune" "runtest" "-p" name "-j" jobs] {with-test} +] +synopsis: "Tezos/Protocol: protocol-specific library for the injector binary" diff --git a/src/proto_018_Proxford/lib_injector/dune b/src/proto_018_Proxford/lib_injector/dune new file mode 100644 index 000000000000..14b8ab530ac2 --- /dev/null +++ b/src/proto_018_Proxford/lib_injector/dune @@ -0,0 +1,23 @@ +; This file was automatically generated, do not edit. +; Edit file manifest/main.ml instead. + +(library + (name octez_injector_Proxford) + (package tezos-injector-018-Proxford) + (instrumentation (backend bisect_ppx)) + (libraries + octez-libs.tezos-base + tezos-protocol-018-Proxford + octez-injector + octez-protocol-018-Proxford-libs.tezos-client + octez-shell-libs.tezos-client-base + octez-protocol-018-Proxford-libs.tezos-protocol-plugin) + (library_flags (:standard -linkall)) + (flags + (:standard) + -open Tezos_base.TzPervasives + -open Tezos_protocol_018_Proxford + -open Octez_injector + -open Tezos_client_018_Proxford + -open Tezos_client_base + -open Tezos_protocol_plugin_018_Proxford)) diff --git a/src/proto_018_Proxford/lib_injector/injector_plugin.ml b/src/proto_018_Proxford/lib_injector/injector_plugin.ml new file mode 100644 index 000000000000..294e73de19ea --- /dev/null +++ b/src/proto_018_Proxford/lib_injector/injector_plugin.ml @@ -0,0 +1,438 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Nomadic Labs *) +(* Copyright (c) 2023 Functori, *) +(* *) +(*****************************************************************************) + +open Protocol +open Alpha_context +open Protocol_client_context +open Injector_sigs +open Injector_server +open Injector_server_operation +module Block_cache = + Aches_lwt.Lache.Make_result + (Aches.Rache.Transfer (Aches.Rache.LRU) (Block_hash)) + +module Proto_client = struct + open Tezos_micheline + + type operation = Injector_server_operation.t + + type state = Injector_server.state + + type unsigned_operation = + Tezos_base.Operation.shell_header * packed_contents_list + + let max_operation_data_length = Constants.max_operation_data_length + + let manager_pass = Operation_repr.manager_pass + + let to_manager_operation : t -> packed_manager_operation = function + | Transaction {amount; destination; parameters} -> + let destination = + Contract.of_b58check destination + |> WithExceptions.Result.to_exn_f ~error:(fun _trace -> + Stdlib.failwith + "Injector_plugin.to_manager_operation: invalid destination") + in + let entrypoint, parameters = + match parameters with + | Some {entrypoint; value} -> + let entrypoint = + Entrypoint.of_string_lax entrypoint + |> WithExceptions.Result.to_exn_f ~error:(fun _trace -> + Stdlib.failwith + "Injector_plugin.to_manager_operation: invalid \ + entrypoint") + in + let expr = + Michelson_v1_parser.parse_expression value + |> Micheline_parser.no_parsing_error + |> WithExceptions.Result.to_exn_f ~error:(fun _trace -> + Stdlib.failwith + "Injector_plugin.to_manager_operation: invalid \ + parameters") + in + (entrypoint, Script.lazy_expr expr.expanded) + | None -> (Entrypoint.default, Script.unit_parameter) + in + Manager + (Transaction + { + amount = Tez.of_mutez_exn amount; + destination; + parameters; + entrypoint; + }) + + let of_manager_operation : type kind. kind manager_operation -> t option = + function + | Transaction {amount; parameters; entrypoint; destination} -> + Option.bind (Data_encoding.force_decode parameters) (fun parameters -> + Some + (Transaction + { + amount = Tez.to_mutez amount; + destination = Contract.to_b58check destination; + parameters = + Some + { + value = + Michelson_v1_printer.micheline_string_of_expression + ~zero_loc:true + parameters; + entrypoint = Entrypoint.to_string entrypoint; + }; + })) + | _ -> None + + let manager_operation_size (Manager operation) = + let contents = + Manager_operation + { + source = Signature.Public_key_hash.zero; + operation; + fee = Tez.zero; + counter = Manager_counter.Internal_for_tests.of_int 0; + gas_limit = Gas.Arith.zero; + storage_limit = Z.zero; + } + in + Data_encoding.Binary.length Operation.contents_encoding (Contents contents) + + let operation_size op = manager_operation_size (to_manager_operation op) + + (* The operation size overhead is an upper bound (in practice) of the overhead + that will be added to a manager operation. To compute it we can use any + manager operation (here a revelation), add an overhead with upper bounds as + values (for the fees, limits, counters, etc.) and compare the encoded + operations with respect to their size. + NOTE: This information is only used to pre-select operations from the + injector queue as a candidate batch. *) + let operation_size_overhead = + let dummy_operation = + Reveal + (Signature.Public_key.of_b58check_exn + "edpkuBknW28nW72KG6RoHtYW7p12T6GKc7nAbwYX5m8Wd9sDVC9yav") + in + let dummy_contents = + Manager_operation + { + source = Signature.Public_key_hash.zero; + operation = dummy_operation; + fee = Tez.of_mutez_exn 3_000_000L; + counter = Manager_counter.Internal_for_tests.of_int 500_000; + gas_limit = Gas.Arith.integral_of_int_exn 500_000; + storage_limit = Z.of_int 500_000; + } + in + let dummy_size = + Data_encoding.Binary.length + Operation.contents_encoding + (Contents dummy_contents) + in + dummy_size - manager_operation_size (Manager dummy_operation) + + let manager_operation_result_status (type kind) + (op_result : kind Apply_results.manager_operation_result) : + operation_status = + match op_result with + | Applied _ -> Successful + | Backtracked (_, None) -> Unsuccessful Backtracked + | Skipped _ -> Unsuccessful Skipped + | Backtracked (_, Some err) + (* Backtracked because internal operation failed *) + | Failed (_, err) -> + Unsuccessful (Failed (Environment.wrap_tztrace err)) + + let operation_result_status (type kind) + (op_result : kind Apply_results.contents_result) : operation_status = + match op_result with + | Preattestation_result _ -> Successful + | Attestation_result _ -> Successful + | Dal_attestation_result _ -> Successful + | Seed_nonce_revelation_result _ -> Successful + | Vdf_revelation_result _ -> Successful + | Double_attestation_evidence_result _ -> Successful + | Double_preattestation_evidence_result _ -> Successful + | Double_baking_evidence_result _ -> Successful + | Activate_account_result _ -> Successful + | Proposals_result -> Successful + | Ballot_result -> Successful + | Drain_delegate_result _ -> Successful + | Manager_operation_result {operation_result; _} -> + manager_operation_result_status operation_result + + let operation_contents_status (type kind) + (contents : kind Apply_results.contents_result_list) ~index : + operation_status tzresult = + let rec rec_status : + type kind. int -> kind Apply_results.contents_result_list -> _ = + fun n -> function + | Apply_results.Single_result _ when n <> 0 -> + error_with "No operation with index %d" index + | Single_result result -> Ok (operation_result_status result) + | Cons_result (result, _rest) when n = 0 -> + Ok (operation_result_status result) + | Cons_result (_result, rest) -> rec_status (n - 1) rest + in + rec_status index contents + + let operation_status_of_receipt (operation : Protocol.operation_receipt) + ~index : operation_status tzresult = + match (operation : _) with + | No_operation_metadata -> + error_with "Cannot find operation status because metadata is missing" + | Operation_metadata {contents} -> operation_contents_status contents ~index + + let get_block_operations = + let ops_cache = Block_cache.create 32 in + fun cctxt block_hash -> + Block_cache.bind_or_put + ops_cache + block_hash + (fun block_hash -> + let open Lwt_result_syntax in + let+ operations = + Alpha_block_services.Operations.operations_in_pass + cctxt + ~chain:cctxt#chain + ~block:(`Hash (block_hash, 0)) + ~metadata:`Always + manager_pass + in + List.fold_left + (fun acc (op : Alpha_block_services.operation) -> + Operation_hash.Map.add op.hash op acc) + Operation_hash.Map.empty + operations) + Lwt.return + + let operation_status (node_ctxt : state) block_hash operation_hash ~index = + let open Lwt_result_syntax in + let* operations = get_block_operations node_ctxt.cctxt block_hash in + match Operation_hash.Map.find_opt operation_hash operations with + | None -> return_none + | Some operation -> ( + match operation.receipt with + | Empty -> + failwith "Cannot find operation status because metadata is empty" + | Too_large -> + failwith + "Cannot find operation status because metadata is too large" + | Receipt receipt -> + let*? status = operation_status_of_receipt receipt ~index in + return_some status) + + let dummy_sk_uri = + WithExceptions.Result.get_ok ~loc:__LOC__ + @@ Tezos_signer_backends.Unencrypted.make_sk + @@ Signature.Secret_key.of_b58check_exn + "edsk3UqeiQWXX7NFEY1wUs6J1t2ez5aQ3hEWdqX5Jr5edZiGLW8nZr" + + let simulate_operations cctxt ~force ~source ~src_pk ~successor_level + ~fee_parameter operations = + let open Lwt_result_syntax in + let fee_parameter : Injection.fee_parameter = + { + minimal_fees = Tez.of_mutez_exn fee_parameter.minimal_fees.mutez; + minimal_nanotez_per_byte = fee_parameter.minimal_nanotez_per_byte; + minimal_nanotez_per_gas_unit = + fee_parameter.minimal_nanotez_per_gas_unit; + force_low_fee = fee_parameter.force_low_fee; + fee_cap = Tez.of_mutez_exn fee_parameter.fee_cap.mutez; + burn_cap = Tez.of_mutez_exn fee_parameter.burn_cap.mutez; + } + in + let open Annotated_manager_operation in + let annotated_operations = + List.map + (fun operation -> + let (Manager operation) = to_manager_operation operation in + Annotated_manager_operation + (Injection.prepare_manager_operation + ~fee:Limit.unknown + ~gas_limit:Limit.unknown + ~storage_limit:Limit.unknown + operation)) + operations + in + let (Manager_list annot_op) = + Annotated_manager_operation.manager_of_list annotated_operations + in + let cctxt = + new Protocol_client_context.wrap_full (cctxt :> Client_context.full) + in + let*! simulation_result = + Injection.inject_manager_operation + cctxt + ~simulation:true (* Only simulation here *) + ~force + ~chain:cctxt#chain + ~block:(`Head 0) + ~source + ~src_pk + ~src_sk:dummy_sk_uri + (* Use dummy secret key as it is not used by simulation *) + ~successor_level + ~fee:Limit.unknown + ~gas_limit:Limit.unknown + ~storage_limit:Limit.unknown + ~fee_parameter + annot_op + in + match simulation_result with + | Error trace -> + let exceeds_quota = + TzTrace.fold + (fun exceeds -> function + | Environment.Ecoproto_error + (Gas.Block_quota_exceeded | Gas.Operation_quota_exceeded) -> + true + | _ -> exceeds) + false + trace + in + fail (if exceeds_quota then `Exceeds_quotas trace else `TzError trace) + | Ok (_oph, packed_op, _contents, results) -> + let nb_ops = List.length operations in + let results = Apply_results.to_list (Contents_result_list results) in + (* packed_op can have reveal operations added automatically. *) + let start_index = List.length results - nb_ops in + (* remove extra reveal operations *) + let operations_statuses = + List.fold_left_i + (fun index_in_batch acc (Apply_results.Contents_result result) -> + if index_in_batch < start_index then acc + else + {index_in_batch; status = operation_result_status result} :: acc) + [] + results + |> List.rev + in + let unsigned_operation = + let {shell; protocol_data = Operation_data {contents; signature = _}} + = + packed_op + in + (shell, Contents_list contents) + in + return {operations_statuses; unsigned_operation} + + let sign_operation cctxt src_sk + ((shell, Contents_list contents) as unsigned_op) = + let open Lwt_result_syntax in + let unsigned_bytes = + Data_encoding.Binary.to_bytes_exn Operation.unsigned_encoding unsigned_op + in + let cctxt = + new Protocol_client_context.wrap_full (cctxt :> Client_context.full) + in + let+ signature = + Client_keys.sign + cctxt + ~watermark:Signature.Generic_operation + src_sk + unsigned_bytes + in + let op : packed_operation = + { + shell; + protocol_data = Operation_data {contents; signature = Some signature}; + } + in + Data_encoding.Binary.to_bytes_exn Operation.encoding op + + let time_until_next_block {minimal_block_delay; delay_increment_per_round; _} + (header : Tezos_base.Block_header.shell_header option) = + let open Result_syntax in + match header with + | None -> minimal_block_delay |> Int64.to_int |> Ptime.Span.of_int_s + | Some header -> + let minimal_block_delay = Period.of_seconds_exn minimal_block_delay in + let delay_increment_per_round = + Period.of_seconds_exn delay_increment_per_round + in + let next_level_timestamp = + let* durations = + Round.Durations.create + ~first_round_duration:minimal_block_delay + ~delay_increment_per_round + in + let* predecessor_round = Fitness.round_from_raw header.fitness in + Round.timestamp_of_round + durations + ~predecessor_timestamp:header.timestamp + ~predecessor_round + ~round:Round.zero + in + let next_level_timestamp = + Result.value + next_level_timestamp + ~default: + (WithExceptions.Result.get_ok + ~loc:__LOC__ + Timestamp.(header.timestamp +? minimal_block_delay)) + in + Ptime.diff + (Time.System.of_protocol_exn next_level_timestamp) + (Time.System.now ()) + + let check_fee_parameters {fee_parameters; _} = + let check_value purpose name compare to_string mempool_default value = + if compare mempool_default value > 0 then + error_with + "Bad configuration fee_parameter.%s for %s. It must be at least %s \ + for operations of the injector to be propagated." + name + (Configuration.string_of_purpose purpose) + (to_string mempool_default) + else Ok () + in + let check purpose + { + Injector_sigs.minimal_fees; + minimal_nanotez_per_byte; + minimal_nanotez_per_gas_unit; + force_low_fee = _; + fee_cap = _; + burn_cap = _; + } = + let open Result_syntax in + let+ () = + check_value + purpose + "minimal_fees" + Int64.compare + Int64.to_string + (Protocol.Alpha_context.Tez.to_mutez + Plugin.Mempool.default_minimal_fees) + minimal_fees.mutez + and+ () = + check_value + purpose + "minimal_nanotez_per_byte" + Q.compare + Q.to_string + Plugin.Mempool.default_minimal_nanotez_per_byte + minimal_nanotez_per_byte + and+ () = + check_value + purpose + "minimal_nanotez_per_gas_unit" + Q.compare + Q.to_string + Plugin.Mempool.default_minimal_nanotez_per_gas_unit + minimal_nanotez_per_gas_unit + in + () + in + check Transaction fee_parameters + + let checks state = check_fee_parameters state +end + +let () = register_proto_client Protocol.hash (module Proto_client) diff --git a/tezt/tests/main.ml b/tezt/tests/main.ml index d6591c48811e..1a125eed03bd 100644 --- a/tezt/tests/main.ml +++ b/tezt/tests/main.ml @@ -151,6 +151,7 @@ let register_protocol_tests_that_use_supports_correctly () = Global_constants.register ~protocols ; Hash_data.register ~protocols ; Increase_paid_storage.register ~protocols ; + Injector_test.register ~protocols ; Large_metadata.register ~protocols ; Light.register ~protocols ; Liquidity_baking_per_block_votes.register ~protocols ; -- GitLab From 22d68e53cebef454d99d83fd7582a0d983f7dabc Mon Sep 17 00:00:00 2001 From: Victor Dumitrescu Date: Tue, 18 Jul 2023 16:41:35 +0200 Subject: [PATCH 08/10] Backport !9194 to Nairobi --- .gitlab/ci/jobs/packaging/opam_package.yml | 4 + contrib/octez_injector_server/dune | 17 +- dune-project | 2 + manifest/main.ml | 28 +- opam/octez-injector-server.opam | 6 +- opam/tezos-injector-017-PtNairob.opam | 24 + src/proto_017_PtNairob/lib_injector/dune | 23 + .../lib_injector/injector_plugin.ml | 438 ++++++++++++++++++ src/proto_018_Proxford/lib_injector/dune | 2 +- 9 files changed, 527 insertions(+), 17 deletions(-) create mode 100644 opam/tezos-injector-017-PtNairob.opam create mode 100644 src/proto_017_PtNairob/lib_injector/dune create mode 100644 src/proto_017_PtNairob/lib_injector/injector_plugin.ml diff --git a/.gitlab/ci/jobs/packaging/opam_package.yml b/.gitlab/ci/jobs/packaging/opam_package.yml index 087f2687ce11..b87b40c2c50d 100644 --- a/.gitlab/ci/jobs/packaging/opam_package.yml +++ b/.gitlab/ci/jobs/packaging/opam_package.yml @@ -878,6 +878,10 @@ opam:tezos-dal-node-services: variables: package: tezos-dal-node-services +# Ignoring unreleased package tezos-injector-017-PtNairob. + +# Ignoring unreleased package tezos-injector-018-Proxford. + # Ignoring unreleased package tezos-injector-alpha. # Ignoring unreleased package tezos-lazy-containers-tests. diff --git a/contrib/octez_injector_server/dune b/contrib/octez_injector_server/dune index a25652bd5dde..329f067f7933 100644 --- a/contrib/octez_injector_server/dune +++ b/contrib/octez_injector_server/dune @@ -15,7 +15,15 @@ octez-shell-libs.tezos-client-base octez-shell-libs.tezos-client-base-unix data-encoding - octez_injector_alpha) + (select void_for_linking-octez_injector_PtNairob from + (octez_injector_PtNairob -> void_for_linking-octez_injector_PtNairob.empty) + (-> void_for_linking-octez_injector_PtNairob.empty)) + (select void_for_linking-octez_injector_Proxford from + (octez_injector_Proxford -> void_for_linking-octez_injector_Proxford.empty) + (-> void_for_linking-octez_injector_Proxford.empty)) + (select void_for_linking-octez_injector_alpha from + (octez_injector_alpha -> void_for_linking-octez_injector_alpha.empty) + (-> void_for_linking-octez_injector_alpha.empty))) (link_flags (:standard) (:include %{workspace_root}/static-link-flags.sexp) @@ -30,3 +38,10 @@ -open Tezos_rpc_http -open Tezos_client_base -open Tezos_client_base_unix)) + +(rule + (action + (progn + (write-file void_for_linking-octez_injector_PtNairob.empty "") + (write-file void_for_linking-octez_injector_Proxford.empty "") + (write-file void_for_linking-octez_injector_alpha.empty "")))) diff --git a/dune-project b/dune-project index 0245b6e0a5e6..c8e01f79d131 100644 --- a/dune-project +++ b/dune-project @@ -95,6 +95,8 @@ (package (name tezos-dac-node-lib-test)(allow_empty)) (package (name tezos-dal-node-lib)) (package (name tezos-dal-node-services)) +(package (name tezos-injector-017-PtNairob)(allow_empty)) +(package (name tezos-injector-018-Proxford)(allow_empty)) (package (name tezos-injector-alpha)(allow_empty)) (package (name tezos-lazy-containers-tests)(allow_empty)) (package (name tezos-micheline-rewriting)) diff --git a/manifest/main.ml b/manifest/main.ml index 67d142f9f695..9b6ae1aca84b 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -6342,7 +6342,7 @@ let hash = Protocol.hash ] in let octez_injector = - only_if N.(number >= 018) @@ fun () -> + only_if N.(number >= 017) @@ fun () -> private_lib (sf "octez_injector_%s" short_hash) ~path:(path // "lib_injector") @@ -7590,19 +7590,19 @@ let _octez_injector_server = ~with_macos_security_framework:true ~linkall:true ~deps: - [ - octez_base |> open_ ~m:"TzPervasives"; - octez_injector_lib |> open_; - octez_stdlib_unix |> open_; - octez_rpc_http_server |> open_; - octez_rpc_http |> open_; - octez_client_base |> open_; - octez_client_base_unix |> open_; - data_encoding; - (* No code from octez_injector_alpha is used, but it's imported in order to *) - (* run the protocol registration code *) - Protocol.(octez_injector alpha |> if_some); - ] + ([ + octez_base |> open_ ~m:"TzPervasives"; + octez_injector_lib |> open_; + octez_stdlib_unix |> open_; + octez_rpc_http_server |> open_; + octez_rpc_http |> open_; + octez_client_base |> open_; + octez_client_base_unix |> open_; + data_encoding; + ] + (* No code from octez_injector_alpha is used, but it's imported in order to *) + (* run the protocol registration code *) + @ Protocol.(all_optionally [octez_injector])) (* We use Dune's select statement and keep uTop optional *) (* Keeping uTop optional lets `make build` succeed, *) diff --git a/opam/octez-injector-server.opam b/opam/octez-injector-server.opam index 56479409d7b4..d3093c448c0d 100644 --- a/opam/octez-injector-server.opam +++ b/opam/octez-injector-server.opam @@ -14,10 +14,14 @@ depends: [ "octez-injector" "octez-shell-libs" "data-encoding" { >= "0.7.1" & < "1.0.0" } +] +depopts: [ + "tezos-injector-017-PtNairob" + "tezos-injector-018-Proxford" "tezos-injector-alpha" ] build: [ - ["rm" "-r" "vendors"] + ["rm" "-r" "vendors" "contrib"] ["dune" "build" "-p" name "-j" jobs] ["dune" "runtest" "-p" name "-j" jobs] {with-test} ] diff --git a/opam/tezos-injector-017-PtNairob.opam b/opam/tezos-injector-017-PtNairob.opam new file mode 100644 index 000000000000..44aa22ab09b0 --- /dev/null +++ b/opam/tezos-injector-017-PtNairob.opam @@ -0,0 +1,24 @@ +# This file was automatically generated, do not edit. +# Edit file manifest/main.ml instead. +opam-version: "2.0" +maintainer: "contact@tezos.com" +authors: ["Tezos devteam"] +homepage: "https://www.tezos.com/" +bug-reports: "https://gitlab.com/tezos/tezos/issues" +dev-repo: "git+https://gitlab.com/tezos/tezos.git" +license: "MIT" +depends: [ + "dune" { >= "3.0" } + "ocaml" { >= "4.14" } + "octez-libs" + "tezos-protocol-017-PtNairob" + "octez-injector" + "octez-protocol-017-PtNairob-libs" + "octez-shell-libs" +] +build: [ + ["rm" "-r" "vendors" "contrib"] + ["dune" "build" "-p" name "-j" jobs] + ["dune" "runtest" "-p" name "-j" jobs] {with-test} +] +synopsis: "Tezos/Protocol: protocol-specific library for the injector binary" diff --git a/src/proto_017_PtNairob/lib_injector/dune b/src/proto_017_PtNairob/lib_injector/dune new file mode 100644 index 000000000000..bd3a08b5a268 --- /dev/null +++ b/src/proto_017_PtNairob/lib_injector/dune @@ -0,0 +1,23 @@ +; This file was automatically generated, do not edit. +; Edit file manifest/main.ml instead. + +(library + (name octez_injector_PtNairob) + (package tezos-injector-017-PtNairob) + (instrumentation (backend bisect_ppx)) + (libraries + octez-libs.tezos-base + tezos-protocol-017-PtNairob.protocol + octez-injector + octez-protocol-017-PtNairob-libs.tezos-client + octez-shell-libs.tezos-client-base + octez-protocol-017-PtNairob-libs.tezos-protocol-plugin) + (library_flags (:standard -linkall)) + (flags + (:standard) + -open Tezos_base.TzPervasives + -open Tezos_protocol_017_PtNairob + -open Octez_injector + -open Tezos_client_017_PtNairob + -open Tezos_client_base + -open Tezos_protocol_plugin_017_PtNairob)) diff --git a/src/proto_017_PtNairob/lib_injector/injector_plugin.ml b/src/proto_017_PtNairob/lib_injector/injector_plugin.ml new file mode 100644 index 000000000000..22c0639f1716 --- /dev/null +++ b/src/proto_017_PtNairob/lib_injector/injector_plugin.ml @@ -0,0 +1,438 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Nomadic Labs *) +(* Copyright (c) 2023 Functori, *) +(* *) +(*****************************************************************************) + +open Protocol +open Alpha_context +open Protocol_client_context +open Injector_sigs +open Injector_server +open Injector_server_operation +module Block_cache = + Aches_lwt.Lache.Make_result + (Aches.Rache.Transfer (Aches.Rache.LRU) (Block_hash)) + +module Proto_client = struct + open Tezos_micheline + + type operation = Injector_server_operation.t + + type state = Injector_server.state + + type unsigned_operation = + Tezos_base.Operation.shell_header * packed_contents_list + + let max_operation_data_length = Constants.max_operation_data_length + + let manager_pass = Operation_repr.manager_pass + + let to_manager_operation : t -> packed_manager_operation = function + | Transaction {amount; destination; parameters} -> + let destination = + Contract.of_b58check destination + |> WithExceptions.Result.to_exn_f ~error:(fun _trace -> + Stdlib.failwith + "Injector_plugin.to_manager_operation: invalid destination") + in + let entrypoint, parameters = + match parameters with + | Some {entrypoint; value} -> + let entrypoint = + Entrypoint.of_string_lax entrypoint + |> WithExceptions.Result.to_exn_f ~error:(fun _trace -> + Stdlib.failwith + "Injector_plugin.to_manager_operation: invalid \ + entrypoint") + in + let expr = + Michelson_v1_parser.parse_expression value + |> Micheline_parser.no_parsing_error + |> WithExceptions.Result.to_exn_f ~error:(fun _trace -> + Stdlib.failwith + "Injector_plugin.to_manager_operation: invalid \ + parameters") + in + (entrypoint, Script.lazy_expr expr.expanded) + | None -> (Entrypoint.default, Script.unit_parameter) + in + Manager + (Transaction + { + amount = Tez.of_mutez_exn amount; + destination; + parameters; + entrypoint; + }) + + let of_manager_operation : type kind. kind manager_operation -> t option = + function + | Transaction {amount; parameters; entrypoint; destination} -> + Option.bind (Data_encoding.force_decode parameters) (fun parameters -> + Some + (Transaction + { + amount = Tez.to_mutez amount; + destination = Contract.to_b58check destination; + parameters = + Some + { + value = + Michelson_v1_printer.micheline_string_of_expression + ~zero_loc:true + parameters; + entrypoint = Entrypoint.to_string entrypoint; + }; + })) + | _ -> None + + let manager_operation_size (Manager operation) = + let contents = + Manager_operation + { + source = Signature.Public_key_hash.zero; + operation; + fee = Tez.zero; + counter = Manager_counter.Internal_for_tests.of_int 0; + gas_limit = Gas.Arith.zero; + storage_limit = Z.zero; + } + in + Data_encoding.Binary.length Operation.contents_encoding (Contents contents) + + let operation_size op = manager_operation_size (to_manager_operation op) + + (* The operation size overhead is an upper bound (in practice) of the overhead + that will be added to a manager operation. To compute it we can use any + manager operation (here a revelation), add an overhead with upper bounds as + values (for the fees, limits, counters, etc.) and compare the encoded + operations with respect to their size. + NOTE: This information is only used to pre-select operations from the + injector queue as a candidate batch. *) + let operation_size_overhead = + let dummy_operation = + Reveal + (Signature.Public_key.of_b58check_exn + "edpkuBknW28nW72KG6RoHtYW7p12T6GKc7nAbwYX5m8Wd9sDVC9yav") + in + let dummy_contents = + Manager_operation + { + source = Signature.Public_key_hash.zero; + operation = dummy_operation; + fee = Tez.of_mutez_exn 3_000_000L; + counter = Manager_counter.Internal_for_tests.of_int 500_000; + gas_limit = Gas.Arith.integral_of_int_exn 500_000; + storage_limit = Z.of_int 500_000; + } + in + let dummy_size = + Data_encoding.Binary.length + Operation.contents_encoding + (Contents dummy_contents) + in + dummy_size - manager_operation_size (Manager dummy_operation) + + let manager_operation_result_status (type kind) + (op_result : kind Apply_results.manager_operation_result) : + operation_status = + match op_result with + | Applied _ -> Successful + | Backtracked (_, None) -> Unsuccessful Backtracked + | Skipped _ -> Unsuccessful Skipped + | Backtracked (_, Some err) + (* Backtracked because internal operation failed *) + | Failed (_, err) -> + Unsuccessful (Failed (Environment.wrap_tztrace err)) + + let operation_result_status (type kind) + (op_result : kind Apply_results.contents_result) : operation_status = + match op_result with + | Preendorsement_result _ -> Successful + | Endorsement_result _ -> Successful + | Dal_attestation_result _ -> Successful + | Seed_nonce_revelation_result _ -> Successful + | Vdf_revelation_result _ -> Successful + | Double_endorsement_evidence_result _ -> Successful + | Double_preendorsement_evidence_result _ -> Successful + | Double_baking_evidence_result _ -> Successful + | Activate_account_result _ -> Successful + | Proposals_result -> Successful + | Ballot_result -> Successful + | Drain_delegate_result _ -> Successful + | Manager_operation_result {operation_result; _} -> + manager_operation_result_status operation_result + + let operation_contents_status (type kind) + (contents : kind Apply_results.contents_result_list) ~index : + operation_status tzresult = + let rec rec_status : + type kind. int -> kind Apply_results.contents_result_list -> _ = + fun n -> function + | Apply_results.Single_result _ when n <> 0 -> + error_with "No operation with index %d" index + | Single_result result -> Ok (operation_result_status result) + | Cons_result (result, _rest) when n = 0 -> + Ok (operation_result_status result) + | Cons_result (_result, rest) -> rec_status (n - 1) rest + in + rec_status index contents + + let operation_status_of_receipt (operation : Protocol.operation_receipt) + ~index : operation_status tzresult = + match (operation : _) with + | No_operation_metadata -> + error_with "Cannot find operation status because metadata is missing" + | Operation_metadata {contents} -> operation_contents_status contents ~index + + let get_block_operations = + let ops_cache = Block_cache.create 32 in + fun cctxt block_hash -> + Block_cache.bind_or_put + ops_cache + block_hash + (fun block_hash -> + let open Lwt_result_syntax in + let+ operations = + Alpha_block_services.Operations.operations_in_pass + cctxt + ~chain:cctxt#chain + ~block:(`Hash (block_hash, 0)) + ~metadata:`Always + manager_pass + in + List.fold_left + (fun acc (op : Alpha_block_services.operation) -> + Operation_hash.Map.add op.hash op acc) + Operation_hash.Map.empty + operations) + Lwt.return + + let operation_status (node_ctxt : state) block_hash operation_hash ~index = + let open Lwt_result_syntax in + let* operations = get_block_operations node_ctxt.cctxt block_hash in + match Operation_hash.Map.find_opt operation_hash operations with + | None -> return_none + | Some operation -> ( + match operation.receipt with + | Empty -> + failwith "Cannot find operation status because metadata is empty" + | Too_large -> + failwith + "Cannot find operation status because metadata is too large" + | Receipt receipt -> + let*? status = operation_status_of_receipt receipt ~index in + return_some status) + + let dummy_sk_uri = + WithExceptions.Result.get_ok ~loc:__LOC__ + @@ Tezos_signer_backends.Unencrypted.make_sk + @@ Signature.Secret_key.of_b58check_exn + "edsk3UqeiQWXX7NFEY1wUs6J1t2ez5aQ3hEWdqX5Jr5edZiGLW8nZr" + + let simulate_operations cctxt ~force ~source ~src_pk ~successor_level + ~fee_parameter operations = + let open Lwt_result_syntax in + let fee_parameter : Injection.fee_parameter = + { + minimal_fees = Tez.of_mutez_exn fee_parameter.minimal_fees.mutez; + minimal_nanotez_per_byte = fee_parameter.minimal_nanotez_per_byte; + minimal_nanotez_per_gas_unit = + fee_parameter.minimal_nanotez_per_gas_unit; + force_low_fee = fee_parameter.force_low_fee; + fee_cap = Tez.of_mutez_exn fee_parameter.fee_cap.mutez; + burn_cap = Tez.of_mutez_exn fee_parameter.burn_cap.mutez; + } + in + let open Annotated_manager_operation in + let annotated_operations = + List.map + (fun operation -> + let (Manager operation) = to_manager_operation operation in + Annotated_manager_operation + (Injection.prepare_manager_operation + ~fee:Limit.unknown + ~gas_limit:Limit.unknown + ~storage_limit:Limit.unknown + operation)) + operations + in + let (Manager_list annot_op) = + Annotated_manager_operation.manager_of_list annotated_operations + in + let cctxt = + new Protocol_client_context.wrap_full (cctxt :> Client_context.full) + in + let*! simulation_result = + Injection.inject_manager_operation + cctxt + ~simulation:true (* Only simulation here *) + ~force + ~chain:cctxt#chain + ~block:(`Head 0) + ~source + ~src_pk + ~src_sk:dummy_sk_uri + (* Use dummy secret key as it is not used by simulation *) + ~successor_level + ~fee:Limit.unknown + ~gas_limit:Limit.unknown + ~storage_limit:Limit.unknown + ~fee_parameter + annot_op + in + match simulation_result with + | Error trace -> + let exceeds_quota = + TzTrace.fold + (fun exceeds -> function + | Environment.Ecoproto_error + (Gas.Block_quota_exceeded | Gas.Operation_quota_exceeded) -> + true + | _ -> exceeds) + false + trace + in + fail (if exceeds_quota then `Exceeds_quotas trace else `TzError trace) + | Ok (_oph, packed_op, _contents, results) -> + let nb_ops = List.length operations in + let results = Apply_results.to_list (Contents_result_list results) in + (* packed_op can have reveal operations added automatically. *) + let start_index = List.length results - nb_ops in + (* remove extra reveal operations *) + let operations_statuses = + List.fold_left_i + (fun index_in_batch acc (Apply_results.Contents_result result) -> + if index_in_batch < start_index then acc + else + {index_in_batch; status = operation_result_status result} :: acc) + [] + results + |> List.rev + in + let unsigned_operation = + let {shell; protocol_data = Operation_data {contents; signature = _}} + = + packed_op + in + (shell, Contents_list contents) + in + return {operations_statuses; unsigned_operation} + + let sign_operation cctxt src_sk + ((shell, Contents_list contents) as unsigned_op) = + let open Lwt_result_syntax in + let unsigned_bytes = + Data_encoding.Binary.to_bytes_exn Operation.unsigned_encoding unsigned_op + in + let cctxt = + new Protocol_client_context.wrap_full (cctxt :> Client_context.full) + in + let+ signature = + Client_keys.sign + cctxt + ~watermark:Signature.Generic_operation + src_sk + unsigned_bytes + in + let op : packed_operation = + { + shell; + protocol_data = Operation_data {contents; signature = Some signature}; + } + in + Data_encoding.Binary.to_bytes_exn Operation.encoding op + + let time_until_next_block {minimal_block_delay; delay_increment_per_round; _} + (header : Tezos_base.Block_header.shell_header option) = + let open Result_syntax in + match header with + | None -> minimal_block_delay |> Int64.to_int |> Ptime.Span.of_int_s + | Some header -> + let minimal_block_delay = Period.of_seconds_exn minimal_block_delay in + let delay_increment_per_round = + Period.of_seconds_exn delay_increment_per_round + in + let next_level_timestamp = + let* durations = + Round.Durations.create + ~first_round_duration:minimal_block_delay + ~delay_increment_per_round + in + let* predecessor_round = Fitness.round_from_raw header.fitness in + Round.timestamp_of_round + durations + ~predecessor_timestamp:header.timestamp + ~predecessor_round + ~round:Round.zero + in + let next_level_timestamp = + Result.value + next_level_timestamp + ~default: + (WithExceptions.Result.get_ok + ~loc:__LOC__ + Timestamp.(header.timestamp +? minimal_block_delay)) + in + Ptime.diff + (Time.System.of_protocol_exn next_level_timestamp) + (Time.System.now ()) + + let check_fee_parameters {fee_parameters; _} = + let check_value purpose name compare to_string mempool_default value = + if compare mempool_default value > 0 then + error_with + "Bad configuration fee_parameter.%s for %s. It must be at least %s \ + for operations of the injector to be propagated." + name + (Configuration.string_of_purpose purpose) + (to_string mempool_default) + else Ok () + in + let check purpose + { + Injector_sigs.minimal_fees; + minimal_nanotez_per_byte; + minimal_nanotez_per_gas_unit; + force_low_fee = _; + fee_cap = _; + burn_cap = _; + } = + let open Result_syntax in + let+ () = + check_value + purpose + "minimal_fees" + Int64.compare + Int64.to_string + (Protocol.Alpha_context.Tez.to_mutez + Plugin.Mempool.default_minimal_fees) + minimal_fees.mutez + and+ () = + check_value + purpose + "minimal_nanotez_per_byte" + Q.compare + Q.to_string + Plugin.Mempool.default_minimal_nanotez_per_byte + minimal_nanotez_per_byte + and+ () = + check_value + purpose + "minimal_nanotez_per_gas_unit" + Q.compare + Q.to_string + Plugin.Mempool.default_minimal_nanotez_per_gas_unit + minimal_nanotez_per_gas_unit + in + () + in + check Transaction fee_parameters + + let checks state = check_fee_parameters state +end + +let () = register_proto_client Protocol.hash (module Proto_client) diff --git a/src/proto_018_Proxford/lib_injector/dune b/src/proto_018_Proxford/lib_injector/dune index 14b8ab530ac2..c39673dd4ac2 100644 --- a/src/proto_018_Proxford/lib_injector/dune +++ b/src/proto_018_Proxford/lib_injector/dune @@ -7,7 +7,7 @@ (instrumentation (backend bisect_ppx)) (libraries octez-libs.tezos-base - tezos-protocol-018-Proxford + tezos-protocol-018-Proxford.protocol octez-injector octez-protocol-018-Proxford-libs.tezos-client octez-shell-libs.tezos-client-base -- GitLab From cc2862bc376a687669559609b6ebfda2d94f731f Mon Sep 17 00:00:00 2001 From: Victor Dumitrescu Date: Fri, 7 Jul 2023 12:03:44 +0200 Subject: [PATCH 09/10] Tezt/lib_tezos: add interface for injector server --- tezt/lib_tezos/constant.ml | 2 + tezt/lib_tezos/injector.ml | 203 ++++++++++++++++++++++++++++++++++++ tezt/lib_tezos/injector.mli | 42 ++++++++ 3 files changed, 247 insertions(+) create mode 100644 tezt/lib_tezos/injector.ml create mode 100644 tezt/lib_tezos/injector.mli diff --git a/tezt/lib_tezos/constant.ml b/tezt/lib_tezos/constant.ml index 193f9afef4c9..870e9f8bd2a8 100644 --- a/tezt/lib_tezos/constant.ml +++ b/tezt/lib_tezos/constant.ml @@ -123,3 +123,5 @@ let tz4_account : Account.key = section of the reference manual. *) let wasm_echo_kernel_boot_sector = "0061736d0100000001280760037f7f7f017f60027f7f017f60057f7f7f7f7f017f60017f0060017f017f60027f7f0060000002610311736d6172745f726f6c6c75705f636f72650a726561645f696e707574000011736d6172745f726f6c6c75705f636f72650c77726974655f6f7574707574000111736d6172745f726f6c6c75705f636f72650b73746f72655f77726974650002030504030405060503010001071402036d656d02000a6b65726e656c5f72756e00060aa401042a01027f41fa002f0100210120002f010021022001200247044041e4004112410041e400410010021a0b0b0800200041c4006b0b5001057f41fe002d0000210341fc002f0100210220002d0000210420002f0100210520011004210620042003460440200041016a200141016b10011a0520052002460440200041076a200610011a0b0b0b1d01017f41dc0141840241901c100021004184022000100541840210030b0b38050041e4000b122f6b65726e656c2f656e762f7265626f6f740041f8000b0200010041fa000b0200020041fc000b0200000041fe000b0101" + +let injector_server_path = "./octez-injector-server" diff --git a/tezt/lib_tezos/injector.ml b/tezt/lib_tezos/injector.ml new file mode 100644 index 000000000000..397bced4c275 --- /dev/null +++ b/tezt/lib_tezos/injector.ml @@ -0,0 +1,203 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Nomadic Labs *) +(* *) +(*****************************************************************************) + +module Parameters = struct + type persistent_state = { + runner : Runner.t option; + uri : Uri.t; + mutable pending_ready : unit option Lwt.u list; + data_dir : string; + node : Node.t; + client : Client.t; + endpoint : Client.endpoint; + } + + type session_state = {mutable ready : bool} + + let injector_path = Constant.injector_server_path + + let base_default_name = "injector" + + let default_uri () = + Uri.make ~scheme:"http" ~host:"127.0.0.1" ~port:(Port.fresh ()) () + + let default_colors = + Log.Color.[|BG.green ++ FG.blue; BG.green ++ FG.gray; BG.green ++ FG.blue|] +end + +open Parameters +include Daemon.Make (Parameters) + +let trigger_ready injector value = + let pending = injector.persistent_state.pending_ready in + injector.persistent_state.pending_ready <- [] ; + List.iter (fun pending -> Lwt.wakeup_later pending value) pending + +let set_ready injector = + (match injector.status with + | Not_running -> () + | Running status -> status.session_state.ready <- true) ; + trigger_ready injector (Some ()) + +let handle_readiness injector (event : event) = + if event.name = "injector_listening.v0" then set_ready injector + +let rpc_host injector = + Uri.host_with_default ~default:"127.0.0.1" injector.persistent_state.uri + +let rpc_port injector = Option.get @@ Uri.port injector.persistent_state.uri + +let data_dir injector = injector.persistent_state.data_dir + +let spawn_command injector = + Process.spawn ~name:injector.name ~color:injector.color injector.path + +let spawn_config_init injector signer = + let signer = Account.(signer.public_key_hash) in + let host_args = ["--address"; rpc_host injector] in + let port_args = + match Uri.port injector.persistent_state.uri with + | None -> [] + | Some port -> ["--port"; Int.to_string port] + in + let block_delay_args = ["--block-delay"; Float.to_string 0.1] in + let data_dir_args = ["--data-dir"; data_dir injector] in + let base_dir_args = + ["--base-dir"; Client.base_dir injector.persistent_state.client] + in + let arguments = + base_dir_args @ ["init-config"] @ [signer] @ host_args @ port_args + @ block_delay_args @ data_dir_args + in + spawn_command injector arguments + +let init_config injector (signer : Account.key) = + let process = spawn_config_init injector signer in + let* output = Process.check_and_read_stdout process in + match output =~* rex "Injector server configuration written in ([^\n]*)" with + | None -> failwith "Injector configuration initialization failed" + | Some filename -> return filename + +let create ?name ?color ?data_dir ?event_pipe ?uri ?runner node client = + let name = match name with None -> fresh_name () | Some name -> name in + let uri = + match uri with None -> Parameters.default_uri () | Some uri -> uri + in + let data_dir = + match data_dir with None -> Temp.dir name | Some dir -> dir + in + let endpoint = Client.Node node in + let injector = + create + ~path:injector_path + ?name:(Some name) + ?color + ?event_pipe + ?runner + {runner; uri; pending_ready = []; data_dir; endpoint; node; client} + in + on_event injector (handle_readiness injector) ; + injector + +let run injector = + (match injector.status with + | Not_running -> () + | Running _ -> Test.fail "injector %s is already running" injector.name) ; + let runner = injector.persistent_state.runner in + let base_dir_args = + ["--base-dir"; Client.base_dir injector.persistent_state.client] + in + let data_dir = injector.persistent_state.data_dir in + let endpoint_args = + [ + "--endpoint"; + Client.(string_of_endpoint injector.persistent_state.endpoint); + ] + in + let arguments = + base_dir_args @ endpoint_args @ ["run"; "--data-dir"; data_dir] + in + let on_terminate _ = + (* Cancel all [Ready] event listeners. *) + trigger_ready injector None ; + unit + in + run injector {ready = false} arguments ~on_terminate ?runner + +module RPC = struct + type status = + | Pending + | Injected of {injected_oph : string; injected_op_index : int} + | Included of { + included_oph : string; + included_op_index : int; + block : string; + level : int; + } + + let make ?data ?query_string = + RPC.make + ?data + ?query_string + ~get_host:rpc_host + ~get_port:rpc_port + ~get_scheme:(Fun.const "http") + + let add_pending_transaction ?parameters amount destination = + let operation = + `O + ([ + ("kind", `String "transaction"); + ("amount", `String (Int64.to_string amount)); + ("destination", `String destination); + ] + @ + match parameters with + | Some (entrypoint, value) -> + [ + ( "parameters", + `O + [("entrypoint", `String entrypoint); ("value", `String value)] + ); + ] + | None -> []) + in + let data : RPC_core.data = Data operation in + make ~data POST ["add_pending_transaction"] JSON.as_string + + let operation_status op_hash = + let query_string = [("op_hash", op_hash)] in + make ~query_string GET ["operation_status"] (fun json -> + Option.map + (fun status -> + let open JSON in + match as_object status with + | [("pending", _)] -> Pending + | [("injected_oph", oph); ("injected_op_index", op_index)] -> + Injected + { + injected_oph = oph |> as_string; + injected_op_index = op_index |> as_int; + } + | [ + ("included_oph", oph); + ("included_op_index", op_index); + ("block", block); + ("level", level); + ] -> + Included + { + included_oph = oph |> as_string; + included_op_index = op_index |> as_int; + block = block |> as_string; + level = level |> as_int; + } + | _ -> assert false) + (JSON.as_opt json)) + + let inject () = make GET ["inject"] (fun _ -> ()) +end diff --git a/tezt/lib_tezos/injector.mli b/tezt/lib_tezos/injector.mli new file mode 100644 index 000000000000..8515fce0bd71 --- /dev/null +++ b/tezt/lib_tezos/injector.mli @@ -0,0 +1,42 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Nomadic Labs *) +(* *) +(*****************************************************************************) + +type t + +val create : + ?name:string -> + ?color:Log.Color.t -> + ?data_dir:string -> + ?event_pipe:string -> + ?uri:Uri.t -> + ?runner:Runner.t -> + Node.t -> + Client.t -> + t + +val init_config : t -> Account.key -> string Lwt.t + +val run : t -> unit Lwt.t + +module RPC : sig + type status = + | Pending + | Injected of {injected_oph : string; injected_op_index : int} + | Included of { + included_oph : string; + included_op_index : int; + block : string; + level : int; + } + + val add_pending_transaction : + ?parameters:string * string -> int64 -> string -> (t, string) RPC_core.t + + val operation_status : string -> (t, status option) RPC_core.t + + val inject : unit -> (t, unit) RPC_core.t +end -- GitLab From 301de366078585ebba4dcf5a087f854cc20189d5 Mon Sep 17 00:00:00 2001 From: Victor Dumitrescu Date: Fri, 7 Jul 2023 12:04:00 +0200 Subject: [PATCH 10/10] Tezt: test injector server --- tezt/tests/injector_test.ml | 149 ++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 tezt/tests/injector_test.ml diff --git a/tezt/tests/injector_test.ml b/tezt/tests/injector_test.ml new file mode 100644 index 000000000000..f677ce0dc88b --- /dev/null +++ b/tezt/tests/injector_test.ml @@ -0,0 +1,149 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Nomadic Labs *) +(* *) +(*****************************************************************************) + +let add_pending_transaction = + (* Increase amount every time a transaction is added to the injector + * queue so that transactions have different hashes. + * TODO: https://gitlab.com/tezos/tezos/-/issues/6332 + * Fix injector so this workaround is no longer required. *) + let transaction_amount = ref 100L in + fun injector -> + let* injector_operation_hash = + RPC.call injector + @@ Injector.RPC.add_pending_transaction + !transaction_amount + Account.Bootstrap.keys.(1).public_key_hash + in + transaction_amount := Int64.succ !transaction_amount ; + return injector_operation_hash + +let add_pending_contract_call = + (* Increase the value passed in every contract call added to the injector + * queue so that transactions have different hashes. *) + let counter = ref 0L in + fun injector contract -> + let* injector_operation_hash = + RPC.call injector + @@ Injector.RPC.add_pending_transaction + ~parameters:(String.empty, Int64.to_string !counter) + 0L + contract + in + counter := Int64.succ !counter ; + return injector_operation_hash + +(* For a list of pending injector operations, query their status with the + * injector RPC, confirm the "included" status by checking the block in which + * operations are reported to have been included, and return the list of + * operations which are still pending. *) +let check_operation_status = + (* Cache the last queried block to avoid repeat RPC calls when checking + * multiple operations included in the same block. *) + let last_queried_block = ref None in + fun client injector ops -> + let open Injector.RPC in + let open JSON in + let check_included op_hash level = + let* ops_list = + match !last_queried_block with + | Some (last_level, ops_list) when level = last_level -> return ops_list + | _ -> + let* ops = + RPC.Client.call client + @@ RPC.get_chain_block_operations ~block:(string_of_int level) () + in + let ops_list = ops |=> 3 |> as_list in + last_queried_block := Some (level, ops_list) ; + return ops_list + in + (* TODO: this checks that the batch this operation is part of + * has been included, could check the individual operation + * in the batch too. *) + assert ( + List.exists (fun op -> op |-> "hash" |> as_string = op_hash) ops_list) ; + return () + in + + let check_status op_hash = + let* status = + RPC.call injector @@ Injector.RPC.operation_status op_hash + in + match status with + | Some Pending -> return [op_hash] + | Some (Injected _info) -> + (* Here, we could check that the operation is indeed in the mempool, + * but it would be a flaky test because by the time we query the mempool + * the operation might have already been injected. *) + return [op_hash] + | Some (Included info) -> + let* () = check_included info.included_oph info.level in + return [] + | None -> Test.fail "Uninjected operation no longer in injector queue" + in + let* () = Client.bake_for client in + let* remaining_ops = Lwt_list.map_p check_status ops in + return (List.flatten remaining_ops) + +let add_one_transaction_and_bake client injector ?contract ops = + let* inj_operation_hash = + match contract with + | None -> add_pending_transaction injector + | Some contract -> add_pending_contract_call injector contract + in + let* ops = check_operation_status client injector ops in + return (inj_operation_hash :: ops) + +let rec add_one_transaction_per_block client injector ?contract n_blocks ops = + if n_blocks = 0 then return ops + else + let* ops = add_one_transaction_and_bake client injector ?contract ops in + add_one_transaction_per_block client injector ?contract (n_blocks - 1) ops + +let test_injector : Protocol.t list -> unit = + Protocol.register_test + ~__FILE__ + ~title:"Injector daemon" + ~tags:["bin_injector"] + @@ fun protocol -> + let nodes_args = Node.[Synchronisation_threshold 0; Private_mode] in + let* node, client = + Client.init_with_protocol `Client ~protocol ~nodes_args () + in + let injector = Injector.create node client in + let* _config_file = + Injector.init_config injector Account.Bootstrap.keys.(0) + in + let* () = Injector.run injector in + let* () = Client.bake_for client in + + (* Originate a simple contract to make calls to *) + let* _alias, contract = + Client.originate_contract_at + ~amount:Tez.zero + ~src:Constant.bootstrap2.alias + client + ~burn_cap:Tez.one + ~init:"3" + ["attic"; "add1"] + protocol + in + + (* Bake 10 blocks: inject a transfer in each of the first 5, then + * inject a contract call in each of the following 5. + * The list of pending operations is threaded throughout in order to + * check their status after baking every block, removing those which + * have been injected. + * Lastly, check that all transactions have been included. *) + let* ops = add_one_transaction_per_block client injector 5 [] in + let* ops = add_one_transaction_per_block client injector ~contract 5 ops in + let* ops = check_operation_status client injector ops in + let* () = Client.bake_for client in + let* ops = check_operation_status client injector ops in + assert (ops = []) ; + unit + +let register ~protocols = test_injector protocols -- GitLab