From 5dcabe9108f6061fa497b9d6d9780f82dc902614 Mon Sep 17 00:00:00 2001 From: Thomas Letan Date: Thu, 24 Apr 2025 15:27:36 +0200 Subject: [PATCH] EVM Node: Add `--replicate` to `run sandbox` This argument can be used to have a sandbox endpoint shadowing an existing chain locally. This can be useful for a number of use cases, notably interacting with the chain in a sandbox environment that reflects changes happening on the network in real time, or evaluating the impact of a (future) kernel on realistic traffic (using `--kernel`). This was already useful to assert that D indeed address the gas price spikes we are seeing on Mainnet. --- etherlink/CHANGES_NODE.md | 3 +- etherlink/bin_node/config/configuration.mli | 2 + etherlink/bin_node/lib_dev/events.ml | 13 +++ etherlink/bin_node/lib_dev/events.mli | 4 + etherlink/bin_node/lib_dev/sequencer.ml | 104 +++++++++++++----- etherlink/bin_node/main.ml | 36 +++++- .../EVM node- list events regression.out | 15 +++ .../evm_sequencer.ml/EVM Node- man.out | 4 + src/lib_clic/tezos_clic.ml | 18 ++- src/lib_clic/tezos_clic.mli | 1 + 10 files changed, 168 insertions(+), 32 deletions(-) diff --git a/etherlink/CHANGES_NODE.md b/etherlink/CHANGES_NODE.md index f9c76c440529..7d7435ea274a 100644 --- a/etherlink/CHANGES_NODE.md +++ b/etherlink/CHANGES_NODE.md @@ -19,7 +19,8 @@ - Adds `--private-rpc-port` to `run observer` to enable the private RPC server from command-line. (!17762) - +- Adds `--replicate` to `run sandbox` to provide an easy way to make local + tests on a local endpoint with realistic traffic. (!17845) - Previously when submitting the same transaction multiple times, if the first fails to be included by the sequencer then all following will fails. Now when submitting multiple time the same transaction, diff --git a/etherlink/bin_node/config/configuration.mli b/etherlink/bin_node/config/configuration.mli index 3a6d194b0a87..62b9c9d4368f 100644 --- a/etherlink/bin_node/config/configuration.mli +++ b/etherlink/bin_node/config/configuration.mli @@ -443,3 +443,5 @@ val pp_time_between_blocks : Format.formatter -> time_between_blocks -> unit val describe : unit -> unit val pp_print_json : data_dir:string -> Format.formatter -> t -> unit + +val observer_evm_node_endpoint : supported_network -> string diff --git a/etherlink/bin_node/lib_dev/events.ml b/etherlink/bin_node/lib_dev/events.ml index 23a26b93e429..3e46667702d9 100644 --- a/etherlink/bin_node/lib_dev/events.ml +++ b/etherlink/bin_node/lib_dev/events.ml @@ -202,6 +202,16 @@ let replay_csv_available = ~level:Notice ("path", Data_encoding.string) +let replicate_transaction_dropped = + Internal_event.Simple.declare_2 + ~section + ~name:"replicate_transaction_dropped" + ~msg:"transaction {hash} was dropped because it is now invalid ({reason})" + ~level:Warning + ~pp1:Ethereum_types.pp_hash + ("hash", Ethereum_types.hash_encoding) + ("reason", Data_encoding.string) + type kernel_log_kind = Application | Simulation type kernel_log_level = Debug | Info | Error | Fatal @@ -532,3 +542,6 @@ let import_finished () = emit import_finished () let import_snapshot_archive_in_progress ~archive_name ~elapsed_time = emit import_snapshot_archive_in_progress (archive_name, elapsed_time) + +let replicate_transaction_dropped hash reason = + emit replicate_transaction_dropped (hash, reason) diff --git a/etherlink/bin_node/lib_dev/events.mli b/etherlink/bin_node/lib_dev/events.mli index a37210effe87..59e46931ae0c 100644 --- a/etherlink/bin_node/lib_dev/events.mli +++ b/etherlink/bin_node/lib_dev/events.mli @@ -171,3 +171,7 @@ val import_finished : unit -> unit Lwt.t explicitly mentions the time elapsed since the extraction started. *) val import_snapshot_archive_in_progress : archive_name:string -> elapsed_time:Time.System.Span.t -> unit Lwt.t + +(** [replicate_transaction_dropped hash reason] advertises that the transaction + [hash] was dropped because it is now invalid in the sandbox. *) +val replicate_transaction_dropped : Ethereum_types.hash -> string -> unit Lwt.t diff --git a/etherlink/bin_node/lib_dev/sequencer.ml b/etherlink/bin_node/lib_dev/sequencer.ml index 31c42404cf6e..dc2c7f946b4b 100644 --- a/etherlink/bin_node/lib_dev/sequencer.ml +++ b/etherlink/bin_node/lib_dev/sequencer.ml @@ -12,6 +12,7 @@ type sandbox_config = { init_from_snapshot : string option; network : Configuration.supported_network option; funded_addresses : Ethereum_types.address list; + parent_chain : Uri.t option; } let install_finalizer_seq server_public_finalizer server_private_finalizer @@ -30,32 +31,79 @@ let install_finalizer_seq server_public_finalizer server_private_finalizer let* () = Signals_publisher.shutdown () in return_unit -let loop_sequencer (sequencer_config : Configuration.sequencer) = +let loop_sequencer backend + (module Tx_container : Services_backend_sig.Tx_container) ?sandbox_config + time_between_blocks = let open Lwt_result_syntax in - let time_between_blocks = sequencer_config.time_between_blocks in - match time_between_blocks with - | Nothing -> - (* Bind on a never-resolved promise ensures this call never returns, - meaning no block will ever be produced. *) - let task, _resolver = Lwt.task () in - let*! () = task in - return_unit - | Time_between_blocks time_between_blocks -> - let rec loop last_produced_block = - let now = Misc.now () in - (* We force if the last produced block is older than [time_between_blocks]. *) - let force = - let diff = Time.Protocol.(diff now last_produced_block) in - diff >= Int64.of_float time_between_blocks - in - let* has_produced_block = - Block_producer.produce_block ~force ~timestamp:now - and* () = Lwt.map Result.ok @@ Lwt_unix.sleep 0.5 in - match has_produced_block with - | `Block_produced _nb_transactions -> loop now - | `No_block -> loop last_produced_block + match sandbox_config with + | Some {parent_chain = Some evm_node_endpoint; _} -> + let*! head = Evm_context.head_info () in + Blueprints_follower.start + ~ping_tx_pool:false + ~time_between_blocks + ~evm_node_endpoint + ~next_blueprint_number:head.next_blueprint_number + @@ fun (Qty number) blueprint -> + let*! {next_blueprint_number = Qty expected_number; _} = + Evm_context.head_info () in - loop Misc.(now ()) + if Compare.Z.(number = expected_number) then + let events = + Evm_events.of_parts + blueprint.delayed_transactions + blueprint.kernel_upgrade + in + let* () = Evm_context.apply_evm_events events in + let*? all_txns = + Blueprint_decoder.transactions blueprint.blueprint.payload + in + let txns = List.filter_map snd all_txns in + let* () = + List.iter_es + (fun raw_tx -> + let* res = Validate.is_tx_valid backend ~mode:Stateless raw_tx in + match res with + | Ok (next_nonce, txn_obj) -> + let raw_tx = Ethereum_types.hex_of_utf8 raw_tx in + let* _ = Tx_container.add ~next_nonce txn_obj ~raw_tx in + return_unit + | Error reason -> + let hash = Ethereum_types.hash_raw_tx raw_tx in + let*! () = Events.replicate_transaction_dropped hash reason in + return_unit) + txns + in + let* _ = + Block_producer.produce_block + ~force:true + ~timestamp:blueprint.blueprint.timestamp + in + return `Continue + else return (`Restart_from (Ethereum_types.Qty expected_number)) + | _ -> ( + match time_between_blocks with + | Configuration.Nothing -> + (* Bind on a never-resolved promise ensures this call never returns, + meaning no block will ever be produced. *) + let task, _resolver = Lwt.task () in + let*! () = task in + return_unit + | Time_between_blocks time_between_blocks -> + let rec loop last_produced_block = + let now = Misc.now () in + (* We force if the last produced block is older than [time_between_blocks]. *) + let force = + let diff = Time.Protocol.(diff now last_produced_block) in + diff >= Int64.of_float time_between_blocks + in + let* has_produced_block = + Block_producer.produce_block ~force ~timestamp:now + and* () = Lwt.map Result.ok @@ Lwt_unix.sleep 0.5 in + match has_produced_block with + | `Block_produced _nb_transactions -> loop now + | `No_block -> loop last_produced_block + in + loop Misc.(now ())) let main ~data_dir ?(genesis_timestamp = Misc.now ()) ~cctxt ~(configuration : Configuration.t) ?kernel ?sandbox_config () = @@ -342,5 +390,11 @@ let main ~data_dir ?(genesis_timestamp = Misc.now ()) ~cctxt finalizer_private_server finalizer_rpc_process in - let* () = loop_sequencer sequencer_config in + let* () = + loop_sequencer + backend + tx_container + ?sandbox_config + sequencer_config.time_between_blocks + in return_unit diff --git a/etherlink/bin_node/main.ml b/etherlink/bin_node/main.ml index 267a5a4ec2c2..71a51146e78c 100644 --- a/etherlink/bin_node/main.ml +++ b/etherlink/bin_node/main.ml @@ -76,6 +76,11 @@ module Params = struct let endpoint = Tezos_clic.parameter (fun _ uri -> Lwt.return_ok (Uri.of_string uri)) + let optional_endpoint = + Tezos_clic.parameter (fun _ uri -> + let open Lwt_result_syntax in + if uri = "" then return_none else return_some (Uri.of_string uri)) + let event_level = Tezos_clic.parameter (fun _ value -> Lwt.return_ok (Internal_event.Level.of_string_exn value)) @@ -555,6 +560,18 @@ let evm_node_endpoint_arg = ~doc:"The address of an EVM node to connect to." Params.evm_node_endpoint +let replicate_arg = + Tezos_clic.arg_or_switch + ~pp_default:(fun fmt -> + Format.fprintf fmt "the official relay endpoint if --network is used") + ~long:"replicate" + ~placeholder:"url" + ~default:"" + ~doc: + "Replicate a chain in real time from the EVM node whose address is \ + provided." + Params.optional_endpoint + let evm_node_private_endpoint_arg = Tezos_clic.arg ~long:"evm-node-private-endpoint" @@ -2266,7 +2283,7 @@ let fund_arg = Tezos_clic.multiple_arg ~long ~doc ~placeholder:"0x..." Params.eth_address let sandbox_config_args = - Tezos_clic.args13 + Tezos_clic.args14 preimages_arg preimages_endpoint_arg native_execution_policy_arg @@ -2280,6 +2297,7 @@ let sandbox_config_args = (supported_network_arg ()) init_from_snapshot_arg fund_arg + replicate_arg let sequencer_command = let open Tezos_clic in @@ -2406,7 +2424,8 @@ let sandbox_command = password_filename, network, init_from_snapshot, - funded_addresses ) ) + funded_addresses, + main_endpoint ) ) () -> let open Lwt_result_syntax in let* restricted_rpcs = @@ -2419,6 +2438,18 @@ let sandbox_command = Option.value ~default:Uri.empty rollup_node_endpoint in let kernel = kernel_from_args network kernel in + let* parent_chain = + match (main_endpoint, network) with + | Some (Some endpoint), _ -> return_some endpoint + | Some None, Some network -> + return_some + (Uri.of_string (Configuration.observer_evm_node_endpoint network)) + | Some None, None -> + failwith + "Cannot infer which EVM node to use to fetch blueprints. Use \ + --network or add an endpoint as the argument of --replicate." + | None, _ -> return_none + in let sandbox_config = Evm_node_lib_dev.Sequencer. { @@ -2427,6 +2458,7 @@ let sandbox_command = init_from_snapshot; network; funded_addresses = Option.value ~default:[] funded_addresses; + parent_chain; } in let config_file = config_filename ~data_dir config_file in diff --git a/etherlink/tezt/tests/expected/evm_rollup.ml/EVM node- list events regression.out b/etherlink/tezt/tests/expected/evm_rollup.ml/EVM node- list events regression.out index 593c26a4db71..06fa5646ac77 100644 --- a/etherlink/tezt/tests/expected/evm_rollup.ml/EVM node- list events regression.out +++ b/etherlink/tezt/tests/expected/evm_rollup.ml/EVM node- list events regression.out @@ -1410,6 +1410,21 @@ replay_csv_available: contain invalid byte sequences. */ string || { "invalid_utf8_string": [ integer ∈ [0, 255] ... ] } +replicate_transaction_dropped: + description: transaction {hash} was dropped because it is now invalid ({reason}) + level: warning + section: evm_node.dev + json format: + { /* replicate_transaction_dropped version 0 */ + "replicate_transaction_dropped.v0": + { "hash": $unistring, + "reason": $unistring } } + $unistring: + /* Universal string representation + Either a plain UTF8 string, or a sequence of bytes for strings that + contain invalid byte sequences. */ + string || { "invalid_utf8_string": [ integer ∈ [0, 255] ... ] } + request_completed_debug: description: {view} {worker_status} level: debug diff --git a/etherlink/tezt/tests/expected/evm_sequencer.ml/EVM Node- man.out b/etherlink/tezt/tests/expected/evm_sequencer.ml/EVM Node- man.out index 7e59394cec39..0cb6875f6d78 100644 --- a/etherlink/tezt/tests/expected/evm_sequencer.ml/EVM Node- man.out +++ b/etherlink/tezt/tests/expected/evm_sequencer.ml/EVM Node- man.out @@ -47,6 +47,7 @@ Run commands: [--kernel ] [-d --wallet-dir ] [-f --password-filename ] [--network ] [--init-from-snapshot [snapshot url]] [--fund <0x...>] + [--replicate [url]] Start the EVM node in sandbox mode. The sandbox mode is a sequencer-like mode that produces blocks with a fake key and no rollup node connection. --data-dir : The path to the EVM node data directory. @@ -125,6 +126,9 @@ Run commands: `https://snapshotter-sandbox.nomadic-labs.eu/local/etherlink-%n/%h/etherlink-%n-%h-latest.gz`. --fund <0x...>: The address of an account to provide with funds in the sandbox (can be repeated to fund multiple accounts). + --replicate [url]: Replicate a chain in real time from the EVM node whose + address is provided. + Defaults to `the official relay endpoint if --network is used`. run proxy [--data-dir ] [--config-file ] [--rpc-addr ] [--rpc-port ] [--rpc-batch-limit ] diff --git a/src/lib_clic/tezos_clic.ml b/src/lib_clic/tezos_clic.ml index 69662fb36a3a..00395c5c56c5 100644 --- a/src/lib_clic/tezos_clic.ml +++ b/src/lib_clic/tezos_clic.ml @@ -110,6 +110,7 @@ type ('a, 'ctx) arg = placeholder : string; kind : ('p, 'ctx) parameter; default : string; + pp_default : (Format.formatter -> unit) option; } -> ('p option, 'ctx) arg | Switch : {label : label; doc : string} -> (bool, 'ctx) arg @@ -258,7 +259,15 @@ let rec print_options_detailed : placeholder print_desc doc - | ArgDefSwitch {label; placeholder; doc; default; _} -> + | ArgDefSwitch {label; placeholder; doc; default; pp_default; _} -> + let pp_default = + match pp_default with + | Some pp -> fun fmt _s -> pp fmt + | None -> Format.pp_print_string + in + let doc = + Format.asprintf "%s\nDefaults to `%a`." doc pp_default default + in Format.fprintf ppf "@[@{%a [%s]@}: %a@]" @@ -266,7 +275,7 @@ let rec print_options_detailed : label placeholder print_desc - (doc ^ "\nDefaults to `" ^ default ^ "`.") + doc | Switch {label; doc} -> Format.fprintf ppf @@ -886,8 +895,9 @@ let default_arg ~doc ?short ~long ~placeholder ~default ?pp_default ?env kind = DefArg {doc; placeholder; label = {long; short}; kind; env; default; pp_default} -let arg_or_switch ~doc ?short ~long ~placeholder ~default kind = - ArgDefSwitch {doc; placeholder; label = {long; short}; kind; default} +let arg_or_switch ~doc ?short ~long ~placeholder ~default ?pp_default kind = + ArgDefSwitch + {doc; placeholder; label = {long; short}; kind; default; pp_default} let map_arg ~f:converter spec = Map {spec; converter} diff --git a/src/lib_clic/tezos_clic.mli b/src/lib_clic/tezos_clic.mli index 5f2e7048f49d..849940a8a65f 100644 --- a/src/lib_clic/tezos_clic.mli +++ b/src/lib_clic/tezos_clic.mli @@ -141,6 +141,7 @@ val arg_or_switch : long:string -> placeholder:string -> default:string -> + ?pp_default:(Format.formatter -> unit) -> ('a, 'ctx) parameter -> ('a option, 'ctx) arg -- GitLab