From 019a78553e15e00cae123f1850fa78f722c00a8b Mon Sep 17 00:00:00 2001 From: Pierrick Couderc Date: Wed, 2 Jul 2025 09:53:43 +0200 Subject: [PATCH 1/5] Tezt/Cloud/DAL: echo rollup metrics --- tezt/tests/cloud/dal.ml | 45 +++++++++++++++++++++ tezt/tests/cloud/echo_rollup.mli | 8 +++- tezt/tests/cloud/metrics.ml | 67 +++++++++++++++++++++++++++++++- tezt/tests/cloud/metrics.mli | 1 + 4 files changed, 118 insertions(+), 3 deletions(-) diff --git a/tezt/tests/cloud/dal.ml b/tezt/tests/cloud/dal.ml index ce80214876d9..4a18b3ec64dc 100644 --- a/tezt/tests/cloud/dal.ml +++ b/tezt/tests/cloud/dal.ml @@ -88,6 +88,49 @@ type t = { otel : string option; } +let get_echo_rollup t ~level = + let slots = Hashtbl.create 32 in + let read_slot (echo_operator : Echo_rollup.operator) slot = + let key = "/output/slot-" ^ string_of_int slot in + let rpc = + Sc_rollup_rpc.get_global_block_durable_state_value + ~pvm_kind:"wasm_2_0_0" + ~operation:Sc_rollup_rpc.Value + ~key + ~block:(string_of_int level) + () + in + let* resp = Sc_rollup_node.RPC.call_json echo_operator.Echo_rollup.sc_rollup_node rpc in + let size = + (* If the RPC fails for whatever reason (mainly because the level asked is + below the origination level), do not make the test crash, simply return + 0. *) + if not (Cohttp.Code.is_success resp.code) then 0 + else + match rpc.decode resp.body with + | None -> 0 + | Some bytes -> `Hex bytes |> Hex.to_string |> Z.of_bits |> Z.to_int + in + Hashtbl.add slots slot size ; + unit + in + let* () = + match t.echo_rollup with + | None -> unit + | Some echo_rollup -> + (* Ensure the rollup is at the expected level before updating. *) + let* _level = + Sc_rollup_node.wait_for_level + ~timeout:30. + echo_rollup.sc_rollup_node + level + in + Lwt_list.iter_s + (read_slot echo_rollup) + t.configuration.dal_node_producers + in + return slots + let get_infos_per_level t ~level ~metadata = let open Metrics in let client = t.bootstrap.client in @@ -221,6 +264,7 @@ let get_infos_per_level t ~level ~metadata = unit else Lwt.return_unit in + let* echo_rollup_fetched_data = get_echo_rollup t ~level in Lwt.return { level; @@ -228,6 +272,7 @@ let get_infos_per_level t ~level ~metadata = attestations; attested_commitments; etherlink_operator_balance_sum; + echo_rollup_fetched_data; } let init_teztale (configuration : configuration) cloud agent = diff --git a/tezt/tests/cloud/echo_rollup.mli b/tezt/tests/cloud/echo_rollup.mli index 97d4f18a1025..33fea99e6f32 100644 --- a/tezt/tests/cloud/echo_rollup.mli +++ b/tezt/tests/cloud/echo_rollup.mli @@ -13,7 +13,13 @@ messages published via DAL slots. *) -type operator +type operator = { + node : Node.t; + client : Client.t; + sc_rollup_node : Sc_rollup_node.t; + sc_rollup_address : string; + operator : Account.key; +} (** [init_echo_rollup_account ~client ~echo_rollup ~alias_prefix] generates a new rollup operator key if [~echo_rollup] is [true]. The key alias will diff --git a/tezt/tests/cloud/metrics.ml b/tezt/tests/cloud/metrics.ml index 0198c7600946..fd7cce884919 100644 --- a/tezt/tests/cloud/metrics.ml +++ b/tezt/tests/cloud/metrics.ml @@ -22,6 +22,7 @@ type per_level_info = { attestations : (public_key_hash, dal_status) Hashtbl.t; attested_commitments : Z.t; etherlink_operator_balance_sum : Tez.t; + echo_rollup_fetched_data : (int, int) Hashtbl.t; } type t = { @@ -42,6 +43,9 @@ type t = { ratio_attested_commitments_per_baker : (public_key_hash, Baker_helpers.per_baker_dal_summary) Hashtbl.t; etherlink_operator_balance_sum : Tez.t; + total_echo_rollup_unattested_slots : int; + total_echo_rollup_fetched_data_size : int; + last_echo_rollup_fetched_data_size : int; } let default = @@ -58,6 +62,9 @@ let default = ratio_published_commitments_last_level = 0.; ratio_attested_commitments_per_baker = Hashtbl.create 0; etherlink_operator_balance_sum = Tez.zero; + total_echo_rollup_unattested_slots = 0; + total_echo_rollup_fetched_data_size = 0; + last_echo_rollup_fetched_data_size = 0; } let aliases = @@ -97,6 +104,9 @@ let pp ~bakers ratio_published_commitments_last_level; ratio_attested_commitments_per_baker; etherlink_operator_balance_sum; + total_echo_rollup_unattested_slots; + total_echo_rollup_fetched_data_size; + last_echo_rollup_fetched_data_size; } = let pp_ratio fmt (num, div) = if div = 0 then Format.fprintf fmt "Not a number: %d/0" num @@ -158,7 +168,16 @@ let pp ~bakers "DAL slots: total attested commitments per slot ( -> ).@.%a" pp_slot_metrics - (Hashtbl.to_seq total_attested_commitments_per_slot) + (Hashtbl.to_seq total_attested_commitments_per_slot) ; + Log.info + "Echo rollup slots: total unattested slots: %d" + total_echo_rollup_unattested_slots ; + Log.info + "Echo rollup slots: total fetched data: %d" + total_echo_rollup_fetched_data_size ; + Log.info + "Echo rollup slots: last fetched data: %d" + last_echo_rollup_fetched_data_size let push ~versions ~cloud { @@ -174,6 +193,9 @@ let push ~versions ~cloud ratio_published_commitments_last_level; ratio_attested_commitments_per_baker; etherlink_operator_balance_sum; + total_echo_rollup_unattested_slots; + total_echo_rollup_fetched_data_size; + last_echo_rollup_fetched_data_size; } = let get_labels public_key_hash = let alias = @@ -316,7 +338,28 @@ let push ~versions ~cloud ~help:"Sum of the balances of the etherlink operator" ~typ:`Gauge ~name:"tezt_etherlink_operator_balance_total" - (Tez.to_float etherlink_operator_balance_sum) + (Tez.to_float etherlink_operator_balance_sum); + Cloud.push_metric + cloud + ~help:"Number of slots unattested from the echo rollup perspective" + ~typ:`Gauge + ~labels:[("kind", "unattested")] + ~name:"tezt_total_echo_rollup_unattested_slots" + (float_of_int total_echo_rollup_unattested_slots) ; + Cloud.push_metric + cloud + ~help:"Total size of data fetched by the echo rollup" + ~typ:`Gauge + ~labels:[("kind", "data")] + ~name:"tezt_total_echo_rollup_fetched_data" + (float_of_int total_echo_rollup_fetched_data_size) ; + Cloud.push_metric + cloud + ~help:"Last size of data fetched by the echo rollup" + ~typ:`Gauge + ~labels:[("kind", "data")] + ~name:"tezt_last_echo_rollup_fetched_data" + (float_of_int last_echo_rollup_fetched_data_size) let published_level_of_attested_level ~attestation_lag level = level - attestation_lag @@ -524,6 +567,18 @@ let update_ratio_attested_commitments_per_baker ~first_level ~infos |> Hashtbl.add_seq table ; table +let update_echo_rollup_metrics infos_per_level metrics = + let data_size, unattested_slots = + Hashtbl.fold + (fun _ v (total_size, unattested) -> + (total_size + v, if v = 0 then unattested + 1 else unattested)) + infos_per_level.echo_rollup_fetched_data + (0, 0) + in + ( metrics.total_echo_rollup_unattested_slots + unattested_slots, + metrics.total_echo_rollup_fetched_data_size + data_size, + data_size ) + let get ~first_level ~attestation_lag ~dal_node_producers ~number_of_slots ~infos infos_per_level metrics = let level_first_commitment_published = @@ -604,6 +659,11 @@ let get ~first_level ~attestation_lag ~dal_node_producers ~number_of_slots metrics.total_published_commitments_per_slot metrics.total_attested_commitments_per_slot in + let ( total_echo_rollup_unattested_slots, + total_echo_rollup_fetched_data_size, + last_echo_rollup_fetched_data_size ) = + update_echo_rollup_metrics infos_per_level metrics + in { level_first_commitment_published; level_first_commitment_attested; @@ -618,4 +678,7 @@ let get ~first_level ~attestation_lag ~dal_node_producers ~number_of_slots ratio_attested_commitments_per_baker; etherlink_operator_balance_sum = infos_per_level.etherlink_operator_balance_sum; + total_echo_rollup_unattested_slots; + total_echo_rollup_fetched_data_size; + last_echo_rollup_fetched_data_size; } diff --git a/tezt/tests/cloud/metrics.mli b/tezt/tests/cloud/metrics.mli index 90d9564ffa02..0aa7a591b47c 100644 --- a/tezt/tests/cloud/metrics.mli +++ b/tezt/tests/cloud/metrics.mli @@ -38,6 +38,7 @@ type per_level_info = { attestations : (public_key_hash, dal_status) Hashtbl.t; attested_commitments : Z.t; etherlink_operator_balance_sum : Tez.t; + echo_rollup_fetched_data : (int, int) Hashtbl.t; } type t -- GitLab From 63838289f2c3723d66005b1cbaf6f265386e6d9c Mon Sep 17 00:00:00 2001 From: Pierrick Couderc Date: Mon, 7 Jul 2025 09:46:58 +0200 Subject: [PATCH 2/5] Tezt/Cloud: add recho rollup origination level --- tezt/tests/cloud/dal.ml | 50 +++++++++++++++----------------- tezt/tests/cloud/echo_rollup.ml | 20 ++++++++++++- tezt/tests/cloud/echo_rollup.mli | 1 + 3 files changed, 43 insertions(+), 28 deletions(-) diff --git a/tezt/tests/cloud/dal.ml b/tezt/tests/cloud/dal.ml index 4a18b3ec64dc..88a60595bb39 100644 --- a/tezt/tests/cloud/dal.ml +++ b/tezt/tests/cloud/dal.ml @@ -90,26 +90,21 @@ type t = { let get_echo_rollup t ~level = let slots = Hashtbl.create 32 in - let read_slot (echo_operator : Echo_rollup.operator) slot = + let read_slot echo_operator slot = let key = "/output/slot-" ^ string_of_int slot in - let rpc = - Sc_rollup_rpc.get_global_block_durable_state_value - ~pvm_kind:"wasm_2_0_0" - ~operation:Sc_rollup_rpc.Value - ~key - ~block:(string_of_int level) - () + let* value_written = + Sc_rollup_node.RPC.call echo_operator.Echo_rollup.sc_rollup_node + @@ Sc_rollup_rpc.get_global_block_durable_state_value + ~pvm_kind:"wasm_2_0_0" + ~operation:Sc_rollup_rpc.Value + ~key + ~block:(string_of_int level) + () in - let* resp = Sc_rollup_node.RPC.call_json echo_operator.Echo_rollup.sc_rollup_node rpc in let size = - (* If the RPC fails for whatever reason (mainly because the level asked is - below the origination level), do not make the test crash, simply return - 0. *) - if not (Cohttp.Code.is_success resp.code) then 0 - else - match rpc.decode resp.body with - | None -> 0 - | Some bytes -> `Hex bytes |> Hex.to_string |> Z.of_bits |> Z.to_int + match value_written with + | None -> 0 + | Some bytes -> `Hex bytes |> Hex.to_string |> Z.of_bits |> Z.to_int in Hashtbl.add slots slot size ; unit @@ -118,16 +113,17 @@ let get_echo_rollup t ~level = match t.echo_rollup with | None -> unit | Some echo_rollup -> - (* Ensure the rollup is at the expected level before updating. *) - let* _level = - Sc_rollup_node.wait_for_level - ~timeout:30. - echo_rollup.sc_rollup_node - level - in - Lwt_list.iter_s - (read_slot echo_rollup) - t.configuration.dal_node_producers + if level < echo_rollup.Echo_rollup.origination_level then unit + else + let* _level = + Sc_rollup_node.wait_for_level + ~timeout:30. + echo_rollup.sc_rollup_node + level + in + Lwt_list.iter_s + (read_slot echo_rollup) + t.configuration.dal_node_producers in return slots diff --git a/tezt/tests/cloud/echo_rollup.ml b/tezt/tests/cloud/echo_rollup.ml index e281b6be76dc..534e11bc8cf6 100644 --- a/tezt/tests/cloud/echo_rollup.ml +++ b/tezt/tests/cloud/echo_rollup.ml @@ -17,6 +17,7 @@ type operator = { sc_rollup_node : Sc_rollup_node.t; sc_rollup_address : string; operator : Account.key; + origination_level : int; } let init_echo_rollup_account ~client ~echo_rollup ~alias_prefix = @@ -165,6 +166,8 @@ let init_echo_rollup cloud ~data_dir ~simulate_network ~external_rpc ~network client in let () = toplog "Init Echo rollup: waiting again, for level %d" (l + 2) in + (* The origination level is an overapproximation, it prevents metrics to fail + if they attempt to read the rollup storage before its origination level. *) let* _ = Node.wait_for_level node (l + 2) in let () = toplog "Init Echo rollup: waiting again, for level %d: done" (l + 2) @@ -173,8 +176,23 @@ let init_echo_rollup cloud ~data_dir ~simulate_network ~external_rpc ~network let* () = Sc_rollup_node.run sc_rollup_node sc_rollup_address [Log_kernel_debug] in + let* genesis_info = + Client.RPC.call client + @@ RPC.get_chain_block_context_smart_rollups_smart_rollup_genesis_info + sc_rollup_address + in + let origination_level = JSON.(genesis_info |-> "level" |> as_int) in let () = toplog "Init Echo rollup: launching the rollup node: done" in - let operator = {node; client; sc_rollup_node; operator; sc_rollup_address} in + let operator = + { + node; + client; + sc_rollup_node; + operator; + sc_rollup_address; + origination_level; + } + in let* () = add_prometheus_source ?dal_node diff --git a/tezt/tests/cloud/echo_rollup.mli b/tezt/tests/cloud/echo_rollup.mli index 33fea99e6f32..0904a9d47c33 100644 --- a/tezt/tests/cloud/echo_rollup.mli +++ b/tezt/tests/cloud/echo_rollup.mli @@ -19,6 +19,7 @@ type operator = { sc_rollup_node : Sc_rollup_node.t; sc_rollup_address : string; operator : Account.key; + origination_level : int; } (** [init_echo_rollup_account ~client ~echo_rollup ~alias_prefix] generates -- GitLab From 964ba0f8a93fbc92418a3d3da9f15180d7832e34 Mon Sep 17 00:00:00 2001 From: Pierrick Couderc Date: Tue, 8 Jul 2025 14:23:05 +0200 Subject: [PATCH 3/5] Tezt: fix sc_rollup_node endpoint when using an external runner --- tezt/lib_tezos/sc_rollup_node.ml | 8 ++++++-- tezt/lib_tezos/sc_rollup_node.mli | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tezt/lib_tezos/sc_rollup_node.ml b/tezt/lib_tezos/sc_rollup_node.ml index 30f499791632..d549a5748292 100644 --- a/tezt/lib_tezos/sc_rollup_node.ml +++ b/tezt/lib_tezos/sc_rollup_node.ml @@ -808,10 +808,14 @@ let import_snapshot ?(apply_unsafe_patches = false) ?(force = false) in Runnable.{value = process; run = Process.check} -let as_rpc_endpoint (t : t) = +let as_rpc_endpoint ?(local = false) (t : t) = let state = t.persistent_state in let scheme = "http" in - Endpoint.make ~scheme ~host:state.rpc_host ~port:state.rpc_port () + let host = + if local || Option.is_none t.persistent_state.runner then state.rpc_host + else Runner.address t.persistent_state.runner + in + Endpoint.make ~scheme ~host ~port:state.rpc_port () let operators t = (t.persistent_state.default_operator, t.persistent_state.operators) diff --git a/tezt/lib_tezos/sc_rollup_node.mli b/tezt/lib_tezos/sc_rollup_node.mli index efec2e688794..5c2377172317 100644 --- a/tezt/lib_tezos/sc_rollup_node.mli +++ b/tezt/lib_tezos/sc_rollup_node.mli @@ -403,7 +403,7 @@ val list_metrics : unit Lwt.t (** Expose the RPC server address of this node as a foreign endpoint. *) -val as_rpc_endpoint : t -> Endpoint.t +val as_rpc_endpoint : ?local:bool -> t -> Endpoint.t (** [operators node] returns a pair with the default operator and all other operators with their purpose. *) -- GitLab From 23288b7a243f88941639f703dca298a89d6e4f7f Mon Sep 17 00:00:00 2001 From: Guillaume Genestier Date: Thu, 24 Jul 2025 15:48:36 +0200 Subject: [PATCH 4/5] DAL-echo-rollup: Remove the return to avoid the function to fail prematurely, making the metric accurate. --- src/kernel_dal_echo/kernel_bandwidth/src/lib.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/kernel_dal_echo/kernel_bandwidth/src/lib.rs b/src/kernel_dal_echo/kernel_bandwidth/src/lib.rs index 563b057ed255..4aba28dfa485 100644 --- a/src/kernel_dal_echo/kernel_bandwidth/src/lib.rs +++ b/src/kernel_dal_echo/kernel_bandwidth/src/lib.rs @@ -37,7 +37,6 @@ fn process_slot( slot_index, published_level ); - return; } Ok(num) => { debug_msg!( @@ -57,8 +56,6 @@ fn process_slot( slot_index, &err.to_string() ); - // Stop fetching pages on error - return; } } } -- GitLab From 41c9c2ad531481f1ca22ca3ed83d31824fdd4f19 Mon Sep 17 00:00:00 2001 From: Guillaume Genestier Date: Fri, 5 Sep 2025 10:27:11 +0200 Subject: [PATCH 5/5] Tezt-cloud/DAL: Move get_echo_rollup function to the Echo_rollup module and rename it --- tezt/tests/cloud/dal.ml | 46 +++++--------------------------- tezt/tests/cloud/echo_rollup.ml | 37 +++++++++++++++++++++++++ tezt/tests/cloud/echo_rollup.mli | 9 +++++++ tezt/tests/cloud/metrics.ml | 2 +- 4 files changed, 53 insertions(+), 41 deletions(-) diff --git a/tezt/tests/cloud/dal.ml b/tezt/tests/cloud/dal.ml index 88a60595bb39..d713a5b225a6 100644 --- a/tezt/tests/cloud/dal.ml +++ b/tezt/tests/cloud/dal.ml @@ -88,45 +88,6 @@ type t = { otel : string option; } -let get_echo_rollup t ~level = - let slots = Hashtbl.create 32 in - let read_slot echo_operator slot = - let key = "/output/slot-" ^ string_of_int slot in - let* value_written = - Sc_rollup_node.RPC.call echo_operator.Echo_rollup.sc_rollup_node - @@ Sc_rollup_rpc.get_global_block_durable_state_value - ~pvm_kind:"wasm_2_0_0" - ~operation:Sc_rollup_rpc.Value - ~key - ~block:(string_of_int level) - () - in - let size = - match value_written with - | None -> 0 - | Some bytes -> `Hex bytes |> Hex.to_string |> Z.of_bits |> Z.to_int - in - Hashtbl.add slots slot size ; - unit - in - let* () = - match t.echo_rollup with - | None -> unit - | Some echo_rollup -> - if level < echo_rollup.Echo_rollup.origination_level then unit - else - let* _level = - Sc_rollup_node.wait_for_level - ~timeout:30. - echo_rollup.sc_rollup_node - level - in - Lwt_list.iter_s - (read_slot echo_rollup) - t.configuration.dal_node_producers - in - return slots - let get_infos_per_level t ~level ~metadata = let open Metrics in let client = t.bootstrap.client in @@ -260,7 +221,12 @@ let get_infos_per_level t ~level ~metadata = unit else Lwt.return_unit in - let* echo_rollup_fetched_data = get_echo_rollup t ~level in + let* echo_rollup_fetched_data = + Echo_rollup.fetch_echo_rollup_data + ~echo_rollup:t.echo_rollup + ~dal_node_producers:t.configuration.dal_node_producers + ~level + in Lwt.return { level; diff --git a/tezt/tests/cloud/echo_rollup.ml b/tezt/tests/cloud/echo_rollup.ml index 534e11bc8cf6..f0b82cbb0ff4 100644 --- a/tezt/tests/cloud/echo_rollup.ml +++ b/tezt/tests/cloud/echo_rollup.ml @@ -20,6 +20,43 @@ type operator = { origination_level : int; } +let fetch_echo_rollup_data ~echo_rollup ~dal_node_producers ~level = + let slots = Hashtbl.create 32 in + let read_slot echo_operator slot = + let key = "/output/slot-" ^ string_of_int slot in + let* value_written = + Sc_rollup_node.RPC.call echo_operator.sc_rollup_node + @@ Sc_rollup_rpc.get_global_block_durable_state_value + ~pvm_kind:"wasm_2_0_0" + ~operation:Sc_rollup_rpc.Value + ~key + ~block:(string_of_int level) + () + in + let size = + match value_written with + | None -> 0 + | Some bytes -> `Hex bytes |> Hex.to_string |> Z.of_bits |> Z.to_int + in + Hashtbl.add slots slot size ; + unit + in + let* () = + match echo_rollup with + | None -> unit + | Some echo_rollup -> + if level < echo_rollup.origination_level then unit + else + let* _level = + Sc_rollup_node.wait_for_level + ~timeout:30. + echo_rollup.sc_rollup_node + level + in + Lwt_list.iter_s (read_slot echo_rollup) dal_node_producers + in + return slots + let init_echo_rollup_account ~client ~echo_rollup ~alias_prefix = if echo_rollup then let () = toplog "Initializing the echo rollup key" in diff --git a/tezt/tests/cloud/echo_rollup.mli b/tezt/tests/cloud/echo_rollup.mli index 0904a9d47c33..690da4286047 100644 --- a/tezt/tests/cloud/echo_rollup.mli +++ b/tezt/tests/cloud/echo_rollup.mli @@ -48,3 +48,12 @@ val init_echo_rollup : Dal_node_helpers.producer list -> Account.key option -> operator option Lwt.t + +(** [fetch_echo_rollup_data ~echo_rollup ~data_node_producers ~level] + fetches payload size in bytes for each DAL slot given by [~data_node_producers] + written by the [~echo_rollup] at a given L1 [~level]. *) +val fetch_echo_rollup_data : + echo_rollup:operator option -> + dal_node_producers:int list -> + level:int -> + (int, int) Hashtbl.t Lwt.t diff --git a/tezt/tests/cloud/metrics.ml b/tezt/tests/cloud/metrics.ml index fd7cce884919..50768660bc1f 100644 --- a/tezt/tests/cloud/metrics.ml +++ b/tezt/tests/cloud/metrics.ml @@ -338,7 +338,7 @@ let push ~versions ~cloud ~help:"Sum of the balances of the etherlink operator" ~typ:`Gauge ~name:"tezt_etherlink_operator_balance_total" - (Tez.to_float etherlink_operator_balance_sum); + (Tez.to_float etherlink_operator_balance_sum) ; Cloud.push_metric cloud ~help:"Number of slots unattested from the echo rollup perspective" -- GitLab