From 776ba4cb7c0c16dc3c1a47ccb97d7aeee9e1bc2a Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Wed, 16 Jul 2025 15:30:51 +0100 Subject: [PATCH 01/11] Tezt_cloud: Layer1: Unify the use of --network option for both scenarios This is needed in order to have a similar behaviour with the DAL scenario, which in turn will be needed for other unifications and refactorings, such as the one for snapshotting logic, which will follow. --- tezt/tests/cloud/layer1.ml | 32 +++++----------- tezt/tests/cloud/network.ml | 5 +++ tezt/tests/cloud/network.mli | 3 ++ tezt/tests/cloud/scenarios_cli.ml | 64 +++++++++++++------------------ 4 files changed, 44 insertions(+), 60 deletions(-) diff --git a/tezt/tests/cloud/layer1.ml b/tezt/tests/cloud/layer1.ml index d605a3d80b22..822bfd225235 100644 --- a/tezt/tests/cloud/layer1.ml +++ b/tezt/tests/cloud/layer1.ml @@ -32,22 +32,6 @@ let get_snapshot_info_level node snapshot_path = let json = JSON.parse ~origin:"snapshot_info" info in Lwt.return JSON.(json |-> "snapshot_header" |-> "level" |> as_int) -module Network = struct - include Network - - type t = [`Mainnet | `Ghostnet] - - let to_string (t : t) = Network.to_string (t :> Network.t) - - let to_octez_network_options (t : t) = - Network.to_octez_network_options (t :> Network.public) - - let block_time = function `Ghostnet -> 5 | `Mainnet -> 8 - - (** Next protocol for both Mainnet and Ghostnet - needs to be updated manually. *) - let migrate_to _network = Protocol.Alpha -end - let yes_crypto_env = String_map.add Tezos_crypto.Helpers.yes_crypto_environment_variable @@ -101,7 +85,7 @@ module Node = struct No_bootstrap_peers; Connections (List.length peers); Synchronisation_threshold (if List.length peers < 2 then 1 else 2); - Network (Network.to_octez_network_options network); + Network Network.(to_octez_network_options @@ to_public network); Expected_pow 0; Cors_origin "*"; Storage_maintenance_delay (string_of_int delay); @@ -129,10 +113,14 @@ module Node = struct match migration_offset with | None -> Lwt.return_unit | Some migration_offset -> - let network_config = + let* network_config = match network with - | `Mainnet -> Node.Config_file.mainnet_network_config - | `Ghostnet -> Node.Config_file.ghostnet_network_config + | `Mainnet -> Lwt.return Node.Config_file.mainnet_network_config + | `Ghostnet -> Lwt.return Node.Config_file.ghostnet_network_config + | _ -> + Lwt.fail_with + "Migration scenarios are only supported for Mainnet and \ + Ghostnet." in let migration_level = level + migration_offset in toplog "Add UAU entry for level : %d" migration_level ; @@ -144,7 +132,7 @@ module Node = struct network_config ) json |> Node.Config_file.update_network_with_user_activated_upgrades - [(migration_level, Network.migrate_to network)]) + [(migration_level, Protocol.Alpha)]) (* If trying to only bootstrap the network from a snapshot, you will have errors about missing block metadata, which is likely (I guess?) to be @@ -171,7 +159,7 @@ module Node = struct toplog "Bootstrapping node using the real world" ; let config = [ - Network (Network.to_octez_network_options network); + Network Network.(to_octez_network_options @@ to_public network); Expected_pow 26; Cors_origin "*"; Synchronisation_threshold 1; diff --git a/tezt/tests/cloud/network.ml b/tezt/tests/cloud/network.ml index d9e6c7c2ab2a..eb26859be82b 100644 --- a/tezt/tests/cloud/network.ml +++ b/tezt/tests/cloud/network.ml @@ -41,6 +41,11 @@ let default_protocol : t -> Protocol.t = function | `Rionet -> R022 | `Seoulnet -> S023 +let block_time : t -> int = function + | `Mainnet -> 8 + | `Ghostnet -> 5 + | _ -> failwith "Block times are only available for Mainnet and Ghostnet." + let public_rpc_endpoint testnet = Endpoint.make ~scheme:"https" diff --git a/tezt/tests/cloud/network.mli b/tezt/tests/cloud/network.mli index de8eee41c36b..3a7a79b0fbb2 100644 --- a/tezt/tests/cloud/network.mli +++ b/tezt/tests/cloud/network.mli @@ -34,6 +34,9 @@ val to_string : [< t] -> string (** Known protocol used by the network *) val default_protocol : t -> Protocol.t +(** Block time value for the network *) +val block_time : t -> int + (** Endpoint publicly available with RPC opened *) val public_rpc_endpoint : public -> Endpoint.t diff --git a/tezt/tests/cloud/scenarios_cli.ml b/tezt/tests/cloud/scenarios_cli.ml index d20e76d7c58e..7b4df13effbf 100644 --- a/tezt/tests/cloud/scenarios_cli.ml +++ b/tezt/tests/cloud/scenarios_cli.ml @@ -100,6 +100,29 @@ let parse_network_simulation_config_from_args simulate_network_arg = "Unexpected network simulation config (--simulation) [%s]" simulate_network_arg +let parse_network = function + | "mainnet" -> Some `Mainnet + | "ghostnet" -> Some `Ghostnet + | "rionet" -> Some `Rionet + | "seoulnet" -> Some `Seoulnet + | s when String.length s = 20 && String.sub s 0 10 = "weeklynet-" -> + (* format: weeklynet-2025-01-29 (with dashes) *) + let date = String.sub s 10 10 in + Some (`Weeklynet date) + | s when String.length s = 16 && String.sub s 0 8 = "nextnet-" -> + (* format: nextnet-20250203 (without dashes) *) + let date = String.sub s 8 8 in + Some (`Nextnet date) + | "sandbox" -> Some `Sandbox + | _ -> None + +let network_typ : Network.t Clap.typ = + Clap.typ + ~name:"network" + ~dummy:`Ghostnet + ~parse:parse_network + ~show:Network.to_string + module type Dal = sig val blocks_history : int @@ -107,8 +130,6 @@ module type Dal = sig val fundraiser : string option - val network_typ : Network.t Clap.typ - val network : Network.t val simulate_network_typ : network_simulation_config Clap.typ @@ -223,29 +244,6 @@ module Dal () : Dal = struct \"produce every level\"." 1 - let parse_network = function - | "mainnet" -> Some `Mainnet - | "ghostnet" -> Some `Ghostnet - | "rionet" -> Some `Rionet - | "seoulnet" -> Some `Seoulnet - | s when String.length s = 20 && String.sub s 0 10 = "weeklynet-" -> - (* format: weeklynet-2025-01-29 (with dashes) *) - let date = String.sub s 10 10 in - Some (`Weeklynet date) - | s when String.length s = 16 && String.sub s 0 8 = "nextnet-" -> - (* format: nextnet-20250203 (without dashes) *) - let date = String.sub s 8 8 in - Some (`Nextnet date) - | "sandbox" -> Some `Sandbox - | _ -> None - - let network_typ : Network.t Clap.typ = - Clap.typ - ~name:"network" - ~dummy:`Ghostnet - ~parse:parse_network - ~show:Network.to_string - let network = Clap.default ~section @@ -648,7 +646,7 @@ module Dal () : Dal = struct end module type Layer1 = sig - val network : [`Mainnet | `Ghostnet] option + val network : Network.t option val stake : int list option @@ -682,23 +680,13 @@ module Layer1 () = struct [cloud] and [layer1] tags on the command line" "LAYER1" - let network : [`Mainnet | `Ghostnet] option = - let typ = - Clap.typ - ~name:"network" - ~dummy:`Ghostnet - ~parse:(function - | "mainnet" -> Some `Mainnet - | "ghostnet" -> Some `Ghostnet - | _ -> None) - ~show:(fun n -> Network.to_string (n :> Network.t)) - in + let network : Network.t option = Clap.optional ~section ~long:"network" ~placeholder:"" ~description:"Allow to specify a network to use for the scenario" - typ + network_typ () let stake = -- GitLab From 491e8ed63b65c9cffbebeef1371ebba155138072 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Wed, 16 Jul 2025 17:24:12 +0100 Subject: [PATCH 02/11] Tezt_cloud: Dal: Add option for snapshots to be downloaded from given URL This is a step towards mimicing the behaviour from the Layer1 scenario --- tezt/tests/cloud/dal.ml | 66 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/tezt/tests/cloud/dal.ml b/tezt/tests/cloud/dal.ml index 8eb3684357ba..724536b02327 100644 --- a/tezt/tests/cloud/dal.ml +++ b/tezt/tests/cloud/dal.ml @@ -59,6 +59,7 @@ let name_of = function type snapshot_config = | Docker_embedded of string | Local_file of string + | Url of string | No_snapshot (* Some DAL nodes (those in operator mode) refuse to start unless they are @@ -277,6 +278,35 @@ module Node = struct ~source:snapshot_path in Lwt.return snapshot_path + | Url url -> + toplog "Trying to download snapshot for %s from %s" name url ; + let downloaded_snapshot_file_path = "snapshot_file" in + let* exit_status = + Process.spawn + ?runner:(runner_of_agent agent) + "wget" + ["-O"; downloaded_snapshot_file_path; sf "%s/rolling" url] + |> Process.wait + in + let* () = + match exit_status with + | WEXITED 0 -> Lwt.return_unit + | WEXITED code -> + toplog + "Could not download the snapshot for %s: wget exit \ + code: %d\n\ + Starting without snapshot. It could last long \ + before the node is bootstrapped" + name + code ; + Lwt.return_unit + | status -> ( + match Process.validate_status status with + | Ok () -> Lwt.return_unit + | Error (`Invalid_status reason) -> + failwith @@ Format.sprintf "wget: %s" reason) + in + Lwt.return downloaded_snapshot_file_path | No_snapshot -> toplog "Trying to download a rolling snapshot for %s" name ; let downloaded_snapshot_file_path = "snapshot_file" in @@ -309,7 +339,7 @@ module Node = struct | Error (`Invalid_status reason) -> failwith @@ Format.sprintf "wget: %s" reason) in - return downloaded_snapshot_file_path + Lwt.return downloaded_snapshot_file_path in let* () = import_snapshot @@ -368,6 +398,35 @@ module Node = struct ~source:snapshot_path in Lwt.return_some snapshot_path + | Url url -> + toplog "Trying to download snapshot for %s from %s" name url ; + let downloaded_snapshot_file_path = "snapshot_file" in + let* exit_status = + Process.spawn + ?runner:(runner_of_agent agent) + "wget" + ["-O"; downloaded_snapshot_file_path; sf "%s/rolling" url] + |> Process.wait + in + let* () = + match exit_status with + | WEXITED 0 -> Lwt.return_unit + | WEXITED code -> + toplog + "Could not download the snapshot for %s: wget exit \ + code: %d\n\ + Starting without snapshot. It could last long \ + before the node is bootstrapped" + name + code ; + Lwt.return_unit + | status -> ( + match Process.validate_status status with + | Ok () -> Lwt.return_unit + | Error (`Invalid_status reason) -> + failwith @@ Format.sprintf "wget: %s" reason) + in + Lwt.return_some downloaded_snapshot_file_path | No_snapshot -> Lwt.return_none in let* snapshot_network = @@ -2885,6 +2944,11 @@ let init_sandbox_and_activate_protocol cloud (configuration : configuration) "https://teztnets.com/" ^ s in Lwt.return network + | Url _url -> + (* FIXME: We can overcome this by either downloading the snapshot, or by + retrieving the name of the network from the URL. I am not sure how much + of a priority this is. *) + Lwt.fail_with "URL snapshot not available for sandbox case" | No_snapshot -> Lwt.return "ghostnet" in let* _filename = -- GitLab From 3e57c6c8bdb6e5170eee737b173067471a124547 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Wed, 16 Jul 2025 17:50:06 +0100 Subject: [PATCH 03/11] Tezt_cloud: Dal: Refactor snapshot verification This is done to resemble the functionality from Layer1 scenario --- tezt/tests/cloud/dal.ml | 182 ++++++++++++---------------------------- 1 file changed, 54 insertions(+), 128 deletions(-) diff --git a/tezt/tests/cloud/dal.ml b/tezt/tests/cloud/dal.ml index 724536b02327..41b5dfa825e9 100644 --- a/tezt/tests/cloud/dal.ml +++ b/tezt/tests/cloud/dal.ml @@ -162,6 +162,58 @@ module Node = struct Tezos_crypto.Helpers.yes_crypto_environment_variable "y" + let download_snapshot ~agent ~url ~name = + let downloaded_snapshot_file_path = "snapshot_file" in + toplog "Trying to download snapshot for %s from %s" name url ; + let* exit_status = + Process.spawn + ?runner:(runner_of_agent agent) + "wget" + ["-O"; downloaded_snapshot_file_path; sf "%s/rolling" url] + |> Process.wait + in + let* () = + match exit_status with + | WEXITED 0 -> Lwt.return_unit + | WEXITED code -> + toplog + "Could not download the snapshot for %s: wget exit code: %d\n\ + Starting without snapshot. It could last long before the node is \ + bootstrapped" + name + code ; + Lwt.return_unit + | status -> ( + match Process.validate_status status with + | Ok () -> Lwt.return_unit + | Error (`Invalid_status reason) -> + failwith @@ Format.sprintf "wget: %s" reason) + in + Lwt.return downloaded_snapshot_file_path + + let ensure_snapshot ~agent ~name ~network = function + | Docker_embedded path -> + toplog "Using locally stored snapshot file: %s" path ; + Lwt.return path + | Local_file path -> + toplog "Copying snapshot to destination" ; + Tezt_cloud.Agent.copy agent ~destination:path ~source:path + | Url url -> download_snapshot ~agent ~url ~name + | No_snapshot -> + download_snapshot ~agent ~url:(Network.snapshot_service network) ~name + + let ensure_snapshot_opt ~agent = function + | Docker_embedded path -> + toplog "Using locally stored snapshot file: %s" path ; + Lwt.return_some path + | Local_file path -> + toplog "Copying snapshot to destination" ; + let* path = + Tezt_cloud.Agent.copy agent ~destination:path ~source:path + in + Lwt.return_some path + | Url _ | No_snapshot -> Lwt.return_none + let import_snapshot ?(delete_snapshot_file = false) ~name node snapshot_file_path = toplog "Importing the snapshot for %s" name ; @@ -261,85 +313,7 @@ module Node = struct toplog "Initializing node configuration for %s" name ; let* () = Node.config_init node [] in let* snapshot_file_path = - match snapshot with - | Docker_embedded snapshot_path -> - let () = - toplog - "Using locally stored snapshot file: %s" - snapshot_path - in - Lwt.return snapshot_path - | Local_file snapshot_path -> - let () = toplog "Copying snapshot to destination" in - let* snapshot_path = - Tezt_cloud.Agent.copy - agent - ~destination:snapshot_path - ~source:snapshot_path - in - Lwt.return snapshot_path - | Url url -> - toplog "Trying to download snapshot for %s from %s" name url ; - let downloaded_snapshot_file_path = "snapshot_file" in - let* exit_status = - Process.spawn - ?runner:(runner_of_agent agent) - "wget" - ["-O"; downloaded_snapshot_file_path; sf "%s/rolling" url] - |> Process.wait - in - let* () = - match exit_status with - | WEXITED 0 -> Lwt.return_unit - | WEXITED code -> - toplog - "Could not download the snapshot for %s: wget exit \ - code: %d\n\ - Starting without snapshot. It could last long \ - before the node is bootstrapped" - name - code ; - Lwt.return_unit - | status -> ( - match Process.validate_status status with - | Ok () -> Lwt.return_unit - | Error (`Invalid_status reason) -> - failwith @@ Format.sprintf "wget: %s" reason) - in - Lwt.return downloaded_snapshot_file_path - | No_snapshot -> - toplog "Trying to download a rolling snapshot for %s" name ; - let downloaded_snapshot_file_path = "snapshot_file" in - let* exit_status = - Process.spawn - ?runner:(runner_of_agent agent) - "wget" - [ - "-O"; - downloaded_snapshot_file_path; - sf "%s/rolling" (Network.snapshot_service network); - ] - |> Process.wait - in - let* () = - match exit_status with - | WEXITED 0 -> Lwt.return_unit - | WEXITED code -> - toplog - "Could not download the snapshot for %s: wget exit \ - code: %d\n\ - Starting without snapshot. It could last long \ - before the node is bootstrapped" - name - code ; - Lwt.return_unit - | status -> ( - match Process.validate_status status with - | Ok () -> Lwt.return_unit - | Error (`Invalid_status reason) -> - failwith @@ Format.sprintf "wget: %s" reason) - in - Lwt.return downloaded_snapshot_file_path + ensure_snapshot ~agent ~name ~network snapshot in let* () = import_snapshot @@ -380,55 +354,7 @@ module Node = struct Node.Agent.create ~net_addr ~rpc_external ~name cloud agent in let* () = Node.config_init node [Cors_origin "*"] in - let* snapshot_path = - match snapshot with - | Docker_embedded snapshot_path -> - let () = - toplog - "Using locally stored snapshot file: %s" - snapshot_path - in - Lwt.return_some snapshot_path - | Local_file snapshot_path -> - let () = toplog "Copying snapshot to destination" in - let* snapshot_path = - Tezt_cloud.Agent.copy - agent - ~destination:snapshot_path - ~source:snapshot_path - in - Lwt.return_some snapshot_path - | Url url -> - toplog "Trying to download snapshot for %s from %s" name url ; - let downloaded_snapshot_file_path = "snapshot_file" in - let* exit_status = - Process.spawn - ?runner:(runner_of_agent agent) - "wget" - ["-O"; downloaded_snapshot_file_path; sf "%s/rolling" url] - |> Process.wait - in - let* () = - match exit_status with - | WEXITED 0 -> Lwt.return_unit - | WEXITED code -> - toplog - "Could not download the snapshot for %s: wget exit \ - code: %d\n\ - Starting without snapshot. It could last long \ - before the node is bootstrapped" - name - code ; - Lwt.return_unit - | status -> ( - match Process.validate_status status with - | Ok () -> Lwt.return_unit - | Error (`Invalid_status reason) -> - failwith @@ Format.sprintf "wget: %s" reason) - in - Lwt.return_some downloaded_snapshot_file_path - | No_snapshot -> Lwt.return_none - in + let* snapshot_path = ensure_snapshot_opt ~agent snapshot in let* snapshot_network = match snapshot_path with | Some path -> -- GitLab From 64edd253f6c882ea6c024f28dd852e127f7b322e Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Wed, 16 Jul 2025 18:35:44 +0100 Subject: [PATCH 04/11] Tezt_cloud: Unify the use of --snapshot option for both scenarios --- tezt/tests/cloud/dal.ml | 36 ++-------- tezt/tests/cloud/layer1.ml | 102 +++++++++++++++++++-------- tezt/tests/cloud/scenarios_cli.ml | 21 ++++-- tezt/tests/cloud/snapshot_helpers.ml | 72 +++++++++++++++++++ 4 files changed, 163 insertions(+), 68 deletions(-) create mode 100644 tezt/tests/cloud/snapshot_helpers.ml diff --git a/tezt/tests/cloud/dal.ml b/tezt/tests/cloud/dal.ml index 41b5dfa825e9..e9af26c328c5 100644 --- a/tezt/tests/cloud/dal.ml +++ b/tezt/tests/cloud/dal.ml @@ -56,12 +56,6 @@ let name_of = function | Echo_rollup_dal_observer {slot_index} -> Format.sprintf "echo-rollup-dal-node-%d" slot_index -type snapshot_config = - | Docker_embedded of string - | Local_file of string - | Url of string - | No_snapshot - (* Some DAL nodes (those in operator mode) refuse to start unless they are connected to an Octez node keeping enough history to play refutation games. *) @@ -142,6 +136,8 @@ end module Node = struct let runner_of_agent = Agent.runner + open Snapshot_helpers + let may_copy_node_identity_file agent node = function | None -> Lwt.return_unit | Some source -> @@ -613,7 +609,7 @@ type configuration = { disconnect : (int * int) option; network : Network.t; simulate_network : Cli.network_simulation_config; - snapshot : snapshot_config; + snapshot : Snapshot_helpers.t; bootstrap : bool; teztale : bool; memtrace : bool; @@ -4778,30 +4774,6 @@ let rec loop t level = let* t = p in loop t (level + 1) -let parse_snapshot_arg snapshot_arg = - let fail path = - Test.fail - "wrong snapshot argument (--snapshot) [%s].@.Use:@.- \ - \"file:path/to/file\" to use a local file that will be uploaded to each \ - agent,@.- \"docker\" to use the docker_embedded_snapshot_file, that \ - must be located in the local path, to embed the snapshot file into the \ - docker image." - path - in - match snapshot_arg with - | Some v -> ( - match v with - | "docker" -> - (* This hardcoded path must be defined as the location path of the snapshot - embedded in the docker image. See the associated dockerfile. *) - Docker_embedded "/tmp/docker_embedded_snapshot_file" - | s when String.starts_with ~prefix:"file:" s -> ( - match String.split_on_char ':' s with - | [_; path] -> Local_file path - | _ -> fail s) - | _ -> fail v) - | None -> No_snapshot - let yes_wallet_exe = Uses.path Constant.yes_wallet let parse_stake_arg ~stake_arg ~simulation_arg = @@ -4923,7 +4895,7 @@ let register (module Cli : Scenarios_cli.Dal) = let echo_rollup = Cli.echo_rollup in let disconnect = Cli.disconnect in let network = Cli.network in - let snapshot = parse_snapshot_arg Cli.snapshot in + let snapshot = Cli.snapshot in let bootstrap = Cli.bootstrap in let etherlink_dal_slots = Cli.etherlink_dal_slots in let teztale = Cli.teztale in diff --git a/tezt/tests/cloud/layer1.ml b/tezt/tests/cloud/layer1.ml index 822bfd225235..d0cb12076224 100644 --- a/tezt/tests/cloud/layer1.ml +++ b/tezt/tests/cloud/layer1.ml @@ -59,23 +59,48 @@ let nb_stresstester network tps = module Node = struct let runner_of_agent = Agent.runner + open Snapshot_helpers include Node - (** If snapshot is an URL, download it on the agent's runner. - If it is a filename, copy it to the agent's runner if needed. *) - let ensure_snapshot agent snapshot = - if Re.Str.(string_match (regexp "^https?://.+$") snapshot 0) then - let url = snapshot in - let filename = "snapshot" in - let* () = - Process.spawn - ?runner:(runner_of_agent agent) - "wget" - ["-q"; "-O"; filename; url] - |> Process.check - in - Lwt.return filename - else Tezt_cloud.Agent.copy agent ~destination:snapshot ~source:snapshot + let download_snapshot ~agent ~url ~name = + let downloaded_snapshot_file_path = "snapshot_file" in + toplog "Trying to download snapshot for %s from %s" name url ; + let* exit_status = + Process.spawn + ?runner:(runner_of_agent agent) + "wget" + ["-O"; downloaded_snapshot_file_path; sf "%s/rolling" url] + |> Process.wait + in + let* () = + match exit_status with + | WEXITED 0 -> Lwt.return_unit + | WEXITED code -> + toplog + "Could not download the snapshot for %s: wget exit code: %d\n\ + Starting without snapshot. It could last long before the node is \ + bootstrapped" + name + code ; + Lwt.return_unit + | status -> ( + match Process.validate_status status with + | Ok () -> Lwt.return_unit + | Error (`Invalid_status reason) -> + failwith @@ Format.sprintf "wget: %s" reason) + in + Lwt.return downloaded_snapshot_file_path + + let ensure_snapshot ~agent ~name ~network = function + | Docker_embedded path -> + toplog "Using locally stored snapshot file: %s" path ; + Lwt.return path + | Local_file path -> + toplog "Copying snapshot to destination" ; + Tezt_cloud.Agent.copy agent ~destination:path ~source:path + | Url url -> download_snapshot ~agent ~url ~name + | No_snapshot -> + download_snapshot ~agent ~url:(Network.snapshot_service network) ~name (** We are running a private network with yes-crypto enabled. We don't want to connect with the real network. @@ -151,9 +176,11 @@ module Node = struct That's why the bootstrap node first syncs for few levels before being disconnected from the real network. *) - let init_bootstrap_node_from_snapshot ~peers (agent, node) snapshot network - migration_offset = - let* snapshot = ensure_snapshot agent snapshot in + let init_bootstrap_node_from_snapshot ~peers (agent, node, name) snapshot + network migration_offset = + let* snapshot = + ensure_snapshot ~agent ~name ~network:(Network.to_public network) snapshot + in let* () = let toplog s = toplog "/!\\ %s /!\\" s in toplog "Bootstrapping node using the real world" ; @@ -193,8 +220,10 @@ module Node = struct - run it with yes-crypto enabled and allowed peer lists *) let init_node_from_snapshot ~delay ~peers ~snapshot ~network ~migration_offset - (agent, node) = - let* snapshot = ensure_snapshot agent snapshot in + (agent, node, name) = + let* snapshot = + ensure_snapshot ~agent ~name ~network:(Network.to_public network) snapshot + in let config = isolated_config ~peers ~network ~delay in let* () = Node.config_init node config in let* () = @@ -228,7 +257,7 @@ module Node = struct let* () = init_bootstrap_node_from_snapshot ~peers - (agent, node) + (agent, node, name) snapshot network migration_offset @@ -262,7 +291,7 @@ module Node = struct ~snapshot ~network ~migration_offset - (agent, node) + (agent, node, name) in let* client = client ~node agent in let* yes_wallet = yes_wallet agent in @@ -285,7 +314,7 @@ module Node = struct ~snapshot ~network ~migration_offset - (agent, node) + (agent, node, name) in let* client = client ~node agent in let* yes_wallet = yes_wallet agent in @@ -341,7 +370,7 @@ type stresstest_conf = {pkh : string; pk : string; tps : int; seed : int} type configuration = { stake : int list; network : Network.t; - snapshot : string; + snapshot : Snapshot_helpers.t; stresstest : stresstest_conf option; maintenance_delay : int; migration_offset : int option; @@ -351,7 +380,7 @@ type configuration = { type partial_configuration = { stake : int list option; network : Network.t option; - snapshot : string option; + snapshot : Snapshot_helpers.t option; stresstest : stresstest_conf option; maintenance_delay : int option; migration_offset : int option; @@ -400,7 +429,7 @@ let configuration_encoding = (obj6 (opt "stake" @@ list int31) (opt "network" network_encoding) - (opt "snapshot" string) + (opt "snapshot" Snapshot_helpers.encoding) (opt "stresstest" stresstest_encoding) (opt "maintenance_delay" int31) (opt "migration_offset" int31)) @@ -618,11 +647,17 @@ let distribute_delegates stake baker_accounts = |> print_list "Distribution" ; List.map (List.map fst) distribution -let number_of_bakers ~snapshot ~network cloud agent = +let number_of_bakers ~snapshot ~network cloud agent name = let* node = Node.Agent.create ~metadata_size_limit:false ~name:"tmp-node" cloud agent in - let* snapshot = Node.ensure_snapshot agent snapshot in + let* snapshot = + Node.ensure_snapshot + ~agent + ~name + ~network:(Network.to_public network) + snapshot + in let* () = Node.config_init node (Node.isolated_config ~peers:[] ~network ~delay:0) in @@ -645,7 +680,7 @@ let init ~(configuration : configuration) cloud next_agent = let () = toplog "Init" in (* First, we allocate agents and node address/port in order to have the peer list known when initializing. *) - let* ((bootstrap_agent, bootstrap_node, _) as bootstrap) = + let* ((bootstrap_agent, bootstrap_node, bootstrap_name) as bootstrap) = agent_and_node next_agent cloud ~name:"bootstrap" in let* stresstest_agents = @@ -664,6 +699,7 @@ let init ~(configuration : configuration) cloud next_agent = ~network:configuration.network cloud bootstrap_agent + bootstrap_name else Lwt.return (List.length configuration.stake) in let () = toplog "Preparing agents for bakers (%d)" n in @@ -1081,7 +1117,10 @@ let register (module Cli : Scenarios_cli.Layer1) = ~proxy_files: ([yes_wallet_exe] @ - match configuration0.snapshot with Some snapshot -> [snapshot] | _ -> []) + match configuration0.snapshot with + | Some snapshot -> ( + match snapshot with Local_file snapshot -> [snapshot] | _ -> []) + | _ -> []) ~__FILE__ ~title:"L1 simulation" ~tags:[] @@ -1098,7 +1137,8 @@ let register (module Cli : Scenarios_cli.Layer1) = in let migration_offset = configuration0.migration_offset in if stake = [] then Test.fail "stake parameter can not be empty" ; - if snapshot = "" then Test.fail "snapshot parameter can not be empty" ; + if snapshot = Snapshot_helpers.No_snapshot then + Test.fail "snapshot parameter can not be empty" ; {stake; network; snapshot; stresstest; maintenance_delay; migration_offset} in toplog "Creating the agents" ; diff --git a/tezt/tests/cloud/scenarios_cli.ml b/tezt/tests/cloud/scenarios_cli.ml index 7b4df13effbf..033b1c6fa9ae 100644 --- a/tezt/tests/cloud/scenarios_cli.ml +++ b/tezt/tests/cloud/scenarios_cli.ml @@ -123,6 +123,15 @@ let network_typ : Network.t Clap.typ = ~parse:parse_network ~show:Network.to_string +let snapshot_typ : Snapshot_helpers.t Clap.typ = + let open Snapshot_helpers in + Clap.typ + ~name:"snapshot" + ~dummy:No_snapshot + ~parse:(fun snapshot -> + try Some (parse_snapshot (Some snapshot)) with _exn -> None) + ~show:to_string + module type Dal = sig val blocks_history : int @@ -136,7 +145,7 @@ module type Dal = sig val simulate_network : network_simulation_config - val snapshot : string option + val snapshot : Snapshot_helpers.t val bootstrap : bool @@ -290,13 +299,14 @@ module Dal () : Dal = struct Disabled let snapshot = - Clap.optional_string + Clap.default ~section ~long:"snapshot" ~description: "Snapshot file, which is stored locally, to initiate the scenario with \ some data" - () + snapshot_typ + No_snapshot let bootstrap = Clap.flag @@ -658,7 +668,7 @@ module type Layer1 = sig val migration_offset : int option - val snapshot : string option + val snapshot : Snapshot_helpers.t option val octez_release : string option @@ -757,12 +767,13 @@ module Layer1 () = struct () let snapshot = - Clap.optional_string + Clap.optional ~section ~long:"snapshot" ~description: "Either a path the a local file or url of the snapshot to use for \ bootstrapping the experiment" + snapshot_typ () let octez_release = diff --git a/tezt/tests/cloud/snapshot_helpers.ml b/tezt/tests/cloud/snapshot_helpers.ml new file mode 100644 index 000000000000..b433297e4e7c --- /dev/null +++ b/tezt/tests/cloud/snapshot_helpers.ml @@ -0,0 +1,72 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2025 Trilitech *) +(* *) +(*****************************************************************************) + +type t = + | Docker_embedded of string + | Local_file of string + | Url of string + | No_snapshot + +(** Parse raw snapshot CLI option. *) +let parse_snapshot = + let fail path = + Test.fail + "wrong snapshot argument (--snapshot) [%s].@.Use:@.- \ + \"file:path/to/file\" to use a local file that will be uploaded to each \ + agent,@.- \"docker\" to use the docker_embedded_snapshot_file, that \ + must be located in the local path, to embed the snapshot file into the \ + docker image." + path + in + function + | None -> No_snapshot + | Some s when Re.Str.(string_match (regexp "^https?://.+$") s 0) -> Url s + | Some s when String.equal s "docker" -> + (* This hardcoded path must be defined as the location path of the snapshot + embedded in the docker image. See the associated dockerfile. *) + Docker_embedded "/tmp/docker_embedded_snapshot_file" + | Some s when String.starts_with ~prefix:"file:" s -> ( + match String.split_on_char ':' s with + | [_; path] -> Local_file path + | _ -> fail s) + | Some s -> fail s + +let to_string = function + | No_snapshot -> "No_snapshot" + | Docker_embedded path -> Format.sprintf "Docker_embedded: %s" path + | Local_file path -> Format.sprintf "Local_file: %s" path + | Url url -> Format.sprintf "Url: %s" url + +let encoding = + let open Data_encoding in + union + [ + case + (Tag 0) + ~title:"No_snapshot" + (constant "no_snapshot") + (function No_snapshot -> Some () | _ -> None) + (fun () -> No_snapshot); + case + (Tag 1) + ~title:"Docker_embedded" + (obj1 (req "docker_embedded" string)) + (function Docker_embedded s -> Some s | _ -> None) + (fun s -> Docker_embedded s); + case + (Tag 2) + ~title:"Local_file" + (obj1 (req "local_file" string)) + (function Local_file s -> Some s | _ -> None) + (fun s -> Local_file s); + case + (Tag 3) + ~title:"Url" + (obj1 (req "url" string)) + (function Url s -> Some s | _ -> None) + (fun s -> Url s); + ] -- GitLab From 2bb9ea754a864fa99308511ba372d4e92e9b10a9 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Wed, 16 Jul 2025 18:41:46 +0100 Subject: [PATCH 05/11] Tezt_cloud: Deduplicate common logic --- tezt/tests/cloud/dal.ml | 69 ++-------------------------- tezt/tests/cloud/layer1.ml | 49 +------------------- tezt/tests/cloud/snapshot_helpers.ml | 68 +++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 114 deletions(-) diff --git a/tezt/tests/cloud/dal.ml b/tezt/tests/cloud/dal.ml index e9af26c328c5..fc8a6235d707 100644 --- a/tezt/tests/cloud/dal.ml +++ b/tezt/tests/cloud/dal.ml @@ -134,8 +134,6 @@ module Disconnect = struct end module Node = struct - let runner_of_agent = Agent.runner - open Snapshot_helpers let may_copy_node_identity_file agent node = function @@ -158,58 +156,6 @@ module Node = struct Tezos_crypto.Helpers.yes_crypto_environment_variable "y" - let download_snapshot ~agent ~url ~name = - let downloaded_snapshot_file_path = "snapshot_file" in - toplog "Trying to download snapshot for %s from %s" name url ; - let* exit_status = - Process.spawn - ?runner:(runner_of_agent agent) - "wget" - ["-O"; downloaded_snapshot_file_path; sf "%s/rolling" url] - |> Process.wait - in - let* () = - match exit_status with - | WEXITED 0 -> Lwt.return_unit - | WEXITED code -> - toplog - "Could not download the snapshot for %s: wget exit code: %d\n\ - Starting without snapshot. It could last long before the node is \ - bootstrapped" - name - code ; - Lwt.return_unit - | status -> ( - match Process.validate_status status with - | Ok () -> Lwt.return_unit - | Error (`Invalid_status reason) -> - failwith @@ Format.sprintf "wget: %s" reason) - in - Lwt.return downloaded_snapshot_file_path - - let ensure_snapshot ~agent ~name ~network = function - | Docker_embedded path -> - toplog "Using locally stored snapshot file: %s" path ; - Lwt.return path - | Local_file path -> - toplog "Copying snapshot to destination" ; - Tezt_cloud.Agent.copy agent ~destination:path ~source:path - | Url url -> download_snapshot ~agent ~url ~name - | No_snapshot -> - download_snapshot ~agent ~url:(Network.snapshot_service network) ~name - - let ensure_snapshot_opt ~agent = function - | Docker_embedded path -> - toplog "Using locally stored snapshot file: %s" path ; - Lwt.return_some path - | Local_file path -> - toplog "Copying snapshot to destination" ; - let* path = - Tezt_cloud.Agent.copy agent ~destination:path ~source:path - in - Lwt.return_some path - | Url _ | No_snapshot -> Lwt.return_none - let import_snapshot ?(delete_snapshot_file = false) ~name node snapshot_file_path = toplog "Importing the snapshot for %s" name ; @@ -242,17 +188,6 @@ module Node = struct in Lwt.return_unit - let get_snapshot_info_network node snapshot_path = - let* info = Node.snapshot_info node ~json:true snapshot_path in - let json = JSON.parse ~origin:"snapshot_info" info in - (match JSON.(json |-> "snapshot_header" |-> "chain_name" |> as_string) with - | "TEZOS_ITHACANET_2022-01-25T15:00:00Z" -> "ghostnet" - | "TEZOS_RIONET_2025-02-19T12:45:00Z" -> "rionet" - | "TEZOS_SEOULNET_2025-07-11T08:00:00Z" -> "seoulnet" - | "TEZOS_MAINNET" -> "mainnet" - | "TEZOS" | _ -> "sandbox") - |> Lwt.return - let init ?(arguments = []) ?data_dir ?identity_file ?dal_config ?env ~rpc_external ~name network ~with_yes_crypto ~snapshot ?ppx_profiling cloud agent = @@ -2855,7 +2790,9 @@ let init_sandbox_and_activate_protocol cloud (configuration : configuration) let* snapshot_network = match configuration.snapshot with | Docker_embedded path | Local_file path -> - let* network = Node.get_snapshot_info_network bootstrap_node path in + let* network = + Snapshot_helpers.get_snapshot_info_network bootstrap_node path + in (* Yes-wallet requires the config url for protocol-specific test networks.*) let network = diff --git a/tezt/tests/cloud/layer1.ml b/tezt/tests/cloud/layer1.ml index d0cb12076224..ac5708a7934f 100644 --- a/tezt/tests/cloud/layer1.ml +++ b/tezt/tests/cloud/layer1.ml @@ -27,11 +27,6 @@ type baker_account = {pk : string; pkh : string} let add_prometheus_source node cloud agent = Scenarios_helpers.add_prometheus_source ~node cloud agent (Agent.name agent) -let get_snapshot_info_level node snapshot_path = - let* info = Node.snapshot_info node ~json:true snapshot_path in - let json = JSON.parse ~origin:"snapshot_info" info in - Lwt.return JSON.(json |-> "snapshot_header" |-> "level" |> as_int) - let yes_crypto_env = String_map.add Tezos_crypto.Helpers.yes_crypto_environment_variable @@ -57,51 +52,9 @@ let nb_stresstester network tps = max n1 n2 module Node = struct - let runner_of_agent = Agent.runner - open Snapshot_helpers include Node - let download_snapshot ~agent ~url ~name = - let downloaded_snapshot_file_path = "snapshot_file" in - toplog "Trying to download snapshot for %s from %s" name url ; - let* exit_status = - Process.spawn - ?runner:(runner_of_agent agent) - "wget" - ["-O"; downloaded_snapshot_file_path; sf "%s/rolling" url] - |> Process.wait - in - let* () = - match exit_status with - | WEXITED 0 -> Lwt.return_unit - | WEXITED code -> - toplog - "Could not download the snapshot for %s: wget exit code: %d\n\ - Starting without snapshot. It could last long before the node is \ - bootstrapped" - name - code ; - Lwt.return_unit - | status -> ( - match Process.validate_status status with - | Ok () -> Lwt.return_unit - | Error (`Invalid_status reason) -> - failwith @@ Format.sprintf "wget: %s" reason) - in - Lwt.return downloaded_snapshot_file_path - - let ensure_snapshot ~agent ~name ~network = function - | Docker_embedded path -> - toplog "Using locally stored snapshot file: %s" path ; - Lwt.return path - | Local_file path -> - toplog "Copying snapshot to destination" ; - Tezt_cloud.Agent.copy agent ~destination:path ~source:path - | Url url -> download_snapshot ~agent ~url ~name - | No_snapshot -> - download_snapshot ~agent ~url:(Network.snapshot_service network) ~name - (** We are running a private network with yes-crypto enabled. We don't want to connect with the real network. *) @@ -652,7 +605,7 @@ let number_of_bakers ~snapshot ~network cloud agent name = Node.Agent.create ~metadata_size_limit:false ~name:"tmp-node" cloud agent in let* snapshot = - Node.ensure_snapshot + Snapshot_helpers.ensure_snapshot ~agent ~name ~network:(Network.to_public network) diff --git a/tezt/tests/cloud/snapshot_helpers.ml b/tezt/tests/cloud/snapshot_helpers.ml index b433297e4e7c..befe64777afd 100644 --- a/tezt/tests/cloud/snapshot_helpers.ml +++ b/tezt/tests/cloud/snapshot_helpers.ml @@ -5,6 +5,8 @@ (* *) (*****************************************************************************) +open Scenarios_helpers + type t = | Docker_embedded of string | Local_file of string @@ -70,3 +72,69 @@ let encoding = (function Url s -> Some s | _ -> None) (fun s -> Url s); ] + +let get_snapshot_info_level node snapshot_path = + let* info = Node.snapshot_info node ~json:true snapshot_path in + let json = JSON.parse ~origin:"snapshot_info" info in + Lwt.return JSON.(json |-> "snapshot_header" |-> "level" |> as_int) + +let get_snapshot_info_network node snapshot_path = + let* info = Node.snapshot_info node ~json:true snapshot_path in + let json = JSON.parse ~origin:"snapshot_info" info in + (match JSON.(json |-> "snapshot_header" |-> "chain_name" |> as_string) with + | "TEZOS_ITHACANET_2022-01-25T15:00:00Z" -> "ghostnet" + | "TEZOS_RIONET_2025-02-19T12:45:00Z" -> "rionet" + | "TEZOS_SEOULNET_2025-07-11T08:00:00Z" -> "seoulnet" + | "TEZOS_MAINNET" -> "mainnet" + | "TEZOS" | _ -> "sandbox") + |> Lwt.return + +let download_snapshot ~agent ~url ~name = + let downloaded_snapshot_file_path = "snapshot_file" in + toplog "Trying to download snapshot for %s from %s" name url ; + let* exit_status = + Process.spawn + ?runner:(Agent.runner agent) + "wget" + ["-O"; downloaded_snapshot_file_path; sf "%s/rolling" url] + |> Process.wait + in + let* () = + match exit_status with + | WEXITED 0 -> Lwt.return_unit + | WEXITED code -> + toplog + "Could not download the snapshot for %s: wget exit code: %d\n\ + Starting without snapshot. It could last long before the node is \ + bootstrapped" + name + code ; + Lwt.return_unit + | status -> ( + match Process.validate_status status with + | Ok () -> Lwt.return_unit + | Error (`Invalid_status reason) -> + failwith @@ Format.sprintf "wget: %s" reason) + in + Lwt.return downloaded_snapshot_file_path + +let ensure_snapshot ~agent ~name ~network = function + | Docker_embedded path -> + toplog "Using locally stored snapshot file: %s" path ; + Lwt.return path + | Local_file path -> + toplog "Copying snapshot to destination" ; + Tezt_cloud.Agent.copy agent ~destination:path ~source:path + | Url url -> download_snapshot ~agent ~url ~name + | No_snapshot -> + download_snapshot ~agent ~url:(Network.snapshot_service network) ~name + +let ensure_snapshot_opt ~agent = function + | Docker_embedded path -> + toplog "Using locally stored snapshot file: %s" path ; + Lwt.return_some path + | Local_file path -> + toplog "Copying snapshot to destination" ; + let* path = Tezt_cloud.Agent.copy agent ~destination:path ~source:path in + Lwt.return_some path + | Url _ | No_snapshot -> Lwt.return_none -- GitLab From 6504ca6b9cd08bc97894d7ac0ecee24b89b85557 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Wed, 16 Jul 2025 18:46:36 +0100 Subject: [PATCH 06/11] Tezt_cloud: Unify snapshot import --- tezt/tests/cloud/dal.ml | 36 +++------------------------- tezt/tests/cloud/layer1.ml | 8 ++++--- tezt/tests/cloud/snapshot_helpers.ml | 32 +++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 36 deletions(-) diff --git a/tezt/tests/cloud/dal.ml b/tezt/tests/cloud/dal.ml index fc8a6235d707..3c8182239928 100644 --- a/tezt/tests/cloud/dal.ml +++ b/tezt/tests/cloud/dal.ml @@ -156,38 +156,6 @@ module Node = struct Tezos_crypto.Helpers.yes_crypto_environment_variable "y" - let import_snapshot ?(delete_snapshot_file = false) ~name node - snapshot_file_path = - toplog "Importing the snapshot for %s" name ; - let* () = - try - let* () = Node.snapshot_import ~no_check:true node snapshot_file_path in - let () = toplog "Snapshot import succeeded for %s." name in - let* () = - if delete_snapshot_file then ( - (* Delete the snapshot downloaded locally *) - toplog "Deleting downloaded snapshot (%s)" snapshot_file_path ; - let* (_ignored_exit_status : Unix.process_status) = - Process.wait (Process.spawn "rm" [snapshot_file_path]) - in - Lwt.return_unit) - else Lwt.return_unit - in - Lwt.return_unit - with _ -> - (* Failing to import the snapshot could happen on a very young - Weeklynet, before the first snapshot is available. In this - case bootstrapping from the genesis block is OK. *) - let () = - toplog - "Snapshot import failed for %s, the node will be bootstrapped from \ - genesis." - name - in - Lwt.return_unit - in - Lwt.return_unit - let init ?(arguments = []) ?data_dir ?identity_file ?dal_config ?env ~rpc_external ~name network ~with_yes_crypto ~snapshot ?ppx_profiling cloud agent = @@ -249,6 +217,7 @@ module Node = struct let* () = import_snapshot ~delete_snapshot_file:(snapshot = No_snapshot) + ~no_check:true ~name node snapshot_file_path @@ -315,7 +284,8 @@ module Node = struct let* () = may_copy_node_identity_file agent node identity_file in let* () = match snapshot_path with - | Some snapshot_path -> import_snapshot ~name node snapshot_path + | Some snapshot_path -> + import_snapshot ~no_check:true ~name node snapshot_path | None -> Lwt.return_unit in let* () = diff --git a/tezt/tests/cloud/layer1.ml b/tezt/tests/cloud/layer1.ml index ac5708a7934f..d528a52ed3da 100644 --- a/tezt/tests/cloud/layer1.ml +++ b/tezt/tests/cloud/layer1.ml @@ -146,7 +146,7 @@ module Node = struct ] in let* () = Node.config_init node config in - let* () = Node.snapshot_import ~no_check:true node snapshot in + let* () = import_snapshot ~no_check:true ~name node snapshot in let* () = (* When bootstrapping from the real network, we want to use the real time. *) let env = String_map.(add "FAKETIME" "+0" empty) in @@ -182,7 +182,7 @@ module Node = struct let* () = add_migration_offset_to_config node snapshot ~migration_offset ~network in - let* () = Node.snapshot_import ~no_check:true node snapshot in + let* () = import_snapshot ~no_check:true ~name node snapshot in let arguments = isolated_args peers in let* () = run ~env:yes_crypto_env node arguments in let* () = wait_for_ready node in @@ -614,7 +614,9 @@ let number_of_bakers ~snapshot ~network cloud agent name = let* () = Node.config_init node (Node.isolated_config ~peers:[] ~network ~delay:0) in - let* () = Node.snapshot_import ~no_check:true node snapshot in + let* () = + Snapshot_helpers.import_snapshot ~no_check:true ~name node snapshot + in let* () = Node.Agent.run node (Node.isolated_args []) in let* () = Node.wait_for_ready node in let* client = diff --git a/tezt/tests/cloud/snapshot_helpers.ml b/tezt/tests/cloud/snapshot_helpers.ml index befe64777afd..bfcd2dab64dc 100644 --- a/tezt/tests/cloud/snapshot_helpers.ml +++ b/tezt/tests/cloud/snapshot_helpers.ml @@ -138,3 +138,35 @@ let ensure_snapshot_opt ~agent = function let* path = Tezt_cloud.Agent.copy agent ~destination:path ~source:path in Lwt.return_some path | Url _ | No_snapshot -> Lwt.return_none + +let import_snapshot ?(delete_snapshot_file = false) ~no_check ~name node + snapshot_file_path = + toplog "Importing the snapshot for %s" name ; + let* () = + try + let* () = Node.snapshot_import ~no_check node snapshot_file_path in + let () = toplog "Snapshot import succeeded for %s." name in + let* () = + if delete_snapshot_file then ( + (* Delete the snapshot downloaded locally *) + toplog "Deleting downloaded snapshot (%s)" snapshot_file_path ; + let* (_ignored_exit_status : Unix.process_status) = + Process.wait (Process.spawn "rm" [snapshot_file_path]) + in + Lwt.return_unit) + else Lwt.return_unit + in + Lwt.return_unit + with _ -> + (* Failing to import the snapshot could happen on a very young + Weeklynet, before the first snapshot is available. In this + case bootstrapping from the genesis block is OK. *) + let () = + toplog + "Snapshot import failed for %s, the node will be bootstrapped from \ + genesis." + name + in + Lwt.return_unit + in + Lwt.return_unit -- GitLab From 0601cec1e2ca4536df894caf7a9fab2129864013 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Thu, 17 Jul 2025 10:43:23 +0100 Subject: [PATCH 07/11] Tezt_cloud: Add documentation for snapshot helpers module --- tezt/tests/cloud/snapshot_helpers.mli | 67 +++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 tezt/tests/cloud/snapshot_helpers.mli diff --git a/tezt/tests/cloud/snapshot_helpers.mli b/tezt/tests/cloud/snapshot_helpers.mli new file mode 100644 index 000000000000..55857c3134c7 --- /dev/null +++ b/tezt/tests/cloud/snapshot_helpers.mli @@ -0,0 +1,67 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2025 Trilitech *) +(* *) +(*****************************************************************************) + +(** Here we provide common utilities for managing snapshot files + in Tezt Cloud-based scenarios. + + Supported snapshot sources: + - Embedded in a Docker image (constant local path) + - Local files (copied to the agent) + - URLs (downloaded) + - No snapshot (fallback to downloading from a known network service) +*) + +type t = + | Docker_embedded of string + | Local_file of string + | Url of string + | No_snapshot + +(** Parse a raw snapshot CLI argument into a [Snapshot_helpers.t]. *) +val parse_snapshot : string option -> t + +(** Pretty-print a snapshot value. *) +val to_string : t -> string + +(** JSON encoding for snapshot representation. *) +val encoding : t Data_encoding.t + +(** Extract the level of a snapshot file using the node's [snapshot info] command. *) +val get_snapshot_info_level : Node.t -> string -> int Lwt.t + +(** Extract the network name associated with a snapshot file. *) +val get_snapshot_info_network : Node.t -> string -> string Lwt.t + +(** [download_snapshot ~agent ~url ~name] downloads a snapshot from a given [~url] + onto an [~agent]. The file is saved in the agent's working directory under [~name]. *) +val download_snapshot : + agent:Agent.t -> url:string -> name:string -> string Lwt.t + +(** [ensure_snapshot ~agent ~name ~network snapshot] ensures a [snapshot] is available + on the [~agent] by either: + - Returning a local Docker-embedded path + - Copying a local file to the [~agent] + - Downloading it from a remote URL provided in [snapshot] + - Fallback to [~network] snapshot service for [No_snapshot] *) +val ensure_snapshot : + agent:Agent.t -> name:string -> network:Network.public -> t -> string Lwt.t + +(** Like [ensure_snapshot], but returns [None] for [Url] or [No_snapshot] + cases instead of downloading. Useful when downloading is optional. *) +val ensure_snapshot_opt : agent:Agent.t -> t -> string option Lwt.t + +(** [import_snapshot ?delete_snapshot_file ~no_check ~name node snapshot_path] imports + a snapshot into a Tezos [node] at [snapshot_path], using [~name] for logging purposes. + [~no_check] can be set to skip validity checks during import. If [?delete_snapshot_file] + is [true], it deletes the snapshot file afterwards. *) +val import_snapshot : + ?delete_snapshot_file:bool -> + no_check:bool -> + name:string -> + Node.t -> + string -> + unit Lwt.t -- GitLab From b9bc0abdcd308ecfe52b436ae0adadb63b849210 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Thu, 17 Jul 2025 14:25:06 +0100 Subject: [PATCH 08/11] Tezt_cloud: Enable ensure_snapshot_opt to download from url --- tezt/tests/cloud/dal.ml | 2 +- tezt/tests/cloud/snapshot_helpers.ml | 7 +++++-- tezt/tests/cloud/snapshot_helpers.mli | 7 ++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tezt/tests/cloud/dal.ml b/tezt/tests/cloud/dal.ml index 3c8182239928..0570f78a49e4 100644 --- a/tezt/tests/cloud/dal.ml +++ b/tezt/tests/cloud/dal.ml @@ -254,7 +254,7 @@ module Node = struct Node.Agent.create ~net_addr ~rpc_external ~name cloud agent in let* () = Node.config_init node [Cors_origin "*"] in - let* snapshot_path = ensure_snapshot_opt ~agent snapshot in + let* snapshot_path = ensure_snapshot_opt ~agent ~name snapshot in let* snapshot_network = match snapshot_path with | Some path -> diff --git a/tezt/tests/cloud/snapshot_helpers.ml b/tezt/tests/cloud/snapshot_helpers.ml index bfcd2dab64dc..467d155e1d33 100644 --- a/tezt/tests/cloud/snapshot_helpers.ml +++ b/tezt/tests/cloud/snapshot_helpers.ml @@ -129,7 +129,7 @@ let ensure_snapshot ~agent ~name ~network = function | No_snapshot -> download_snapshot ~agent ~url:(Network.snapshot_service network) ~name -let ensure_snapshot_opt ~agent = function +let ensure_snapshot_opt ~agent ~name = function | Docker_embedded path -> toplog "Using locally stored snapshot file: %s" path ; Lwt.return_some path @@ -137,7 +137,10 @@ let ensure_snapshot_opt ~agent = function toplog "Copying snapshot to destination" ; let* path = Tezt_cloud.Agent.copy agent ~destination:path ~source:path in Lwt.return_some path - | Url _ | No_snapshot -> Lwt.return_none + | Url url -> + let* path = download_snapshot ~agent ~url ~name in + Lwt.return_some path + | No_snapshot -> Lwt.return_none let import_snapshot ?(delete_snapshot_file = false) ~no_check ~name node snapshot_file_path = diff --git a/tezt/tests/cloud/snapshot_helpers.mli b/tezt/tests/cloud/snapshot_helpers.mli index 55857c3134c7..9deef1083f2c 100644 --- a/tezt/tests/cloud/snapshot_helpers.mli +++ b/tezt/tests/cloud/snapshot_helpers.mli @@ -50,9 +50,10 @@ val download_snapshot : val ensure_snapshot : agent:Agent.t -> name:string -> network:Network.public -> t -> string Lwt.t -(** Like [ensure_snapshot], but returns [None] for [Url] or [No_snapshot] - cases instead of downloading. Useful when downloading is optional. *) -val ensure_snapshot_opt : agent:Agent.t -> t -> string option Lwt.t +(** Like [ensure_snapshot], but returns [None] for [No_snapshot] + cases instead of downloading. *) +val ensure_snapshot_opt : + agent:Agent.t -> name:string -> t -> string option Lwt.t (** [import_snapshot ?delete_snapshot_file ~no_check ~name node snapshot_path] imports a snapshot into a Tezos [node] at [snapshot_path], using [~name] for logging purposes. -- GitLab From c344b0623267601a10d471f4d45e5e029693223b Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Thu, 17 Jul 2025 15:44:38 +0100 Subject: [PATCH 09/11] Tezt_cloud: Factorise ensure_snapshot functions --- tezt/tests/cloud/snapshot_helpers.ml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/tezt/tests/cloud/snapshot_helpers.ml b/tezt/tests/cloud/snapshot_helpers.ml index 467d155e1d33..46e987d152a1 100644 --- a/tezt/tests/cloud/snapshot_helpers.ml +++ b/tezt/tests/cloud/snapshot_helpers.ml @@ -118,17 +118,6 @@ let download_snapshot ~agent ~url ~name = in Lwt.return downloaded_snapshot_file_path -let ensure_snapshot ~agent ~name ~network = function - | Docker_embedded path -> - toplog "Using locally stored snapshot file: %s" path ; - Lwt.return path - | Local_file path -> - toplog "Copying snapshot to destination" ; - Tezt_cloud.Agent.copy agent ~destination:path ~source:path - | Url url -> download_snapshot ~agent ~url ~name - | No_snapshot -> - download_snapshot ~agent ~url:(Network.snapshot_service network) ~name - let ensure_snapshot_opt ~agent ~name = function | Docker_embedded path -> toplog "Using locally stored snapshot file: %s" path ; @@ -142,6 +131,13 @@ let ensure_snapshot_opt ~agent ~name = function Lwt.return_some path | No_snapshot -> Lwt.return_none +let ensure_snapshot ~agent ~name ~network = function + | No_snapshot -> + download_snapshot ~agent ~url:(Network.snapshot_service network) ~name + | snapshot -> + let* path = ensure_snapshot_opt ~agent ~name snapshot in + Lwt.return @@ Option.get path + let import_snapshot ?(delete_snapshot_file = false) ~no_check ~name node snapshot_file_path = toplog "Importing the snapshot for %s" name ; -- GitLab From c114e46b813717795825e90801f82f9a92707d62 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Mon, 21 Jul 2025 09:55:30 +0100 Subject: [PATCH 10/11] Tezt_cloud: Update copyrights --- tezt/tests/cloud/dal.ml | 1 + tezt/tests/cloud/network.ml | 1 + tezt/tests/cloud/network.mli | 1 + tezt/tests/cloud/scenarios_cli.ml | 1 + tezt/tests/cloud/snapshot_helpers.ml | 1 + tezt/tests/cloud/snapshot_helpers.mli | 1 + 6 files changed, 6 insertions(+) diff --git a/tezt/tests/cloud/dal.ml b/tezt/tests/cloud/dal.ml index 0570f78a49e4..bb0188d7a4d8 100644 --- a/tezt/tests/cloud/dal.ml +++ b/tezt/tests/cloud/dal.ml @@ -2,6 +2,7 @@ (* *) (* SPDX-License-Identifier: MIT *) (* SPDX-FileCopyrightText: 2024 Nomadic Labs *) +(* Copyright (c) 2025 Trilitech *) (* *) (*****************************************************************************) diff --git a/tezt/tests/cloud/network.ml b/tezt/tests/cloud/network.ml index eb26859be82b..3234366dc290 100644 --- a/tezt/tests/cloud/network.ml +++ b/tezt/tests/cloud/network.ml @@ -2,6 +2,7 @@ (* *) (* SPDX-License-Identifier: MIT *) (* SPDX-FileCopyrightText: 2024 Nomadic Labs *) +(* Copyright (c) 2025 Trilitech *) (* *) (*****************************************************************************) diff --git a/tezt/tests/cloud/network.mli b/tezt/tests/cloud/network.mli index 3a7a79b0fbb2..b6cb0a5ab796 100644 --- a/tezt/tests/cloud/network.mli +++ b/tezt/tests/cloud/network.mli @@ -2,6 +2,7 @@ (* *) (* SPDX-License-Identifier: MIT *) (* SPDX-FileCopyrightText: 2024 Nomadic Labs *) +(* Copyright (c) 2025 Trilitech *) (* *) (*****************************************************************************) diff --git a/tezt/tests/cloud/scenarios_cli.ml b/tezt/tests/cloud/scenarios_cli.ml index 033b1c6fa9ae..e2625a620a0e 100644 --- a/tezt/tests/cloud/scenarios_cli.ml +++ b/tezt/tests/cloud/scenarios_cli.ml @@ -2,6 +2,7 @@ (* *) (* SPDX-License-Identifier: MIT *) (* SPDX-FileCopyrightText: 2024 Nomadic Labs *) +(* Copyright (c) 2025 Trilitech *) (* *) (*****************************************************************************) diff --git a/tezt/tests/cloud/snapshot_helpers.ml b/tezt/tests/cloud/snapshot_helpers.ml index 46e987d152a1..0d1cd4919d33 100644 --- a/tezt/tests/cloud/snapshot_helpers.ml +++ b/tezt/tests/cloud/snapshot_helpers.ml @@ -1,6 +1,7 @@ (*****************************************************************************) (* *) (* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2025 Nomadic Labs, *) (* Copyright (c) 2025 Trilitech *) (* *) (*****************************************************************************) diff --git a/tezt/tests/cloud/snapshot_helpers.mli b/tezt/tests/cloud/snapshot_helpers.mli index 9deef1083f2c..e5886fffad76 100644 --- a/tezt/tests/cloud/snapshot_helpers.mli +++ b/tezt/tests/cloud/snapshot_helpers.mli @@ -1,6 +1,7 @@ (*****************************************************************************) (* *) (* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2025 Nomadic Labs, *) (* Copyright (c) 2025 Trilitech *) (* *) (*****************************************************************************) -- GitLab From fe8f84e033dd77d1d865fa61963e3ecab6a118b6 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Mon, 21 Jul 2025 17:01:08 +0100 Subject: [PATCH 11/11] Tezt: Network: Add parsing function --- tezt/tests/cloud/network.ml | 16 ++++++++++++++++ tezt/tests/cloud/network.mli | 3 +++ tezt/tests/cloud/scenarios_cli.ml | 20 ++------------------ 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/tezt/tests/cloud/network.ml b/tezt/tests/cloud/network.ml index 3234366dc290..a5564192f568 100644 --- a/tezt/tests/cloud/network.ml +++ b/tezt/tests/cloud/network.ml @@ -33,6 +33,22 @@ let to_string = function | `Rionet -> "rionet" | `Seoulnet -> "seoulnet" +let parse = function + | "mainnet" -> Some `Mainnet + | "ghostnet" -> Some `Ghostnet + | "rionet" -> Some `Rionet + | "seoulnet" -> Some `Seoulnet + | s when String.length s = 20 && String.sub s 0 10 = "weeklynet-" -> + (* format: weeklynet-2025-01-29 (with dashes) *) + let date = String.sub s 10 10 in + Some (`Weeklynet date) + | s when String.length s = 16 && String.sub s 0 8 = "nextnet-" -> + (* format: nextnet-20250203 (without dashes) *) + let date = String.sub s 8 8 in + Some (`Nextnet date) + | "sandbox" -> Some `Sandbox + | _ -> None + let default_protocol : t -> Protocol.t = function | `Mainnet -> R022 | `Ghostnet -> R022 diff --git a/tezt/tests/cloud/network.mli b/tezt/tests/cloud/network.mli index b6cb0a5ab796..f679f521b28a 100644 --- a/tezt/tests/cloud/network.mli +++ b/tezt/tests/cloud/network.mli @@ -32,6 +32,9 @@ val to_public : t -> public (** ["mainnet" | "ghostnet" | "rionet" | "seoulnet" | "nextnet-%s" | "weeklynet-%s" | "sandbox"] *) val to_string : [< t] -> string +(** Parse the given [string] into an available network option. *) +val parse : string -> t option + (** Known protocol used by the network *) val default_protocol : t -> Protocol.t diff --git a/tezt/tests/cloud/scenarios_cli.ml b/tezt/tests/cloud/scenarios_cli.ml index e2625a620a0e..3fb6e8de615a 100644 --- a/tezt/tests/cloud/scenarios_cli.ml +++ b/tezt/tests/cloud/scenarios_cli.ml @@ -101,27 +101,11 @@ let parse_network_simulation_config_from_args simulate_network_arg = "Unexpected network simulation config (--simulation) [%s]" simulate_network_arg -let parse_network = function - | "mainnet" -> Some `Mainnet - | "ghostnet" -> Some `Ghostnet - | "rionet" -> Some `Rionet - | "seoulnet" -> Some `Seoulnet - | s when String.length s = 20 && String.sub s 0 10 = "weeklynet-" -> - (* format: weeklynet-2025-01-29 (with dashes) *) - let date = String.sub s 10 10 in - Some (`Weeklynet date) - | s when String.length s = 16 && String.sub s 0 8 = "nextnet-" -> - (* format: nextnet-20250203 (without dashes) *) - let date = String.sub s 8 8 in - Some (`Nextnet date) - | "sandbox" -> Some `Sandbox - | _ -> None - let network_typ : Network.t Clap.typ = Clap.typ ~name:"network" ~dummy:`Ghostnet - ~parse:parse_network + ~parse:Network.parse ~show:Network.to_string let snapshot_typ : Snapshot_helpers.t Clap.typ = @@ -318,7 +302,7 @@ module Dal () : Dal = struct let stake_repartition_typ : Network.stake_repartition Clap.typ = let open Network in let parse_public_network (net : string) : public option = - try Option.map to_public (parse_network net) with _ -> None + try Option.map to_public (parse net) with _ -> None in Clap.typ ~name:"stake_repartition" -- GitLab