From cd4db986b6a8849ac24b9ea74b46ac9f3452311e Mon Sep 17 00:00:00 2001 From: Pierrick Couderc Date: Mon, 1 Sep 2025 14:06:05 +0200 Subject: [PATCH 1/4] Tezt/Cloud: add function to generate config file from CLI parameters --- tezt/lib_cloud/cli.ml | 58 +++++++++++++++++++++++++++++++++++ tezt/lib_cloud/cli.mli | 7 +++++ tezt/lib_cloud/tezt_cloud.ml | 2 ++ tezt/lib_cloud/tezt_cloud.mli | 5 +++ 4 files changed, 72 insertions(+) diff --git a/tezt/lib_cloud/cli.ml b/tezt/lib_cloud/cli.ml index bd0592d6dd17..8d47bd7eb9fd 100644 --- a/tezt/lib_cloud/cli.ml +++ b/tezt/lib_cloud/cli.ml @@ -907,3 +907,61 @@ let slack_bot_token = module Types = struct let dir_typ = dir_typ end + +let to_json_config ?scenario_config () = + let scenario_specific = + match scenario_config with + | None -> scenario_specific + | Some _ -> scenario_config + in + { + localhost = Some localhost; + ssh_host; + monitoring = Some monitoring; + destroy = Some destroy; + keep_alive = Some keep_alive; + project_id; + vms; + vm_base_port = Some vm_base_port; + ports_per_vm = Some ports_per_vm; + proxy = Some proxy; + os = Some os; + grafana = Some grafana; + grafana_legacy_source = Some grafana_legacy_source; + alert_handlers; + prometheus = Some prometheus; + prometheus_export = Some prometheus_export; + prometheus_port = Some prometheus_port; + prometheus_export_path; + prometheus_snapshots; + prometheus_scrape_interval = Some prometheus_scrape_interval; + process_monitoring = Some process_monitoring; + website = Some website; + machine_type = Some machine_type; + dockerfile_alias; + website_port = Some website_port; + max_run_duration = Some max_run_duration; + no_max_run_duration = Some no_max_run_duration; + tezt_cloud; + dns_domains; + no_dns = Some no_dns; + open_telemetry = Some open_telemetry; + macosx = Some macosx; + check_file_consistency = Some check_file_consistency; + docker_host_network = Some docker_host_network; + push_docker = Some push_docker; + auto_approve = Some auto_approve; + faketime; + binaries_path = Some binaries_path; + log_rotation = Some log_rotation; + slack_channel_id; + slack_bot_token; + retrieve_daily_logs = Some retrieve_daily_logs; + retrieve_ppx_profiling_traces = Some retrieve_ppx_profiling_traces; + scenario_specific; + tc_delay; + tc_jitter; + artifacts_dir; + teztale_artifacts = Some teztale_artifacts; + } + |> Data_encoding.Json.construct encoding diff --git a/tezt/lib_cloud/cli.mli b/tezt/lib_cloud/cli.mli index d4e0883da809..edd19d6c3169 100644 --- a/tezt/lib_cloud/cli.mli +++ b/tezt/lib_cloud/cli.mli @@ -180,3 +180,10 @@ val slack_bot_token : string option module Types : sig val dir_typ : name:string -> cli_parameter:string -> string option Clap.typ end + +(** [to_json_config ~scenario_name ?scenario_config ()] generates the full + configuration file encoded in JSON, each option being retrieved from the CLI + or default parameters. If [scenario_config] is not given, it encodes the + one from the CLI. *) +val to_json_config : + ?scenario_config:string * Data_encoding.Json.t -> unit -> Data_encoding.Json.t diff --git a/tezt/lib_cloud/tezt_cloud.ml b/tezt/lib_cloud/tezt_cloud.ml index 21fc8570cc86..948cf1faecf4 100644 --- a/tezt/lib_cloud/tezt_cloud.ml +++ b/tezt/lib_cloud/tezt_cloud.ml @@ -314,4 +314,6 @@ module Tezt_cloud_cli = struct let artifacts_dir = Cli.artifacts_dir let teztale_artifacts = Cli.teztale_artifacts + + let to_json_config = Cli.to_json_config end diff --git a/tezt/lib_cloud/tezt_cloud.mli b/tezt/lib_cloud/tezt_cloud.mli index 3a90233090f7..f92fe5a97f70 100644 --- a/tezt/lib_cloud/tezt_cloud.mli +++ b/tezt/lib_cloud/tezt_cloud.mli @@ -273,6 +273,11 @@ module Tezt_cloud_cli : sig val artifacts_dir : string option val teztale_artifacts : bool + + val to_json_config : + ?scenario_config:string * Data_encoding.Json.t -> + unit -> + Data_encoding.Json.t end (** [register ~tags] register a set of jobs that can be used for setting -- GitLab From 9af69192f66829bd4cc616518b431dcf8dca09a7 Mon Sep 17 00:00:00 2001 From: Pierrick Couderc Date: Mon, 1 Sep 2025 15:39:26 +0200 Subject: [PATCH 2/4] Tezt/Cloud: create `Artifact_helpers` module --- tezt/lib_cloud/tezt_cloud.ml | 11 +++++++++++ tezt/lib_cloud/tezt_cloud.mli | 8 ++++++++ tezt/tests/cloud/agent_kind.ml | 19 +++++++------------ tezt/tests/cloud/agent_kind.mli | 6 ------ tezt/tests/cloud/tezos.ml | 4 ++-- 5 files changed, 28 insertions(+), 20 deletions(-) diff --git a/tezt/lib_cloud/tezt_cloud.ml b/tezt/lib_cloud/tezt_cloud.ml index 948cf1faecf4..aa1c758775d9 100644 --- a/tezt/lib_cloud/tezt_cloud.ml +++ b/tezt/lib_cloud/tezt_cloud.ml @@ -317,3 +317,14 @@ module Tezt_cloud_cli = struct let to_json_config = Cli.to_json_config end + +module Artifact_helpers = struct + let local_path path = + List.fold_left + (fun prefix subdir -> + let prefix = prefix // subdir in + if not (Sys.file_exists prefix) then Sys.mkdir prefix 0o755 ; + prefix) + "" + path +end diff --git a/tezt/lib_cloud/tezt_cloud.mli b/tezt/lib_cloud/tezt_cloud.mli index f92fe5a97f70..a3e399dc6b50 100644 --- a/tezt/lib_cloud/tezt_cloud.mli +++ b/tezt/lib_cloud/tezt_cloud.mli @@ -280,6 +280,14 @@ module Tezt_cloud_cli : sig Data_encoding.Json.t end +module Artifact_helpers : sig + (** [local_path path] ensures that the path (as a list of subdirectories) is a + valid directory, creates it otherwise, then returns the path. If the first + directory starts with `/`, it will start from the root of the + filesystem. *) + val local_path : string list -> string +end + (** [register ~tags] register a set of jobs that can be used for setting requirements related to cloud scenarios. Some tags can be given for all the registered jobs. *) diff --git a/tezt/tests/cloud/agent_kind.ml b/tezt/tests/cloud/agent_kind.ml index e8ed1333534e..61e4a0ffe77f 100644 --- a/tezt/tests/cloud/agent_kind.ml +++ b/tezt/tests/cloud/agent_kind.ml @@ -65,15 +65,6 @@ let name_of_daemon = function | Etherlink_producer_node name -> Format.asprintf "etherlink-%s-node" name module Logs = struct - let local_path path = - List.fold_left - (fun prefix subdir -> - let prefix = prefix // subdir in - if not (Sys.file_exists prefix) then Sys.mkdir prefix 0o755 ; - prefix) - "" - path - let scp_profiling ~destination_root ~daemon_name agent = let agent_name = Agent.name agent in (* This is not compatible with the --proxy mode as the Agent's location of @@ -88,7 +79,8 @@ module Logs = struct Lwt.return_unit | Some _runner -> let runner_local_path = - local_path [destination_root; agent_name; daemon_name] + Artifact_helpers.local_path + [destination_root; agent_name; daemon_name] in let process = Agent.docker_run_command agent "ls" [tezt_root_path // daemon_name] @@ -104,7 +96,9 @@ module Logs = struct in Lwt_list.iter_s (fun file -> - let local_path = local_path [runner_local_path; "profiling"] in + let local_path = + Artifact_helpers.local_path [runner_local_path; "profiling"] + in Lwt.catch (fun () -> Agent.scp @@ -134,7 +128,8 @@ module Logs = struct Lwt.return_unit | Some _runner -> let local_path = - local_path [destination_root; agent_name; daemon_name] + Artifact_helpers.local_path + [destination_root; agent_name; daemon_name] in Lwt.catch (fun () -> diff --git a/tezt/tests/cloud/agent_kind.mli b/tezt/tests/cloud/agent_kind.mli index 235a52feedc6..b9167c586c30 100644 --- a/tezt/tests/cloud/agent_kind.mli +++ b/tezt/tests/cloud/agent_kind.mli @@ -57,12 +57,6 @@ type daemon = val name_of_daemon : daemon -> string module Logs : sig - (** [local_path path] ensures that the path (as a list of subdirectories) is a - valid directory, creates it otherwise, then returns the path. If the first - directory starts with `/`, it will start from the root of the - filesystem. *) - val local_path : string list -> string - (** [scp_logs ~destination_root ~daemon_name agent] uses scp to copy the `daily_logs` directory from the VM hosting the [agent]'s actor given by [~daemon_name] into [~destination_root//~daemon_name/daily_logs]. diff --git a/tezt/tests/cloud/tezos.ml b/tezt/tests/cloud/tezos.ml index 69300ee83d78..dd2884a9520e 100644 --- a/tezt/tests/cloud/tezos.ml +++ b/tezt/tests/cloud/tezos.ml @@ -1099,7 +1099,7 @@ module Teztale = struct Lwt_list.iter_s (fun _archiver -> let db_destination = - Agent_kind.Logs.local_path + Artifact_helpers.local_path [dir; Filename.basename server.filenames.db_filename] in let* () = @@ -1111,7 +1111,7 @@ module Teztale = struct `FromRunner in let conf_destination = - Agent_kind.Logs.local_path + Artifact_helpers.local_path [dir; Filename.basename server.filenames.conf_filename] in Agent.scp -- GitLab From ad919c257c5eb452b87e4ba2b0a53dc4f178d86c Mon Sep 17 00:00:00 2001 From: Pierrick Couderc Date: Mon, 1 Sep 2025 15:09:12 +0200 Subject: [PATCH 3/4] Tezt/Cloud/Layer1: prepare the artifacts --- tezt/lib_cloud/tezt_cloud.ml | 26 ++++++++++++++++++++++++++ tezt/lib_cloud/tezt_cloud.mli | 6 ++++++ tezt/tests/cloud/layer1.ml | 8 ++++++++ 3 files changed, 40 insertions(+) diff --git a/tezt/lib_cloud/tezt_cloud.ml b/tezt/lib_cloud/tezt_cloud.ml index aa1c758775d9..6f7b3e3a58b3 100644 --- a/tezt/lib_cloud/tezt_cloud.ml +++ b/tezt/lib_cloud/tezt_cloud.ml @@ -319,6 +319,16 @@ module Tezt_cloud_cli = struct end module Artifact_helpers = struct + let parse_path path = + let path = String.split_on_char '/' path in + (* Check if the path is absolute or relative *) + let starts_from_root = match path with "" :: _ -> true | _ -> false in + (* Removes empty dir that would be the result of "//" *) + let path = List.filter (( <> ) "") path in + match path with + | dir :: subpath when starts_from_root -> ("/" ^ dir) :: subpath + | _ -> path + let local_path path = List.fold_left (fun prefix subdir -> @@ -327,4 +337,20 @@ module Artifact_helpers = struct prefix) "" path + + let prepare_artifacts ?scenario_config () = + match Tezt_cloud_cli.artifacts_dir with + | None -> () + | Some artifacts_dir -> + (* Ensures the artifact directory exists and creates it otherwise *) + let artifacts_dir = local_path (parse_path artifacts_dir) in + let full_configuration = + Tezt_cloud_cli.to_json_config ?scenario_config () + in + let full_configuration_string = + Data_encoding.Json.to_string full_configuration + in + write_file + (artifacts_dir // "configuration.json") + ~contents:full_configuration_string end diff --git a/tezt/lib_cloud/tezt_cloud.mli b/tezt/lib_cloud/tezt_cloud.mli index a3e399dc6b50..23b6dc98d5b6 100644 --- a/tezt/lib_cloud/tezt_cloud.mli +++ b/tezt/lib_cloud/tezt_cloud.mli @@ -286,6 +286,12 @@ module Artifact_helpers : sig directory starts with `/`, it will start from the root of the filesystem. *) val local_path : string list -> string + + (** [prepate_artifacts ?scenario_config ()] ensures the + `Tezt_cloud_cli.artifacts_dir` is a valid dir (and creates it if needed), + and push the full configuration in it. *) + val prepare_artifacts : + ?scenario_config:string * Data_encoding.json -> unit -> unit end (** [register ~tags] register a set of jobs that can be used for setting diff --git a/tezt/tests/cloud/layer1.ml b/tezt/tests/cloud/layer1.ml index 8775c13baca9..0dd80fb7de97 100644 --- a/tezt/tests/cloud/layer1.ml +++ b/tezt/tests/cloud/layer1.ml @@ -1183,6 +1183,14 @@ let register (module Cli : Scenarios_cli.Layer1) = ~title:"L1 simulation" ~tags:[] @@ fun cloud -> + toplog "Prepare artifacts directory and export the full configuration" ; + Artifact_helpers.prepare_artifacts + ~scenario_config: + ( "LAYER1", + Data_encoding.Json.construct + Scenarios_configuration.LAYER1.encoding + configuration ) + () ; toplog "Creating the agents" ; let agents = Cloud.agents cloud in toplog "Created %d agents" (List.length agents) ; -- GitLab From 2f7b4bda7b13f81affd48cc14227a341ab79f1a4 Mon Sep 17 00:00:00 2001 From: Pierrick Couderc Date: Mon, 1 Sep 2025 15:55:44 +0200 Subject: [PATCH 4/4] Tezt/Cloud/Dal: prepare the artifacts --- tezt/tests/cloud/dal.ml | 8 ++++++++ tezt/tests/cloud/scenarios_cli.ml | 2 ++ 2 files changed, 10 insertions(+) diff --git a/tezt/tests/cloud/dal.ml b/tezt/tests/cloud/dal.ml index 4f8e2e3c2c2e..ce80214876d9 100644 --- a/tezt/tests/cloud/dal.ml +++ b/tezt/tests/cloud/dal.ml @@ -1414,6 +1414,14 @@ let register (module Cli : Scenarios_cli.Dal) = ~title:"DAL node benchmark" ~tags:[] (fun cloud -> + toplog "Prepare artifacts directory and export the full configuration" ; + Artifact_helpers.prepare_artifacts + ~scenario_config: + ( "DAL", + Data_encoding.Json.construct + Scenarios_configuration.DAL.encoding + Cli.config ) + () ; toplog "Creating the agents" ; if (not configuration.with_dal) diff --git a/tezt/tests/cloud/scenarios_cli.ml b/tezt/tests/cloud/scenarios_cli.ml index 69ed6e6c7b50..071748a24c58 100644 --- a/tezt/tests/cloud/scenarios_cli.ml +++ b/tezt/tests/cloud/scenarios_cli.ml @@ -122,6 +122,8 @@ module type Dal = sig val slot_size : int option val number_of_slots : int option + + val config : Scenarios_configuration.DAL.t end module Dal () : Dal = struct -- GitLab