diff --git a/tezt/lib_tezos/constant.ml b/tezt/lib_tezos/constant.ml index 317bc835e34f32661cf5df671d2872569ec2c3f1..c7c94d196e76afc598843ad2ef1d11bce7d35961 100644 --- a/tezt/lib_tezos/constant.ml +++ b/tezt/lib_tezos/constant.ml @@ -72,10 +72,10 @@ let smart_rollup_installer = let _octez_smart_rollup_wasm_debugger = Uses.make ~tag:"wasm_debugger" ~path:"./octez-smart-rollup-wasm-debugger" -let _teztale_archiver = +let teztale_archiver = Uses.make ~tag:"teztale_archiver" ~path:"./octez-teztale-archiver" -let _teztale_server = +let teztale_server = Uses.make ~tag:"teztale_server" ~path:"./octez-teztale-server" module WASM = struct diff --git a/tezt/lib_tezos/teztale.ml b/tezt/lib_tezos/teztale.ml new file mode 100644 index 0000000000000000000000000000000000000000..85d678025d7a7351aaf41a321be02dcfacd2a406 --- /dev/null +++ b/tezt/lib_tezos/teztale.ml @@ -0,0 +1,176 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2024 Nomadic Labs, *) +(* *) +(*****************************************************************************) + +type user = {login : string; password : string} + +type interface = {address : string; port : int} + +let fresh_name base = + let i = ref (-1) in + fun () -> + incr i ; + base ^ "-" ^ string_of_int !i + +module Server = struct + let fresh_name = fresh_name "teztale-server" + + type conf = { + name : string; + interface : interface; + users : user list; + admin : user; + } + + type filenames = {conf_filename : string; db_filename : string} + + type t = {process : Process.t; filenames : filenames; conf : conf} + + let make_conf ~name ~address ~port ~users ~admin = + {name; interface = {address; port}; users; admin} + + (* We could use the libs from teztale in order to build the conf file using ocaml types + and printing it as json into a file, but using a string also tests that nothing + changed/broke with existing conf files. + *) + let dump_conf ?runner conf = + let tmp fn = Temp.file ?runner fn in + let conf_filename = tmp (Printf.sprintf "%s.conf.json" conf.name) in + let db_filename = tmp (Printf.sprintf "%s.sqlite" conf.name) in + let pp_login fmt {login; password} = + Format.fprintf fmt {|{"login": "%s", "password": "%s"}|} login password + in + let pp_login_list = + Format.pp_print_list + ~pp_sep:(fun fmt () -> Format.fprintf fmt ";") + pp_login + in + let pp_interface fmt {address; port} = + Format.fprintf fmt {|{"address": "%s", "port": %d}|} address port + in + let contents = + (* Verbosity needs to be set to a minimum level of INFO + because wait_for_readiness can't be used with a lower verbosity *) + Format.asprintf + {|{ + "db": "sqlite3:%s", + "interfaces": [%a], + "users": [%a], + "admins": [%a], + "with_transaction": "FULL", + "verbosity": "INFO" +}|} + db_filename + pp_interface + conf.interface + pp_login_list + conf.users + pp_login + conf.admin + in + let* () = + match runner with + | None -> write_file conf_filename ~contents |> Lwt.return + | Some runner -> + let cmd = + Runner.Shell.( + redirect_stdout (cmd [] "echo" [contents]) conf_filename) + in + let cmd, args = Runner.wrap_with_ssh runner cmd in + Process.run cmd args + in + Lwt.return {conf_filename; db_filename} + + let make ?name ?(address = "127.0.0.1") ?port ?(users = []) + ?(admin = {login = "admin"; password = "password"}) () = + let port = match port with Some port -> port | None -> Port.fresh () in + let name = match name with Some name -> name | None -> fresh_name () in + make_conf ~name ~address ~port ~users ~admin + + let run ?runner ?(path = Uses.path Constant.teztale_server) ?name ?address + ?port ?users ?admin () = + let conf = make ?name ?address ?port ?users ?admin () in + let* filenames = dump_conf ?runner conf in + let process = + Process.spawn ~name:conf.name ?runner path [filenames.conf_filename] + in + Lwt.return {process; filenames; conf} + + let wait_for_readiness t = + Log.info "Wait for %s to be ready" t.conf.name ; + let stdout = Process.stdout t.process in + let suffix = + Printf.sprintf + "Server listening at %s:%d." + t.conf.interface.address + t.conf.interface.port + in + let rec wait () = + let* line = Lwt_io.read_line stdout in + if String.ends_with ~suffix line then Lwt.return_unit else wait () + in + wait () + + (** Return *) + let add_user {conf = {interface; admin; _}; _} user = + let user = + JSON.parse + ~origin:__LOC__ + (Printf.sprintf + {|{"login": "%s", "password": "%s"}|} + user.login + user.password) + in + let url = + Format.asprintf + "http://%s:%s@%s:%d/user" + admin.login + admin.password + interface.address + interface.port + in + Curl.put url user |> Runnable.run + |> Lwt.map (fun json -> + match JSON.(get "status" json |> as_string) with + | "OK" -> Ok () + | status -> + let msg = + Printf.sprintf "%s: teztale answered with: %s" __LOC__ status + in + Error (Failure msg) + | exception e -> Error e) +end + +module Archiver = struct + let fresh_name = fresh_name "teztale-archiver" + + type conf = {name : string; user : user; feed : interface list} + + type t = {process : Process.t; conf : conf} + + let run ?runner ?(path = Uses.path Constant.teztale_archiver) ?name ~node_port + user feed = + let name = match name with Some name -> name | None -> fresh_name () in + let node_endpoint = Format.asprintf "http://127.0.0.1:%d" node_port in + let conf = {name; user; feed} in + let args = + "--endpoint" :: node_endpoint + :: List.fold_left + (fun acc feed -> + "feed" + :: Printf.sprintf + "http://%s:%s@%s:%d" + user.login + user.password + feed.address + feed.port + :: acc) + [] + conf.feed + in + let process = Process.spawn ~name:conf.name ?runner path args in + Lwt.return {process; conf} +end diff --git a/tezt/lib_tezos/teztale.mli b/tezt/lib_tezos/teztale.mli new file mode 100644 index 0000000000000000000000000000000000000000..6f05827071889b93f87e463107c13d8fc09fb092 --- /dev/null +++ b/tezt/lib_tezos/teztale.mli @@ -0,0 +1,91 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2024 Nomadic Labs, *) +(* *) +(*****************************************************************************) + +(** A user is either an archiver or an admin *) +type user = {login : string; password : string} + +type interface = {address : string; port : int} + +module Server : sig + (** See parameters of {!run} function for details. *) + type conf = { + name : string; + interface : interface; + users : user list; + admin : user; + } + + (** Expose paths of the files involved in the test: + - Server configuration file + - SQLite database + *) + type filenames = {conf_filename : string; db_filename : string} + + type t = {process : Process.t; filenames : filenames; conf : conf} + + (** [run ?runner ?path ?name ?address ?port ?users ?admin ()] + + Spawn a teztale server with some given parameters: + - runner: runner used to spawn the process + - path: path of the teztale server executable + - name: name the the server used in logs + - address: where teztale server will be listening on (default is [127.0.0.1]) + - port: port associated with [address] + - users: list of archivers allowed to feed the server. You can add + users once the server is started using the {!add_user} function. + - admin: credential used for adming tasks such as adding an allowed archiver + default is [admin:password] + *) + val run : + ?runner:Runner.t -> + ?path:string -> + ?name:string -> + ?address:string -> + ?port:int -> + ?users:user list -> + ?admin:user -> + unit -> + t Lwt.t + + (** Wait until teztale server is listening to its defined interface *) + val wait_for_readiness : t -> unit Lwt.t + + (** [add_user server user] + Add an archiver (its credentials) allowed to feed the database. + *) + val add_user : t -> user -> (unit, exn) Result.t Lwt.t +end + +module Archiver : sig + (** See parameters of {!run} function for details. *) + type conf = {name : string; user : user; feed : interface list} + + type t = {process : Process.t; conf : conf} + + (** [run ?runner ?path ?name ~node_port user feed] + + Spawn a teztale archiver with some given parameters: + - runner: runner used to spawn the process + - path: path of the teztale server executable + - name: name the the server used in logs + - node_port: port used for the octez node RPCs + - user: login and password used by the archiver. Make sure that + login and password used have been allowed in the server. See + [?users] argument of {!Server.run} or {!Server.add_user}. + - feed: list of interfaces for teztale servers to feed. If the archiver + is feeding multiple servers, every server must accept login info defined + by [user] parameter. + *) + val run : + ?runner:Runner.t -> + ?path:string -> + ?name:string -> + node_port:int -> + user -> + interface list -> + t Lwt.t +end diff --git a/tezt/tests/cloud/dal.ml b/tezt/tests/cloud/dal.ml index ddb9ebcb7547ff4dd0711e4e485c1e744e1f0ce0..3582fe8af5c45967424b3d54ba923538c223f5ad 100644 --- a/tezt/tests/cloud/dal.ml +++ b/tezt/tests/cloud/dal.ml @@ -248,92 +248,33 @@ end module Teztale = struct type t = { - server_daemon : Process.t; - port : int; - mutable archivers : Process.t list; + server : Teztale.Server.t; + mutable archivers : Teztale.Archiver.t list; } - let make_configuration ~port = - let teztale_sqlite = - Format.asprintf - "sqlite3:%s/teztale.sqlite" - (Filename.get_temp_dir_name ()) - in - Format.asprintf - {| -{ - "db": "%s", - "interfaces": [ - { - "address": "127.0.0.1", - "port": %d - } - ], - "admins": [ - { - "login": "admin", - "password": "saucisse" - } - ], - "users": [ - { - "login": "user", - "password": "saucisse" - } - ], - "with_transaction": "FULL" -} -|} - teztale_sqlite - port + let fresh_user = + let i = ref (-1) in + fun () : Teztale.user -> + incr i ; + let login = "teztale-archiver-" ^ string_of_int !i in + {login; password = login} let run_server ?(path = Uses.(path (make ~tag:"codec" ~path:"./octez-teztale-server"))) agent = - let runner = Agent.runner agent in - let configuration_file = - Filename.get_temp_dir_name () // "teztale-config.json" - in - let port = Agent.next_available_port agent in - let configuration = make_configuration ~port in - let* () = - match runner with - | None -> - write_file configuration_file ~contents:configuration ; - Lwt.return_unit - | Some runner -> - let cmd = - Runner.Shell.( - redirect_stdout (cmd [] "echo" [configuration]) configuration_file) - in - let cmd, args = Runner.wrap_with_ssh runner cmd in - Process.spawn cmd args |> Process.check - in - let* path = Agent.copy agent ~source:path in - let server_daemon = - Process.spawn ~name:"teztale-server" ?runner path [configuration_file] - in - (* Wait a bit it starts. *) - let* () = Lwt_unix.sleep 0.5 in - Lwt.return {server_daemon; port; archivers = []} + let* server = Teztale.Server.run ~path agent () in + Lwt.return {server; archivers = []} - let run_archiver + let wait_server t = Teztale.Server.wait_for_readiness t.server + + let add_archiver ?(path = Uses.(path (make ~tag:"codec" ~path:"./octez-teztale-archiver"))) t agent ~node_port = - let runner = Agent.runner agent in - let node_endpoint = Format.asprintf "http://127.0.0.1:%d" node_port in - let teztale_endpoint = - Format.asprintf "http://user:saucisse@127.0.0.1:%d" t.port - in - let* path = Agent.copy agent ~source:path in - let archiver_daemon = - Process.spawn - ~name:"teztale-archiver" - ?runner - path - ["--endpoint"; node_endpoint; "feed"; teztale_endpoint] - in - t.archivers <- archiver_daemon :: t.archivers ; + let user = fresh_user () in + let feed : Teztale.interface list = [t.server.conf.interface] in + let* () = Lwt_result.get_exn (Teztale.Server.add_user t.server user) in + let* archiver = Teztale.Archiver.run agent ~path user feed ~node_port in + t.archivers <- archiver :: t.archivers ; Lwt.return_unit end @@ -1119,8 +1060,9 @@ let add_etherlink_source cloud agent ~job_name ?dal_node node sc_rollup_node let init_teztale agent node = if Cli.teztale then let* teztale = Teztale.run_server agent in + let* () = Teztale.wait_server teztale in let* () = - Teztale.run_archiver teztale agent ~node_port:(Node.rpc_port node) + Teztale.add_archiver teztale agent ~node_port:(Node.rpc_port node) in Lwt.return_some teztale else Lwt.return_none diff --git a/tezt/tests/cloud/tezos.ml b/tezt/tests/cloud/tezos.ml index e5819d5214e58902057b19ad03a3db0add7b0f66..c36cfea158d70e51dfc2c4143718d9217d99f588 100644 --- a/tezt/tests/cloud/tezos.ml +++ b/tezt/tests/cloud/tezos.ml @@ -210,3 +210,40 @@ module Baker = struct client end end + +module Teztale = struct + include Teztale + + module Server = struct + include Teztale.Server + + let run agent ?(path = Uses.path Constant.teztale_server) ?name ?address + ?port ?users ?admin () = + let runner = Agent.runner agent in + let address = + match address with + | Some address -> address + | None -> Agent.point agent |> Option.map fst + in + let port = + match port with + | Some port -> port + | None -> ( + match Agent.point agent |> Option.map snd with + | Some port -> port + | None -> Agent.next_available_port agent) + in + let* path = Agent.copy agent ~source:path in + Teztale.Server.run ?runner ~path ~port ?name ?address ?users ?admin () + end + + module Archiver = struct + include Teztale.Archiver + + let run agent ?(path = Uses.path Constant.teztale_archiver) ?name ~node_port + user feed = + let runner = Agent.runner agent in + let* path = Agent.copy agent ~source:path in + Teztale.Archiver.run ?runner ~path ?name ~node_port user feed + end +end