From 12dd87b1bf9adef49a072b662a77b8fc778e08eb Mon Sep 17 00:00:00 2001 From: Julien Sagot Date: Thu, 29 Aug 2024 10:48:54 +0200 Subject: [PATCH 1/3] Tezt/Teztale: add Teztale module in lib_tezos --- tezt/lib_tezos/constant.ml | 4 +- tezt/lib_tezos/teztale.ml | 158 +++++++++++++++++++++++++++++++++++++ tezt/lib_tezos/teztale.mli | 51 ++++++++++++ tezt/tests/cloud/dal.ml | 95 ++++------------------ tezt/tests/cloud/tezos.ml | 37 +++++++++ 5 files changed, 265 insertions(+), 80 deletions(-) create mode 100644 tezt/lib_tezos/teztale.ml create mode 100644 tezt/lib_tezos/teztale.mli diff --git a/tezt/lib_tezos/constant.ml b/tezt/lib_tezos/constant.ml index 317bc835e34f..c7c94d196e76 100644 --- a/tezt/lib_tezos/constant.ml +++ b/tezt/lib_tezos/constant.ml @@ -72,10 +72,10 @@ let smart_rollup_installer = let _octez_smart_rollup_wasm_debugger = Uses.make ~tag:"wasm_debugger" ~path:"./octez-smart-rollup-wasm-debugger" -let _teztale_archiver = +let teztale_archiver = Uses.make ~tag:"teztale_archiver" ~path:"./octez-teztale-archiver" -let _teztale_server = +let teztale_server = Uses.make ~tag:"teztale_server" ~path:"./octez-teztale-server" module WASM = struct diff --git a/tezt/lib_tezos/teztale.ml b/tezt/lib_tezos/teztale.ml new file mode 100644 index 000000000000..d8e513a3dbc4 --- /dev/null +++ b/tezt/lib_tezos/teztale.ml @@ -0,0 +1,158 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2024 Nomadic Labs, *) +(* *) +(*****************************************************************************) + +type user = {login : string; password : string} + +type interface = {address : string; port : int} + +let fresh_name base = + let i = ref (-1) in + fun () -> + incr i ; + base ^ "-" ^ string_of_int !i + +module Server = struct + let fresh_name = fresh_name "teztale-server" + + type conf = { + name : string; + interface : interface; + users : user list; + admin : user; + } + + type filenames = {conf_filename : string; db_filename : string} + + type t = {process : Process.t; filenames : filenames; conf : conf} + + let make_conf ~name ~address ~port ~users ~admin = + {name; interface = {address; port}; users; admin} + + (* We could use the libs from teztale in order to build the conf file using ocaml types + and printing it as json into a file, but using a string also tests that nothing + changed/broke with existing conf files. + *) + let dump_conf ?runner conf = + let tmp fn = Temp.file ?runner fn in + let conf_filename = tmp (Printf.sprintf "%s.conf.json" conf.name) in + let db_filename = tmp (Printf.sprintf "%s.sqlite" conf.name) in + let pp_login fmt {login; password} = + Format.fprintf fmt {|{"login": "%s", "password": "%s"}|} login password + in + let pp_login_list = + Format.pp_print_list + ~pp_sep:(fun fmt () -> Format.fprintf fmt ";") + pp_login + in + let pp_interface fmt {address; port} = + Format.fprintf fmt {|{"address": "%s", "port": %d}|} address port + in + let contents = + Format.asprintf + {|{ + "db": "sqlite3:%s", + "interfaces": [%a], + "users": [%a], + "admins": [%a], + "with_transaction": "FULL" +}|} + db_filename + pp_interface + conf.interface + pp_login_list + conf.users + pp_login + conf.admin + in + let* () = + match runner with + | None -> write_file conf_filename ~contents |> Lwt.return + | Some runner -> + let cmd = + Runner.Shell.( + redirect_stdout (cmd [] "echo" [contents]) conf_filename) + in + let cmd, args = Runner.wrap_with_ssh runner cmd in + Process.run cmd args + in + Lwt.return {conf_filename; db_filename} + + let make ?name ?(address = "127.0.0.1") ?port ?(users = []) + ?(admin = {login = "admin"; password = "password"}) () = + let port = match port with Some port -> port | None -> Port.fresh () in + let name = match name with Some name -> name | None -> fresh_name () in + make_conf ~name ~address ~port ~users ~admin + + let run ?runner ?(path = Uses.path Constant.teztale_server) ?name ?address + ?port ?users ?admin () = + let conf = make ?name ?address ?port ?users ?admin () in + let* filenames = dump_conf ?runner conf in + let process = + Process.spawn ~name:conf.name ?runner path [filenames.conf_filename] + in + Lwt.return {process; filenames; conf} + + (** Return *) + let add_user {conf = {interface; admin; _}; _} user = + let user = + JSON.parse + ~origin:__LOC__ + (Printf.sprintf + {|{"login": "%s", "password": "%s"}|} + user.login + user.password) + in + let url = + Format.asprintf + "http://%s:%s@%s:%d/user" + admin.login + admin.password + interface.address + interface.port + in + Curl.put url user |> Runnable.run + |> Lwt.map (fun json -> + match JSON.(get "status" json |> as_string) with + | "OK" -> Ok () + | status -> + let msg = + Printf.sprintf "%s: teztale answered with: %s" __LOC__ status + in + Error (Failure msg) + | exception e -> Error e) +end + +module Archiver = struct + let fresh_name = fresh_name "teztale-archiver" + + type conf = {name : string; user : user; feed : interface list} + + type t = {process : Process.t; conf : conf} + + let run ?runner ?(path = Uses.path Constant.teztale_archiver) ?name ~node_port + user feed = + let name = match name with Some name -> name | None -> fresh_name () in + let node_endpoint = Format.asprintf "http://127.0.0.1:%d" node_port in + let conf = {name; user; feed} in + let args = + "--endpoint" :: node_endpoint + :: List.fold_left + (fun acc feed -> + "feed" + :: Printf.sprintf + "http://%s:%s@%s:%d" + user.login + user.password + feed.address + feed.port + :: acc) + [] + conf.feed + in + let process = Process.spawn ~name:conf.name ?runner path args in + Lwt.return {process; conf} +end diff --git a/tezt/lib_tezos/teztale.mli b/tezt/lib_tezos/teztale.mli new file mode 100644 index 000000000000..55b1df4e9291 --- /dev/null +++ b/tezt/lib_tezos/teztale.mli @@ -0,0 +1,51 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2024 Nomadic Labs, *) +(* *) +(*****************************************************************************) + +type user = {login : string; password : string} + +type interface = {address : string; port : int} + +module Server : sig + type conf = { + name : string; + interface : interface; + users : user list; + admin : user; + } + + type filenames = {conf_filename : string; db_filename : string} + + type t = {process : Process.t; filenames : filenames; conf : conf} + + val run : + ?runner:Runner.t -> + ?path:string -> + ?name:string -> + ?address:string -> + ?port:int -> + ?users:user list -> + ?admin:user -> + unit -> + t Lwt.t + + val add_user : t -> user -> (unit, exn) Result.t Lwt.t +end + +module Archiver : sig + type conf = {name : string; user : user; feed : interface list} + + type t = {process : Process.t; conf : conf} + + val run : + ?runner:Runner.t -> + ?path:string -> + ?name:string -> + node_port:int -> + user -> + interface list -> + t Lwt.t +end diff --git a/tezt/tests/cloud/dal.ml b/tezt/tests/cloud/dal.ml index ddb9ebcb7547..1482ca214d39 100644 --- a/tezt/tests/cloud/dal.ml +++ b/tezt/tests/cloud/dal.ml @@ -248,92 +248,31 @@ end module Teztale = struct type t = { - server_daemon : Process.t; - port : int; - mutable archivers : Process.t list; + server : Teztale.Server.t; + mutable archivers : Teztale.Archiver.t list; } - let make_configuration ~port = - let teztale_sqlite = - Format.asprintf - "sqlite3:%s/teztale.sqlite" - (Filename.get_temp_dir_name ()) - in - Format.asprintf - {| -{ - "db": "%s", - "interfaces": [ - { - "address": "127.0.0.1", - "port": %d - } - ], - "admins": [ - { - "login": "admin", - "password": "saucisse" - } - ], - "users": [ - { - "login": "user", - "password": "saucisse" - } - ], - "with_transaction": "FULL" -} -|} - teztale_sqlite - port + let fresh_user = + let i = ref (-1) in + fun () : Teztale.user -> + incr i ; + let login = "teztale-archiver-" ^ string_of_int !i in + {login; password = login} let run_server ?(path = Uses.(path (make ~tag:"codec" ~path:"./octez-teztale-server"))) agent = - let runner = Agent.runner agent in - let configuration_file = - Filename.get_temp_dir_name () // "teztale-config.json" - in - let port = Agent.next_available_port agent in - let configuration = make_configuration ~port in - let* () = - match runner with - | None -> - write_file configuration_file ~contents:configuration ; - Lwt.return_unit - | Some runner -> - let cmd = - Runner.Shell.( - redirect_stdout (cmd [] "echo" [configuration]) configuration_file) - in - let cmd, args = Runner.wrap_with_ssh runner cmd in - Process.spawn cmd args |> Process.check - in - let* path = Agent.copy agent ~source:path in - let server_daemon = - Process.spawn ~name:"teztale-server" ?runner path [configuration_file] - in - (* Wait a bit it starts. *) - let* () = Lwt_unix.sleep 0.5 in - Lwt.return {server_daemon; port; archivers = []} + let* server = Teztale.Server.run ~path agent () in + Lwt.return {server; archivers = []} - let run_archiver + let add_archiver ?(path = Uses.(path (make ~tag:"codec" ~path:"./octez-teztale-archiver"))) t agent ~node_port = - let runner = Agent.runner agent in - let node_endpoint = Format.asprintf "http://127.0.0.1:%d" node_port in - let teztale_endpoint = - Format.asprintf "http://user:saucisse@127.0.0.1:%d" t.port - in - let* path = Agent.copy agent ~source:path in - let archiver_daemon = - Process.spawn - ~name:"teztale-archiver" - ?runner - path - ["--endpoint"; node_endpoint; "feed"; teztale_endpoint] - in - t.archivers <- archiver_daemon :: t.archivers ; + let user = fresh_user () in + let feed : Teztale.interface list = [t.server.conf.interface] in + let* () = Lwt_result.get_exn (Teztale.Server.add_user t.server user) in + let* archiver = Teztale.Archiver.run agent ~path user feed ~node_port in + t.archivers <- archiver :: t.archivers ; Lwt.return_unit end @@ -1120,7 +1059,7 @@ let init_teztale agent node = if Cli.teztale then let* teztale = Teztale.run_server agent in let* () = - Teztale.run_archiver teztale agent ~node_port:(Node.rpc_port node) + Teztale.add_archiver teztale agent ~node_port:(Node.rpc_port node) in Lwt.return_some teztale else Lwt.return_none diff --git a/tezt/tests/cloud/tezos.ml b/tezt/tests/cloud/tezos.ml index e5819d5214e5..c36cfea158d7 100644 --- a/tezt/tests/cloud/tezos.ml +++ b/tezt/tests/cloud/tezos.ml @@ -210,3 +210,40 @@ module Baker = struct client end end + +module Teztale = struct + include Teztale + + module Server = struct + include Teztale.Server + + let run agent ?(path = Uses.path Constant.teztale_server) ?name ?address + ?port ?users ?admin () = + let runner = Agent.runner agent in + let address = + match address with + | Some address -> address + | None -> Agent.point agent |> Option.map fst + in + let port = + match port with + | Some port -> port + | None -> ( + match Agent.point agent |> Option.map snd with + | Some port -> port + | None -> Agent.next_available_port agent) + in + let* path = Agent.copy agent ~source:path in + Teztale.Server.run ?runner ~path ~port ?name ?address ?users ?admin () + end + + module Archiver = struct + include Teztale.Archiver + + let run agent ?(path = Uses.path Constant.teztale_archiver) ?name ~node_port + user feed = + let runner = Agent.runner agent in + let* path = Agent.copy agent ~source:path in + Teztale.Archiver.run ?runner ~path ?name ~node_port user feed + end +end -- GitLab From 64aa5ca84e054730a58a6a743e77fc16727b1ef5 Mon Sep 17 00:00:00 2001 From: Julien Sagot Date: Tue, 3 Sep 2024 17:42:13 +0200 Subject: [PATCH 2/3] Tezt/Teztale: add documentation --- tezt/lib_tezos/teztale.mli | 39 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tezt/lib_tezos/teztale.mli b/tezt/lib_tezos/teztale.mli index 55b1df4e9291..e9d1c8f34570 100644 --- a/tezt/lib_tezos/teztale.mli +++ b/tezt/lib_tezos/teztale.mli @@ -5,11 +5,14 @@ (* *) (*****************************************************************************) +(** A user is either an archiver or an admin *) type user = {login : string; password : string} type interface = {address : string; port : int} module Server : sig + + (** See parameters of {!run} function for details. *) type conf = { name : string; interface : interface; @@ -17,10 +20,27 @@ module Server : sig admin : user; } + (** Expose paths of the files involved in the test: + - Server configuration file + - SQLite database + *) type filenames = {conf_filename : string; db_filename : string} type t = {process : Process.t; filenames : filenames; conf : conf} + (** [run ?runner ?path ?name ?address ?port ?users ?admin ()] + + Spawn a teztale server with some given parameters: + - runner: runner used to spawn the process + - path: path of the teztale server executable + - name: name the the server used in logs + - address: where teztale server will be listening on (default is [127.0.0.1]) + - port: port associated with [address] + - users: list of archivers allowed to feed the server. You can add + users once the server is started using the {!add_user} function. + - admin: credential used for adming tasks such as adding an allowed archiver + default is [admin:password] + *) val run : ?runner:Runner.t -> ?path:string -> @@ -32,14 +52,33 @@ module Server : sig unit -> t Lwt.t + (** [add_user server user] + Add an archiver (its credentials) allowed to feed the database. + *) val add_user : t -> user -> (unit, exn) Result.t Lwt.t end module Archiver : sig + + (** See parameters of {!run} function for details. *) type conf = {name : string; user : user; feed : interface list} type t = {process : Process.t; conf : conf} + (** [run ?runner ?path ?name ~node_port user feed] + + Spawn a teztale archiver with some given parameters: + - runner: runner used to spawn the process + - path: path of the teztale server executable + - name: name the the server used in logs + - node_port: port used for the octez node RPCs + - user: login and password used by the archiver. Make sure that + login and password used have been allowed in the server. See + [?users] argument of {!Server.run} or {!Server.add_user}. + - feed: list of interfaces for teztale servers to feed. If the archiver + is feeding multiple servers, every server must accept login info defined + by [user] parameter. + *) val run : ?runner:Runner.t -> ?path:string -> -- GitLab From fa2254fc79f7ea146689240c3f117c11af45c511 Mon Sep 17 00:00:00 2001 From: Julien Sagot Date: Wed, 4 Sep 2024 17:54:25 +0200 Subject: [PATCH 3/3] Tezt/Teztale: wait_for_readiness: make sure that server is ready to communicate --- tezt/lib_tezos/teztale.ml | 20 +++++++++++++++++++- tezt/lib_tezos/teztale.mli | 5 +++-- tezt/tests/cloud/dal.ml | 3 +++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/tezt/lib_tezos/teztale.ml b/tezt/lib_tezos/teztale.ml index d8e513a3dbc4..85d678025d7a 100644 --- a/tezt/lib_tezos/teztale.ml +++ b/tezt/lib_tezos/teztale.ml @@ -52,13 +52,16 @@ module Server = struct Format.fprintf fmt {|{"address": "%s", "port": %d}|} address port in let contents = + (* Verbosity needs to be set to a minimum level of INFO + because wait_for_readiness can't be used with a lower verbosity *) Format.asprintf {|{ "db": "sqlite3:%s", "interfaces": [%a], "users": [%a], "admins": [%a], - "with_transaction": "FULL" + "with_transaction": "FULL", + "verbosity": "INFO" }|} db_filename pp_interface @@ -96,6 +99,21 @@ module Server = struct in Lwt.return {process; filenames; conf} + let wait_for_readiness t = + Log.info "Wait for %s to be ready" t.conf.name ; + let stdout = Process.stdout t.process in + let suffix = + Printf.sprintf + "Server listening at %s:%d." + t.conf.interface.address + t.conf.interface.port + in + let rec wait () = + let* line = Lwt_io.read_line stdout in + if String.ends_with ~suffix line then Lwt.return_unit else wait () + in + wait () + (** Return *) let add_user {conf = {interface; admin; _}; _} user = let user = diff --git a/tezt/lib_tezos/teztale.mli b/tezt/lib_tezos/teztale.mli index e9d1c8f34570..6f0582707188 100644 --- a/tezt/lib_tezos/teztale.mli +++ b/tezt/lib_tezos/teztale.mli @@ -11,7 +11,6 @@ type user = {login : string; password : string} type interface = {address : string; port : int} module Server : sig - (** See parameters of {!run} function for details. *) type conf = { name : string; @@ -52,6 +51,9 @@ module Server : sig unit -> t Lwt.t + (** Wait until teztale server is listening to its defined interface *) + val wait_for_readiness : t -> unit Lwt.t + (** [add_user server user] Add an archiver (its credentials) allowed to feed the database. *) @@ -59,7 +61,6 @@ module Server : sig end module Archiver : sig - (** See parameters of {!run} function for details. *) type conf = {name : string; user : user; feed : interface list} diff --git a/tezt/tests/cloud/dal.ml b/tezt/tests/cloud/dal.ml index 1482ca214d39..3582fe8af5c4 100644 --- a/tezt/tests/cloud/dal.ml +++ b/tezt/tests/cloud/dal.ml @@ -265,6 +265,8 @@ module Teztale = struct let* server = Teztale.Server.run ~path agent () in Lwt.return {server; archivers = []} + let wait_server t = Teztale.Server.wait_for_readiness t.server + let add_archiver ?(path = Uses.(path (make ~tag:"codec" ~path:"./octez-teztale-archiver"))) t agent ~node_port = @@ -1058,6 +1060,7 @@ let add_etherlink_source cloud agent ~job_name ?dal_node node sc_rollup_node let init_teztale agent node = if Cli.teztale then let* teztale = Teztale.run_server agent in + let* () = Teztale.wait_server teztale in let* () = Teztale.add_archiver teztale agent ~node_port:(Node.rpc_port node) in -- GitLab