From a960cad98c44bb7afb8027c28711cd9606a5d472 Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Mon, 7 Oct 2024 14:18:15 +0200 Subject: [PATCH 1/7] Agnostic_baker: cleannig and refactoring --- src/bin_agnostic_baker/main_agnostic_baker.ml | 98 +++++++++++++------ 1 file changed, 67 insertions(+), 31 deletions(-) diff --git a/src/bin_agnostic_baker/main_agnostic_baker.ml b/src/bin_agnostic_baker/main_agnostic_baker.ml index 0a9658d89c23..4c37e6226082 100644 --- a/src/bin_agnostic_baker/main_agnostic_baker.ml +++ b/src/bin_agnostic_baker/main_agnostic_baker.ml @@ -1,7 +1,7 @@ (*****************************************************************************) (* *) (* SPDX-License-Identifier: MIT *) -(* Copyright (c) 2018-2024 Nomadic Labs, *) +(* Copyright (c) 2024 Nomadic Labs, *) (* *) (*****************************************************************************) @@ -72,13 +72,30 @@ module Events = struct ("proto", Protocol_hash.encoding) ~pp1:Protocol_hash.pp_short + let stopping_baker = + declare_1 + ~section + ~level:Notice + ~name:"stopping_baker" + ~msg:"stopping baker for protocol {proto}" + ("proto", Protocol_hash.encoding) + ~pp1:Protocol_hash.pp_short + let starting_daemon = declare_0 ~section ~alternative_color ~level:Notice ~name:"starting_daemon" - ~msg:"starting agnostic daemon" + ~msg:"agnostic baker started" + () + + let stopping_daemon = + declare_0 + ~section + ~level:Notice + ~name:"stopping_daemon" + ~msg:"stopping agnostic daemon" () let protocol_encountered = @@ -178,7 +195,11 @@ module Parameters = struct end module Baker = struct - type t = Lwt_process.process_none option + type t = { + protocol_hash : Protocol_hash.t; + binary_path : string; + process : Lwt_process.process_none; + } let baker_path ?(user_path = "./") proto_hash = let short_name = @@ -186,9 +207,13 @@ module Baker = struct in Format.sprintf "%soctez-baker-%s" user_path short_name - let _shutdown p = p#terminate + let shutdown {process; protocol_hash; _} = + let open Lwt_syntax in + let* () = Events.(emit stopping_baker) protocol_hash in + process#terminate ; + Lwt.return_unit - let spawn_baker proto_hash ~binaries_directory ~baker_args = + let spawn_baker protocol_hash ~binaries_directory ~baker_args = let open Lwt_result_syntax in let args_as_string = Format.asprintf @@ -198,18 +223,24 @@ module Baker = struct Format.pp_print_string) baker_args in - let*! () = Events.(emit starting_baker) (proto_hash, args_as_string) in - let path = baker_path ?user_path:binaries_directory proto_hash in - let baker_args = path :: baker_args in + let*! () = Events.(emit starting_baker) (protocol_hash, args_as_string) in + let binary_path = baker_path ?user_path:binaries_directory protocol_hash in + let baker_args = binary_path :: baker_args in let baker_args = Array.of_list baker_args in - let p = + let process = Lwt_process.open_process_none ~stdout:`Keep ~stderr:`Keep - (path, baker_args) + (binary_path, baker_args) + in + let*! () = Events.(emit baker_running) protocol_hash in + let t = {protocol_hash; binary_path; process} in + let _ccid = + Lwt_exit.register_clean_up_callback ~loc:__LOC__ (fun _ -> + let*! () = shutdown t in + Lwt.return_unit) in - let*! () = Events.(emit baker_running) proto_hash in - return p + return t end module RPC = struct @@ -265,15 +296,6 @@ module RPC = struct in let uri = Format.sprintf "%s/chains/main/blocks/head/metadata" node_addr in call_and_wrap_rpc ~node_addr ~uri ~f -end - -module Daemon = struct - type state = { - binaries_directory : string option; - node_endpoint : string; - baker_args : string list; - mutable current_baker : Baker.t; - } let get_current_proposal ~node_addr = let open Lwt_result_syntax in @@ -288,7 +310,7 @@ module Daemon = struct "%s/chains/main/blocks/head/votes/current_proposal" node_addr in - RPC.call_and_wrap_rpc ~node_addr ~uri ~f + call_and_wrap_rpc ~node_addr ~uri ~f let get_current_period ~node_addr = let open Lwt_result_syntax in @@ -331,7 +353,16 @@ module Daemon = struct let uri = Format.sprintf "%s/chains/main/blocks/head/votes/current_period" node_addr in - RPC.call_and_wrap_rpc ~node_addr ~uri ~f + call_and_wrap_rpc ~node_addr ~uri ~f +end + +module Daemon = struct + type state = { + binaries_directory : string option; + node_endpoint : string; + baker_args : string list; + mutable current_baker : Baker.t option; + } let monitor_heads ~node_addr = let open Lwt_result_syntax in @@ -363,7 +394,7 @@ module Daemon = struct let*! v = Lwt_stream.get head_stream in match v with | Some _tick -> - let* period_kind, remaining = get_current_period ~node_addr in + let* period_kind, remaining = RPC.get_current_period ~node_addr in let*! () = Events.(emit period_status) (period_kind, remaining) in loop () | None -> return_unit @@ -433,8 +464,13 @@ module Daemon = struct let open Lwt_result_syntax in let node_addr = state.node_endpoint in let*! () = Events.(emit starting_daemon) () in + let _ccid = + Lwt_exit.register_clean_up_callback ~loc:__LOC__ (fun _ -> + let*! () = Events.(emit stopping_daemon) () in + Lwt.return_unit) + in let* () = may_start_initial_baker state in - let* _protocol_proposal = get_current_proposal ~node_addr in + let* _protocol_proposal = RPC.get_current_proposal ~node_addr in let* head_stream = monitor_heads ~node_addr in (* Monitoring voting periods through heads monitoring to avoid missing UAUs. *) @@ -553,15 +589,15 @@ let run () = let () = let open Lwt_result_syntax in - let main_promise () = run () in + let main_promise = + Lwt.catch run (function + | Failure msg -> failwith "%s" msg + | exn -> failwith "%s" (Printexc.to_string exn)) + in Stdlib.exit (Lwt_main.run (let*! retcode = - let*! r = - Lwt.catch main_promise (function - | Failure msg -> failwith "%s" msg - | exn -> failwith "%s" (Printexc.to_string exn)) - in + let*! r = Lwt_exit.wrap_and_exit main_promise in match r with | Ok () -> Lwt.return 0 | Error errs -> -- GitLab From 28885db0d8be1cf5801218516209d6143ae61588 Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Wed, 9 Oct 2024 11:37:05 +0200 Subject: [PATCH 2/7] Agnostic_baker: parameters as standalone module --- src/bin_agnostic_baker/main_agnostic_baker.ml | 75 +------------- src/bin_agnostic_baker/parameters.ml | 98 +++++++++++++++++++ 2 files changed, 102 insertions(+), 71 deletions(-) create mode 100644 src/bin_agnostic_baker/parameters.ml diff --git a/src/bin_agnostic_baker/main_agnostic_baker.ml b/src/bin_agnostic_baker/main_agnostic_baker.ml index 4c37e6226082..5a2d13b4e5c4 100644 --- a/src/bin_agnostic_baker/main_agnostic_baker.ml +++ b/src/bin_agnostic_baker/main_agnostic_baker.ml @@ -104,7 +104,8 @@ module Events = struct ~level:Notice ~name:"protocol_encountered" ~msg:"the {status} protocol {proto_hash} was encountered" - ("status", string) + ("status", Parameters.status_encoding) + ~pp1:Parameters.pp_status ("proto_hash", Protocol_hash.encoding) ~pp2:Protocol_hash.pp_short @@ -129,71 +130,6 @@ module Events = struct ("remaining", int31) end -module Parameters = struct - let default_node_addr = "http://127.0.0.1:8732" - - let default_daily_logs_path = Some "octez-agnostic-baker" - - let log_config ~base_dir = - let base_dir : string = - match base_dir with - | Some p -> p - | None -> Tezos_client_base_unix.Client_config.Cfg_file.default.base_dir - in - let daily_logs_path = - default_daily_logs_path - |> Option.map Filename.Infix.(fun logdir -> base_dir // "logs" // logdir) - in - Tezos_base_unix.Internal_event_unix.make_with_defaults - ?enable_default_daily_logs_at:daily_logs_path - ~log_cfg:Tezos_base_unix.Logs_simple_config.default_cfg - () - - type status = Active | Frozen - - let pp_status fmt status = - Format.fprintf - fmt - "%s" - (match status with Active -> "active" | Frozen -> "frozen") - - (* From Manifest/Product_octez/Protocol*) - let protocol_info = function - | ( "ProtoGenesisGenesisGenesisGenesisGenesisGenesk612im" - | "Ps9mPmXaRzmzk35gbAYNCAw6UXdE2qoABTHbN2oEEc1qM7CwT9P" - | "PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY" - | "PsYLVpVvgbLhAhoqAkMFUo6gudkJ9weNXhUYCiLDzcUpFpkk8Wt" - | "PsddFKi32cMJ2qPjf43Qv5GDWLDPZb3T3bF6fLKiF5HtvHNU7aP" - | "Pt24m4xiPbLDhVgVfABUjirbmda3yohdN82Sp9FeuAXJ4eV9otd" - | "PsBABY5HQTSkA4297zNHfsZNKtxULfL18y95qb3m53QJiXGmrbU" - | "PsBabyM1eUXZseaJdmXFApDSBqj8YBfwELoxZHHW77EMcAbbwAS" - | "PsCARTHAGazKbHtnKfLzQg3kms52kSRpgnDY982a9oYsSXRLQEb" - | "PsDELPH1Kxsxt8f9eWbxQeRxkjfbxoqM52jvs5Y5fBxWWh4ifpo" - | "PtEdoTezd3RHSC31mpxxo1npxFjoWWcFgQtxapi51Z8TLu6v6Uq" - | "PtEdo2ZkT9oKpimTah6x2embF25oss54njMuPzkJTEi5RqfdZFA" - | "PsFLorenaUUuikDWvMDr6fGBRG8kt3e3D3fHoXK1j1BFRxeSH4i" - | "PtGRANADsDU8R9daYKAgWnQYAJ64omN1o3KMGVCykShA97vQbvV" - | "PtHangz2aRngywmSRGGvrcTyMbbdpWdpFKuS4uMWxg2RaH9i1qx" - | "Psithaca2MLRFYargivpo7YvUr7wUDqyxrdhC5CQq78mRvimz6A" - | "PtJakart2xVj7pYXJBXrqHgd82rdkLey5ZeeGwDgPp9rhQUbSqY" - | "PtKathmankSpLLDALzWw7CGD2j2MtyveTwboEYokqUCP4a1LxMg" - | "PtLimaPtLMwfNinJi9rCfDPWea8dFgTZ1MeJ9f1m2SRic6ayiwW" - | "PtMumbai2TmsJHNGRkD8v8YDbtao7BLUC3wjASn1inAKLFCjaH1" - | "PtNairobiyssHuh87hEhfVBGCVrK3WnS8Z2FT4ymB5tAa4r1nQf" - | "ProxfordYmVfjWnRcgjWH36fW6PArwqykTFzotUxRs6gmTcZDuH" - | "PtParisBxoLz5gzMmn3d9WBQNoPSZakgnkMC2VNuQ3KXfUtUQeZ" ) as full_hash -> - (String.sub full_hash 0 8, Frozen) - | ( "PsParisCZo7KAh1Z1smVd9ZMZ1HHn5gkzbM94V3PLCpknFWhUAi" - | "PsquebeCaYyvBEESCaXL8B8Tn8BcEhps2Zke1xMVtyr7X4qMfxT" ) as full_hash -> - (String.sub full_hash 0 8, Active) - | "ProtoALphaALphaALphaALphaALphaALphaALphaALphaDdp3zK" -> ("alpha", Active) - | _ -> (*We assume that unmatched protocols are beta ones*) ("beta", Active) - - let protocol_short_hash h = fst (protocol_info h) - - let protocol_status h = snd (protocol_info h) -end - module Baker = struct type t = { protocol_hash : Protocol_hash.t; @@ -421,9 +357,7 @@ module Daemon = struct | None -> Lwt.return_unit | Some h -> if not (Protocol_hash.equal h protocol_hash) then - Events.(emit protocol_encountered) - ( Format.asprintf "%a" Parameters.pp_status proto_status, - protocol_hash ) + Events.(emit protocol_encountered) (proto_status, protocol_hash) else Lwt.return_unit in match proto_status with @@ -443,8 +377,7 @@ module Daemon = struct | None -> let*! () = Events.(emit protocol_encountered) - ( Format.asprintf "%a" Parameters.pp_status proto_status, - protocol_hash ) + (proto_status, protocol_hash) in let*! () = Events.(emit waiting_for_active_protocol) () in monitor_heads ~node_addr:state.node_endpoint diff --git a/src/bin_agnostic_baker/parameters.ml b/src/bin_agnostic_baker/parameters.ml new file mode 100644 index 000000000000..967790a22e87 --- /dev/null +++ b/src/bin_agnostic_baker/parameters.ml @@ -0,0 +1,98 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2024 Nomadic Labs, *) +(* *) +(*****************************************************************************) + +let default_node_addr = "http://127.0.0.1:8732" + +let default_daily_logs_path = Some "octez-agnostic-baker" + +let log_config ~base_dir = + let base_dir : string = + match base_dir with + | Some p -> p + | None -> Tezos_client_base_unix.Client_config.Cfg_file.default.base_dir + in + let daily_logs_path = + default_daily_logs_path + |> Option.map Filename.Infix.(fun logdir -> base_dir // "logs" // logdir) + in + Tezos_base_unix.Internal_event_unix.make_with_defaults + ?enable_default_daily_logs_at:daily_logs_path + ~log_cfg:Tezos_base_unix.Logs_simple_config.default_cfg + () + +type status = Active | Frozen + +let pp_status fmt status = + Format.fprintf + fmt + "%s" + (match status with Active -> "active" | Frozen -> "frozen") + +let status_encoding = + let open Data_encoding in + def + "status" + ~title:"protocol status" + ~description:"Status of a protocol" + (union + ~tag_size:`Uint8 + [ + case + ~title:"active" + ~description: + "Active protocols are currently used on a network, and thus, they \ + have dedicated delegate binaries." + (Tag 0) + (constant "active") + (function Active -> Some () | _ -> None) + (fun () -> Active); + case + ~title:"frozen" + ~description: + "Frozen protocols are currently unused on any network, and thus, \ + they do nothave dedicated delegate binaries." + (Tag 1) + (constant "frozen") + (function Frozen -> Some () | _ -> None) + (fun () -> Frozen); + ]) + +(* From Manifest/Product_octez/Protocol*) +let protocol_info = function + | ( "ProtoGenesisGenesisGenesisGenesisGenesisGenesk612im" + | "Ps9mPmXaRzmzk35gbAYNCAw6UXdE2qoABTHbN2oEEc1qM7CwT9P" + | "PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY" + | "PsYLVpVvgbLhAhoqAkMFUo6gudkJ9weNXhUYCiLDzcUpFpkk8Wt" + | "PsddFKi32cMJ2qPjf43Qv5GDWLDPZb3T3bF6fLKiF5HtvHNU7aP" + | "Pt24m4xiPbLDhVgVfABUjirbmda3yohdN82Sp9FeuAXJ4eV9otd" + | "PsBABY5HQTSkA4297zNHfsZNKtxULfL18y95qb3m53QJiXGmrbU" + | "PsBabyM1eUXZseaJdmXFApDSBqj8YBfwELoxZHHW77EMcAbbwAS" + | "PsCARTHAGazKbHtnKfLzQg3kms52kSRpgnDY982a9oYsSXRLQEb" + | "PsDELPH1Kxsxt8f9eWbxQeRxkjfbxoqM52jvs5Y5fBxWWh4ifpo" + | "PtEdoTezd3RHSC31mpxxo1npxFjoWWcFgQtxapi51Z8TLu6v6Uq" + | "PtEdo2ZkT9oKpimTah6x2embF25oss54njMuPzkJTEi5RqfdZFA" + | "PsFLorenaUUuikDWvMDr6fGBRG8kt3e3D3fHoXK1j1BFRxeSH4i" + | "PtGRANADsDU8R9daYKAgWnQYAJ64omN1o3KMGVCykShA97vQbvV" + | "PtHangz2aRngywmSRGGvrcTyMbbdpWdpFKuS4uMWxg2RaH9i1qx" + | "Psithaca2MLRFYargivpo7YvUr7wUDqyxrdhC5CQq78mRvimz6A" + | "PtJakart2xVj7pYXJBXrqHgd82rdkLey5ZeeGwDgPp9rhQUbSqY" + | "PtKathmankSpLLDALzWw7CGD2j2MtyveTwboEYokqUCP4a1LxMg" + | "PtLimaPtLMwfNinJi9rCfDPWea8dFgTZ1MeJ9f1m2SRic6ayiwW" + | "PtMumbai2TmsJHNGRkD8v8YDbtao7BLUC3wjASn1inAKLFCjaH1" + | "PtNairobiyssHuh87hEhfVBGCVrK3WnS8Z2FT4ymB5tAa4r1nQf" + | "ProxfordYmVfjWnRcgjWH36fW6PArwqykTFzotUxRs6gmTcZDuH" + | "PtParisBxoLz5gzMmn3d9WBQNoPSZakgnkMC2VNuQ3KXfUtUQeZ" ) as full_hash -> + (String.sub full_hash 0 8, Frozen) + | ( "PsParisCZo7KAh1Z1smVd9ZMZ1HHn5gkzbM94V3PLCpknFWhUAi" + | "PsquebeCaYyvBEESCaXL8B8Tn8BcEhps2Zke1xMVtyr7X4qMfxT" ) as full_hash -> + (String.sub full_hash 0 8, Active) + | "ProtoALphaALphaALphaALphaALphaALphaALphaALphaDdp3zK" -> ("alpha", Active) + | _ -> (*We assume that unmatched protocols are beta ones*) ("beta", Active) + +let protocol_short_hash h = fst (protocol_info h) + +let protocol_status h = snd (protocol_info h) -- GitLab From e8091eee266488bc587cb65bb934b8b04a659d79 Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Tue, 8 Oct 2024 17:12:55 +0200 Subject: [PATCH 3/7] Agnostic_baker: Quebec as active protocol --- src/bin_agnostic_baker/parameters.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin_agnostic_baker/parameters.ml b/src/bin_agnostic_baker/parameters.ml index 967790a22e87..d683a7e55ce2 100644 --- a/src/bin_agnostic_baker/parameters.ml +++ b/src/bin_agnostic_baker/parameters.ml @@ -88,7 +88,7 @@ let protocol_info = function | "PtParisBxoLz5gzMmn3d9WBQNoPSZakgnkMC2VNuQ3KXfUtUQeZ" ) as full_hash -> (String.sub full_hash 0 8, Frozen) | ( "PsParisCZo7KAh1Z1smVd9ZMZ1HHn5gkzbM94V3PLCpknFWhUAi" - | "PsquebeCaYyvBEESCaXL8B8Tn8BcEhps2Zke1xMVtyr7X4qMfxT" ) as full_hash -> + | "PsQuebecnLByd3JwTiGadoG4nGWi3HYiLXUjkibeFV8dCFeVMUg" ) as full_hash -> (String.sub full_hash 0 8, Active) | "ProtoALphaALphaALphaALphaALphaALphaALphaALphaDdp3zK" -> ("alpha", Active) | _ -> (*We assume that unmatched protocols are beta ones*) ("beta", Active) -- GitLab From 4624bb41a8bac90fe0aba61882573cb430b0e210 Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Wed, 9 Oct 2024 16:15:16 +0200 Subject: [PATCH 4/7] Agnostic_baker: hot swap bakers on protocol change --- src/bin_agnostic_baker/main_agnostic_baker.ml | 53 ++++++++++++++++--- 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/src/bin_agnostic_baker/main_agnostic_baker.ml b/src/bin_agnostic_baker/main_agnostic_baker.ml index 5a2d13b4e5c4..697575c2dc84 100644 --- a/src/bin_agnostic_baker/main_agnostic_baker.ml +++ b/src/bin_agnostic_baker/main_agnostic_baker.ml @@ -135,6 +135,7 @@ module Baker = struct protocol_hash : Protocol_hash.t; binary_path : string; process : Lwt_process.process_none; + ccid : Lwt_exit.clean_up_callback_id; } let baker_path ?(user_path = "./") proto_hash = @@ -143,7 +144,7 @@ module Baker = struct in Format.sprintf "%soctez-baker-%s" user_path short_name - let shutdown {process; protocol_hash; _} = + let shutdown protocol_hash process = let open Lwt_syntax in let* () = Events.(emit stopping_baker) protocol_hash in process#terminate ; @@ -170,13 +171,12 @@ module Baker = struct (binary_path, baker_args) in let*! () = Events.(emit baker_running) protocol_hash in - let t = {protocol_hash; binary_path; process} in - let _ccid = + let ccid = Lwt_exit.register_clean_up_callback ~loc:__LOC__ (fun _ -> - let*! () = shutdown t in + let*! () = shutdown protocol_hash process in Lwt.return_unit) in - return t + return {protocol_hash; binary_path; process; ccid} end module RPC = struct @@ -324,14 +324,53 @@ module Daemon = struct ignore (loop () : unit Lwt.t) ; return stream - let monitor_voting_periods ~node_addr head_stream = + let hot_swap_baker ~state ~next_protocol_hash = + let open Lwt_result_syntax in + let next_proto_status = + Parameters.protocol_status (Protocol_hash.to_b58check next_protocol_hash) + in + let*! () = + Events.(emit protocol_encountered) (next_proto_status, next_protocol_hash) + in + let*! () = + match state.current_baker with + | None -> Lwt.return_unit (* Could be assert false*) + | Some b -> + let*! () = Baker.shutdown b.protocol_hash b.process in + let () = Lwt_exit.unregister_clean_up_callback b.ccid in + state.current_baker <- None ; + Lwt.return_unit + in + let* new_baker = + Baker.spawn_baker + next_protocol_hash + ~binaries_directory:state.binaries_directory + ~baker_args:state.baker_args + in + state.current_baker <- Some new_baker ; + return_unit + + let monitor_voting_periods ~state head_stream = let open Lwt_result_syntax in + let node_addr = state.node_endpoint in let rec loop () = let*! v = Lwt_stream.get head_stream in match v with | Some _tick -> let* period_kind, remaining = RPC.get_current_period ~node_addr in let*! () = Events.(emit period_status) (period_kind, remaining) in + let* next_protocol_hash = RPC.get_next_protocol_hash ~node_addr in + let current_protocol_hash = + match state.current_baker with + | None -> assert false + | Some v -> v.protocol_hash + in + let* () = + if + not (Protocol_hash.equal current_protocol_hash next_protocol_hash) + then hot_swap_baker ~state ~next_protocol_hash + else return_unit + in loop () | None -> return_unit in @@ -407,7 +446,7 @@ module Daemon = struct let* head_stream = monitor_heads ~node_addr in (* Monitoring voting periods through heads monitoring to avoid missing UAUs. *) - let* () = monitor_voting_periods ~node_addr head_stream in + let* () = monitor_voting_periods ~state head_stream in return_unit end -- GitLab From 50b59ac445fdff2f640c6a031d883c1e39b85894 Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Tue, 8 Oct 2024 16:17:16 +0200 Subject: [PATCH 5/7] Tezt: introduce Agnostic_baker wrapper --- tezt/lib_tezos/agnostic_baker.ml | 174 ++++++++++++++++++++++++++++ tezt/lib_tezos/agnostic_baker.mli | 185 ++++++++++++++++++++++++++++++ tezt/lib_tezos/constant.ml | 3 + 3 files changed, 362 insertions(+) create mode 100644 tezt/lib_tezos/agnostic_baker.ml create mode 100644 tezt/lib_tezos/agnostic_baker.mli diff --git a/tezt/lib_tezos/agnostic_baker.ml b/tezt/lib_tezos/agnostic_baker.ml new file mode 100644 index 000000000000..1c11e49f2efa --- /dev/null +++ b/tezt/lib_tezos/agnostic_baker.ml @@ -0,0 +1,174 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2024 Nomadic Labs *) +(* *) +(*****************************************************************************) + +type liquidity_baking_vote = Off | On | Pass + +let liquidity_baking_vote_to_string = function + | Off -> "off" + | On -> "on" + | Pass -> "pass" + +module Parameters = struct + type persistent_state = { + delegates : string list; + runner : Runner.t option; + base_dir : string; + node_data_dir : string; + node_rpc_endpoint : Endpoint.t; + mutable pending_ready : unit option Lwt.u list; + remote_mode : bool; + liquidity_baking_toggle_vote : liquidity_baking_vote option; + } + + type session_state = {mutable ready : bool} + + let base_default_name = "agnostic-baker" + + let default_colors = Log.Color.[|FG.green|] +end + +open Parameters +include Daemon.Make (Parameters) + +let trigger_ready agnostic_baker value = + let pending = agnostic_baker.persistent_state.pending_ready in + agnostic_baker.persistent_state.pending_ready <- [] ; + List.iter (fun pending -> Lwt.wakeup_later pending value) pending + +let set_ready agnostic_baker = + (match agnostic_baker.status with + | Not_running -> () + | Running status -> status.session_state.ready <- true) ; + trigger_ready agnostic_baker (Some ()) + +let create_from_uris ?runner ?(path = Uses.path Constant.octez_agnostic_baker) + ?name ?color ?event_pipe ?(delegates = []) ?(remote_mode = false) + ?(liquidity_baking_toggle_vote = Some Pass) ~base_dir ~node_data_dir + ~node_rpc_endpoint () = + let agnostic_baker = + create + ~path + ?name + ?color + ?event_pipe + ?runner + { + delegates; + runner; + base_dir; + node_data_dir; + node_rpc_endpoint; + pending_ready = []; + remote_mode; + liquidity_baking_toggle_vote; + } + in + agnostic_baker + +let handle_event node ({name; _} : event) = + match name with "starting_daemon.v0" -> set_ready node | _ -> () + +let create ?runner ?path ?name ?color ?event_pipe ?(delegates = []) + ?(remote_mode = false) ?(liquidity_baking_toggle_vote = Some Pass) node + client = + let agnostic_baker = + create_from_uris + ?runner + ?path + ?name + ?color + ?event_pipe + ~delegates + ~remote_mode + ~liquidity_baking_toggle_vote + ~base_dir:(Client.base_dir client) + ~node_data_dir:(Node.data_dir node) + ~node_rpc_endpoint:(Node.as_rpc_endpoint node) + () + in + on_event agnostic_baker (handle_event agnostic_baker) ; + agnostic_baker + +let run ?event_level ?event_sections_levels (agnostic_baker : t) = + (match agnostic_baker.status with + | Not_running -> () + | Running _ -> + Test.fail "agnostic_baker %s is already running" agnostic_baker.name) ; + let delegates = agnostic_baker.persistent_state.delegates in + let runner = agnostic_baker.persistent_state.runner in + let node_data_dir = agnostic_baker.persistent_state.node_data_dir in + let base_dir = agnostic_baker.persistent_state.base_dir in + let node_addr = + Endpoint.as_string agnostic_baker.persistent_state.node_rpc_endpoint + in + let run_args = + if agnostic_baker.persistent_state.remote_mode then ["remotely"] + else ["with"; "local"; "node"; node_data_dir] + in + let liquidity_baking_toggle_vote = + Cli_arg.optional_arg + "liquidity-baking-toggle-vote" + liquidity_baking_vote_to_string + agnostic_baker.persistent_state.liquidity_baking_toggle_vote + in + let arguments = + ["--"; "--endpoint"; node_addr; "--base-dir"; base_dir; "run"] + @ run_args @ delegates @ liquidity_baking_toggle_vote + in + + let on_terminate _ = + (* Cancel all [Ready] event listeners. *) + trigger_ready agnostic_baker None ; + unit + in + run + ?event_level + ?event_sections_levels + agnostic_baker + {ready = false} + arguments + ~on_terminate + ?runner + +let check_event ?where agnostic_baker name promise = + let* result = promise in + match result with + | None -> + raise + (Terminated_before_event + {daemon = agnostic_baker.name; event = name; where}) + | Some x -> return x + +let wait_for_ready agnostic_baker = + match agnostic_baker.status with + | Running {session_state = {ready = true; _}; _} -> unit + | Not_running | Running {session_state = {ready = false; _}; _} -> + let promise, resolver = Lwt.task () in + agnostic_baker.persistent_state.pending_ready <- + resolver :: agnostic_baker.persistent_state.pending_ready ; + check_event agnostic_baker "agnostic baker started" promise + +let init ?runner ?(path = Uses.path Constant.octez_agnostic_baker) ?name ?color + ?event_level ?event_pipe ?event_sections_levels ?(delegates = []) + ?remote_mode ?liquidity_baking_toggle_vote node client = + let* () = Node.wait_for_ready node in + let agnostic_baker = + create + ?runner + ~path + ?name + ?color + ?event_pipe + ?remote_mode + ?liquidity_baking_toggle_vote + ~delegates + node + client + in + let* () = run ?event_level ?event_sections_levels agnostic_baker in + let* () = wait_for_ready agnostic_baker in + return agnostic_baker diff --git a/tezt/lib_tezos/agnostic_baker.mli b/tezt/lib_tezos/agnostic_baker.mli new file mode 100644 index 000000000000..6f037ba170b0 --- /dev/null +++ b/tezt/lib_tezos/agnostic_baker.mli @@ -0,0 +1,185 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2024 Nomadic Labs *) +(* *) +(*****************************************************************************) + +(** An agnostic baker instance *) +type t + +(** See [Daemon.Make.name] *) +val name : t -> string + +(** Send SIGTERM and wait for the process to terminate. + + Default [timeout] is 30 seconds, after which SIGKILL is sent. *) +val terminate : ?timeout:float -> t -> unit Lwt.t + +(** Send SIGKILL and wait for the process to terminate. *) +val kill : t -> unit Lwt.t + +(** Send SIGSTOP to the process. *) +val stop : t -> unit Lwt.t + +(** Send SIGCONT to the process. *) +val continue : t -> unit Lwt.t + +(** See [Daemon.Make.log_events]. *) +val log_events : ?max_length:int -> t -> unit + +(** See [Daemon.Make.wait_for]. *) +val wait_for : ?where:string -> t -> string -> (JSON.t -> 'a option) -> 'a Lwt.t + +(** Wait until the agnostic baker is ready. + + More precisely, wait until a "Agnostic Baker started" event occurs. + If this event alreay happened, return immediately. + *) +val wait_for_ready : t -> unit Lwt.t + +(** Spawn [octez-agnostic-baker run]. + + The resulting promise is fulfilled as soon as the agnostic baker has been + spawned. It continues running in the background. *) +val run : + ?event_level:Daemon.Level.default_level -> + ?event_sections_levels:(string * Daemon.Level.level) list -> + t -> + unit Lwt.t + +(** Liquidity baking vote values. *) +type liquidity_baking_vote = Off | On | Pass + +(** Returns the string representation of a [liquidity_baking_vote]. *) +val liquidity_baking_vote_to_string : liquidity_baking_vote -> string + +(** Create a agnostic baker. + + This function just creates a value of type [t], it does not call {!val:run}. + + [path] provides the path to the agnostic baker binary, the default being the one + derived from the [protocol]. + + The standard output and standard error output of the agnostic baker will + be logged with prefix [name] and color [color]. + + Default [event_pipe] is a temporary file whose name is derived + from [name]. It will be created as a named pipe so that agnostic baker + events can be received. + + The [Node.t] parameter is the node used by the agnostic baker. The agnostic baker + is configured to use the node's data dir. + + The [Client.t] parameter is the client used by the agnostic baker. The agnostic baker + is configured to use the client's base directory. + + If [runner] is specified, the agnostic baker will be spawned on this + runner using SSH. + + [delegates] is a list of account aliases (see {!val:Account.key.alias}), e.g., + bootstrap accounts (see {!val:Constant.bootstrap_keys}), delegated to this + agnostic baker. This defaults to the empty list, which is a shortcut for "every known + account". + + [liquidity_baking_toggle_vote] is passed to the agnostic baker + daemon through the flags [--liquidity-baking-toggle-vote]. If + [--liquidity-baking-toggle-vote] is [None], then + [--liquidity-baking-toggle-vote] is not passed. If it is [Some x] + then [--liquidity-baking-toggle-vote x] is passed. The default + value is [Some Pass]. + + If [remote_mode] is specified, the agnostic baker will run in RPC-only mode. + *) +val create : + ?runner:Runner.t -> + ?path:string -> + ?name:string -> + ?color:Log.Color.t -> + ?event_pipe:string -> + ?delegates:string list -> + ?remote_mode:bool -> + ?liquidity_baking_toggle_vote:liquidity_baking_vote option -> + Node.t -> + Client.t -> + t + +(** Similar to {!create}, but nodes RPCs addresses, wallet base directory and L1 + data directory are directly provided instead of a {!Client.t}, a {!Node.t} + and optionally a {!Dal_node.t}. + + The [node_data_dir] parameter provides the (local) node's data directory + used for baking. + + The [node_rpc_endpoint] parameter provides the (local) node's RPC server + endpoint to which the agnostic baker will connect to. + + The [base_dir] parameter contains needed information about the wallets used + by the agnostic baker. + + *) +val create_from_uris : + ?runner:Runner.t -> + ?path:string -> + ?name:string -> + ?color:Log.Color.t -> + ?event_pipe:string -> + ?delegates:string list -> + ?remote_mode:bool -> + ?liquidity_baking_toggle_vote:liquidity_baking_vote option -> + base_dir:string -> + node_data_dir:string -> + node_rpc_endpoint:Endpoint.t -> + unit -> + t + +(** Initialize an agnositc baker. + + This creates agnostic baker, waits for it to be ready, and then + returns it. + + As the agnostic baker usually relies on a node, we first wait for + the node to be ready and then, run the agnostic baker. + + The path to the agnostic baker binary is chosen from the + [protocol]. + + The standard output and standard error output of the agnostic + baker will be logged with prefix [name] and color [color]. + + Default [event_pipe] is a temporary file whose name is derived + from [name]. It will be created as a named pipe so that agnostic + baker events can be received. + + The [Node.t] parameter is the node used by the agnostic baker. The + agnostic baker is configured to use the node's data dir. + + The [Client.t] parameter is the client used by the agnostic + baker. The agnostic baker is configured to use the client's base + directory. + + If [runner] is specified, the agnostic baker will be spawned on + this runner using SSH. + + [delegates] is a list of account aliases (see + {!val:Account.key.alias}), e.g., bootstrap accounts (see + {!val:Constant.bootstrap_keys}), delegated to this agnostic + baker. This defaults to the empty list, which is a shortcut for + "every known account". + + If [remote_mode] is specified, the agnostic baker will run in + RPC-only mode. *) +val init : + ?runner:Runner.t -> + ?path:string -> + ?name:string -> + ?color:Log.Color.t -> + ?event_level:Daemon.Level.default_level -> + ?event_pipe:string -> + ?event_sections_levels:(string * Daemon.Level.level) list -> + ?delegates:string list -> + ?remote_mode:bool -> + ?liquidity_baking_toggle_vote:liquidity_baking_vote option -> + Node.t -> + Client.t -> + t Lwt.t diff --git a/tezt/lib_tezos/constant.ml b/tezt/lib_tezos/constant.ml index 1680e93efc2f..7910e5009ab6 100644 --- a/tezt/lib_tezos/constant.ml +++ b/tezt/lib_tezos/constant.ml @@ -135,6 +135,9 @@ module WASM = struct Uses.make ~tag:"tx_kernel_dal" ~path:"tx_kernel_dal.wasm" () end +let octez_agnostic_baker = + Uses.make ~tag:"agnostic_baker" ~path:"./octez-agnostic-baker" () + (* TODO: tezos/tezos#4803 Can we do better than to depend on script-inputs? *) -- GitLab From d7abc520b8a722e2d1b13bff997d8429461ceda3 Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Tue, 8 Oct 2024 16:17:45 +0200 Subject: [PATCH 6/7] Tezt: introduce simple agnostic_baking tests --- tezt/manual_tests/agnostic_baking.ml | 170 +++++++++++++++++++++++++++ tezt/manual_tests/main.ml | 3 + 2 files changed, 173 insertions(+) create mode 100644 tezt/manual_tests/agnostic_baking.ml diff --git a/tezt/manual_tests/agnostic_baking.ml b/tezt/manual_tests/agnostic_baking.ml new file mode 100644 index 000000000000..5af174763b97 --- /dev/null +++ b/tezt/manual_tests/agnostic_baking.ml @@ -0,0 +1,170 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2024 Nomadic Labs *) +(* *) +(*****************************************************************************) + +(* Testing + ------- + Component: Agnostic baker (octez-agnostic-baker) + Invocation: dune exec tezt/manual_tests/main.exe -- --file agnostic_baking.ml + Subject: Ensure that the agnostic baker behaves as expected +*) + +(** Boilerplate code to create a user-migratable node. Used in the + tests below. **) +let user_migratable_node_init ?node_name ?client_name ?(more_node_args = []) + ~migration_level ~migrate_to () = + let* node = + Node.init + ?name:node_name + ~patch_config: + (Node.Config_file.set_sandbox_network_with_user_activated_upgrades + [(migration_level, migrate_to)]) + ([Node.Synchronisation_threshold 0; Private_mode] @ more_node_args) + in + let* client = Client.(init ?name:client_name ~endpoint:(Node node) ()) in + Lwt.return (client, node) + +(** [block_check ?level ~expected_block_type ~migrate_to ~migrate_from client] + is generic check that a block of type [expected_block_type] contains + (protocol) metatadata conforming to its type at [level]. **) +let block_check ?level ~expected_block_type ~migrate_to ~migrate_from client = + let block = + match level with Some level -> Some (string_of_int level) | None -> None + in + let* metadata = + Client.RPC.call client @@ RPC.get_chain_block_metadata ?block () + in + let protocol = metadata.protocol in + let next_protocol = metadata.next_protocol in + (match expected_block_type with + | `Migration -> + Check.( + (next_protocol = Protocol.hash migrate_to) + string + ~error_msg:"expected next protocol to be %R, got %L") ; + Check.( + (protocol = Protocol.hash migrate_from) + string + ~error_msg:"expected (from) protocol to be %R, got %L") + | `Non_migration -> + Check.( + (next_protocol = protocol) + string + ~error_msg:"expected a non migration block ")) ; + Lwt.return_unit + +let wait_for_active_protocol_waiting agnostic_baker = + Agnostic_baker.wait_for + agnostic_baker + "waiting_for_active_protocol.v0" + (fun _json -> Some ()) + +(* Performs a protocol migration thanks to a UAU and the agnostic + baker. *) +let perform_protocol_migration ?node_name ?client_name ?parameter_file + ~blocks_per_cycle ~migration_level ~migrate_from ~migrate_to + ~baked_blocks_after_migration () = + assert (migration_level >= blocks_per_cycle) ; + Log.info "Node starting" ; + let* client, node = + user_migratable_node_init + ?node_name + ?client_name + ~migration_level + ~migrate_to + () + in + Log.info "Node %s initialized" (Node.name node) ; + let baker = Agnostic_baker.create node client in + let wait_for_active_protocol_waiting = + wait_for_active_protocol_waiting baker + in + Log.info "Starting agnostic baker" ; + let* () = Agnostic_baker.run baker in + let* () = wait_for_active_protocol_waiting in + let* () = Agnostic_baker.wait_for_ready baker in + let* () = + Client.activate_protocol + ~protocol:migrate_from + client + ?parameter_file + ~timestamp:Client.Now + in + Log.info "Protocol %s activated" (Protocol.hash migrate_from) ; + Log.info "Baking at least %d blocks to trigger migration" migration_level ; + let* _level = Node.wait_for_level node migration_level in + (* Ensure that the block before migration is consistent *) + Log.info "Checking migration block consistency" ; + let* () = + block_check + ~expected_block_type:`Migration + client + ~migrate_from + ~migrate_to + ~level:migration_level + in + let* _level = Node.wait_for_level node (migration_level + 1) in + (* Ensure that we migrated *) + Log.info "Checking migration block consistency" ; + let* () = + block_check + ~expected_block_type:`Non_migration + client + ~migrate_from + ~migrate_to + ~level:(migration_level + 1) + in + (* Test that we can still bake after migration *) + let* _level = Node.wait_for_level node baked_blocks_after_migration in + let* () = Agnostic_baker.terminate baker in + Lwt.return_unit + +let migrate ~migrate_from ~migrate_to = + let parameters = JSON.parse_file (Protocol.parameter_file migrate_to) in + let blocks_per_cycle = JSON.(get "blocks_per_cycle" parameters |> as_int) in + let consensus_rights_delay = + JSON.(get "consensus_rights_delay" parameters |> as_int) + in + (* Migration level is set arbitrarily *) + let migration_level = blocks_per_cycle in + Test.register + ~__FILE__ + ~title: + (Format.sprintf + "Protocol migration (from %s to %s) with agnostic baker" + (Protocol.tag migrate_from) + (Protocol.tag migrate_to)) + ~tags:["protocol"; "migration"; "agnostic"; "baker"] + ~uses:[Constant.octez_agnostic_baker] + @@ fun () -> + let* () = + perform_protocol_migration + ~blocks_per_cycle + ~migration_level + ~migrate_from + ~migrate_to + ~baked_blocks_after_migration: + (2 * consensus_rights_delay * blocks_per_cycle) + () + in + unit + +let start_stop = + Test.register + ~__FILE__ + ~title:"Agnostic baker starts and stops" + ~tags:["sandbox"; "agnostic"; "baker"; "init"] + ~uses:[Constant.octez_agnostic_baker] + @@ fun () -> + let* node, client = Client.init_with_node `Client () in + let* baker = Agnostic_baker.init node client in + let* () = Agnostic_baker.wait_for_ready baker in + let* () = Agnostic_baker.terminate baker in + unit + +let register ~migrate_from ~migrate_to = + start_stop ; + migrate ~migrate_from ~migrate_to diff --git a/tezt/manual_tests/main.ml b/tezt/manual_tests/main.ml index d4823982d7cf..8b363ba60dce 100644 --- a/tezt/manual_tests/main.ml +++ b/tezt/manual_tests/main.ml @@ -29,6 +29,9 @@ let () = (* Since manual tests do not run in the CI, we do not care about [~uses]. *) + Agnostic_baking.register + ~migrate_from:(Option.get @@ Protocol.previous_protocol Protocol.Alpha) + ~migrate_to:Protocol.Alpha ; Tezt_wrapper.error_mode_for_missing_use := Ignore ; Tezt_wrapper.error_mode_for_useless_use := Ignore ; Stresstest_command.register ~protocols:Protocol.all ; -- GitLab From 7c7edca42c6fe50b8e1b2ce0f09308d5e4ed8785 Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Thu, 17 Oct 2024 15:56:06 +0200 Subject: [PATCH 7/7] Tezt: update regression tests --- .../expected/tezt_wrapper.ml/runtime-dependency-tags.out | 1 + 1 file changed, 1 insertion(+) diff --git a/tezt/lib_wrapper/expected/tezt_wrapper.ml/runtime-dependency-tags.out b/tezt/lib_wrapper/expected/tezt_wrapper.ml/runtime-dependency-tags.out index 81ca99f123a7..d4a813f36427 100644 --- a/tezt/lib_wrapper/expected/tezt_wrapper.ml/runtime-dependency-tags.out +++ b/tezt/lib_wrapper/expected/tezt_wrapper.ml/runtime-dependency-tags.out @@ -9,6 +9,7 @@ accuser_psparisc: octez-accuser-PsParisC accuser_psquebec: octez-accuser-PsQuebec accuser_alpha: octez-accuser-alpha admin_client: octez-admin-client +agnostic_baker: octez-agnostic-baker baker_psparisc: octez-baker-PsParisC baker_psquebec: octez-baker-PsQuebec baker_alpha: octez-baker-alpha -- GitLab