From 18ce1cc4ab95b9e224032d52ed9f0072161bf799 Mon Sep 17 00:00:00 2001 From: Martin Tomazic Date: Mon, 24 Apr 2023 11:51:23 +0200 Subject: [PATCH 01/13] DAC: Add missing license to RPC_services.mli --- src/lib_dac/RPC_services.mli | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/lib_dac/RPC_services.mli b/src/lib_dac/RPC_services.mli index 0c7a8eb44505..90530a9723f2 100644 --- a/src/lib_dac/RPC_services.mli +++ b/src/lib_dac/RPC_services.mli @@ -1,3 +1,29 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Trili Tech, *) +(* Copyright (c) 2023 Marigold *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + (** POST dac/store_preimage to post a payload using a given [pagination_scheme]. It returns the base58 encoded root page hash and the raw bytes. *) -- GitLab From 40042712ed4d84255829b19082b1659ae4d4abf6 Mon Sep 17 00:00:00 2001 From: Martin Tomazic Date: Tue, 9 May 2023 12:46:39 +0200 Subject: [PATCH 02/13] DAC: Remove register_get_verify_signature from Coordinator mode --- src/lib_dac_node/RPC_server.ml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/lib_dac_node/RPC_server.ml b/src/lib_dac_node/RPC_server.ml index 2e9f1602afd6..e82cfc503cd2 100644 --- a/src/lib_dac_node/RPC_server.ml +++ b/src/lib_dac_node/RPC_server.ml @@ -325,14 +325,10 @@ module Coordinator = struct let hash_streamer = coordinator_node_ctxt.Node_context.Coordinator.hash_streamer in - let public_keys_opt = - Node_context.Coordinator.public_keys_opt coordinator_node_ctxt - in let certificate_streamers = coordinator_node_ctxt.certificate_streamers in let committee_members = coordinator_node_ctxt.committee_members in Tezos_rpc.Directory.empty |> register_post_preimage dac_plugin hash_streamer page_store - |> register_get_verify_signature dac_plugin public_keys_opt |> register_get_preimage dac_plugin page_store |> register_monitor_root_hashes hash_streamer |> register_monitor_certificate -- GitLab From fabe9beba68c8e9d03faadcc8d4e1602e66d002f Mon Sep 17 00:00:00 2001 From: Martin Tomazic Date: Mon, 24 Apr 2023 12:22:04 +0200 Subject: [PATCH 03/13] DAC: Define `Api` module for versioning DAC API --- src/lib_dac/RPC_services.ml | 19 +++++++++++++++++++ src/lib_dac/RPC_services.mli | 15 +++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/src/lib_dac/RPC_services.ml b/src/lib_dac/RPC_services.ml index a8d5821dfced..06bc40d29727 100644 --- a/src/lib_dac/RPC_services.ml +++ b/src/lib_dac/RPC_services.ml @@ -24,6 +24,25 @@ (* *) (*****************************************************************************) +module Api = struct + type version = V0 | V1 + + let version_rpc_arg = + let construct = function V0 -> "v0" | V1 -> "v1" in + let destruct version = + match version with + | "v0" -> Ok V0 + | "v1" -> Ok V1 + | _ -> Error "Invalid API version." + in + Tezos_rpc.Arg.make + ~descr:"API version" + ~name:"api_version" + ~destruct + ~construct + () +end + (* A variant of [Sc_rollup_reveal_hash.encoding] that prefers hex encoding over b58check encoding for JSON. *) let store_preimage_request_encoding = diff --git a/src/lib_dac/RPC_services.mli b/src/lib_dac/RPC_services.mli index 90530a9723f2..7b61ce5c30b1 100644 --- a/src/lib_dac/RPC_services.mli +++ b/src/lib_dac/RPC_services.mli @@ -24,6 +24,21 @@ (* *) (*****************************************************************************) +(** [Api] module is used for versioning DAC API. *) +module Api : sig + (** [version] type is used to version DAC API. *) + type version = + | V0 + (** [V0] will get deprecated soon, once we refactor old legacy tests. + Do not use! *) + | V1 + (** [V1] is a version that corresponds to the first public release of + the DAC API. *) + + (** [version_rpc_arg] is a API version argument for the RPCs.*) + val version_rpc_arg : version Tezos_rpc.Arg.arg +end + (** POST dac/store_preimage to post a payload using a given [pagination_scheme]. It returns the base58 encoded root page hash and the raw bytes. *) -- GitLab From cff2e5e5046017175a4fa5e7433e52c4395e7fa8 Mon Sep 17 00:00:00 2001 From: Martin Tomazic Date: Tue, 9 May 2023 12:35:03 +0200 Subject: [PATCH 04/13] Fixup: pp version_rpc_arg correctly --- src/lib_dac/RPC_services.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib_dac/RPC_services.ml b/src/lib_dac/RPC_services.ml index 06bc40d29727..99fbf26e32fc 100644 --- a/src/lib_dac/RPC_services.ml +++ b/src/lib_dac/RPC_services.ml @@ -33,7 +33,7 @@ module Api = struct match version with | "v0" -> Ok V0 | "v1" -> Ok V1 - | _ -> Error "Invalid API version." + | invalid_version -> Error invalid_version in Tezos_rpc.Arg.make ~descr:"API version" -- GitLab From 5bb2896a4b71dc789aa77c9db6624111de6fd6fa Mon Sep 17 00:00:00 2001 From: Martin Tomazic Date: Mon, 24 Apr 2023 12:01:54 +0200 Subject: [PATCH 05/13] DAC: Version DAC API with `v0` and `v1` DAC API is currently versioned via two major versions: 1. V0 that corresponds to the API that consists of the old legacy endpoints also used by the `legacy` (super) mode. It is meant to be used during the demo. 2. V1 that corresponds to the API that will be released as a first public API. --- src/lib_dac/RPC_services.ml | 20 +- src/lib_dac/RPC_services.mli | 26 +- src/lib_dac_client/dac_node_client.ml | 45 +- src/lib_dac_client/dac_node_client.mli | 44 +- src/lib_dac_node/RPC_server.ml | 392 ++++++++++-------- src/lib_dac_node/handler.ml | 32 +- src/lib_dac_node/monitor_services.ml | 20 +- src/lib_dac_node/monitor_services.mli | 6 +- src/lib_dac_node/page_store.ml | 54 ++- src/lib_dac_node/page_store.mli | 12 +- src/lib_dac_node/signature_manager.ml | 82 ++-- src/lib_dac_node/signature_manager.mli | 2 + .../test/test_dac_pages_encoding.ml | 9 +- .../test/test_dac_pages_encoding.ml | 9 +- tezt/lib_tezos/dac_rpc.ml | 39 +- tezt/lib_tezos/dac_rpc.mli | 56 ++- tezt/tests/dac.ml | 170 +++++--- 17 files changed, 635 insertions(+), 383 deletions(-) diff --git a/src/lib_dac/RPC_services.ml b/src/lib_dac/RPC_services.ml index 99fbf26e32fc..0a2bf7920e23 100644 --- a/src/lib_dac/RPC_services.ml +++ b/src/lib_dac/RPC_services.ml @@ -70,7 +70,7 @@ let post_store_preimage = ~query:Tezos_rpc.Query.empty ~input:store_preimage_request_encoding ~output:store_preimage_response_encoding - Tezos_rpc.Path.(open_root / "store_preimage") + Tezos_rpc.Path.(open_root /: Api.version_rpc_arg / "store_preimage") (* DAC/FIXME: https://gitlab.com/tezos/tezos/-/issues/4263 remove this endpoint once end-to-end tests are in place. *) @@ -79,14 +79,16 @@ let get_verify_signature = ~description:"Verify signature of an external message to inject in L1" ~query:external_message_query ~output:Data_encoding.bool - Tezos_rpc.Path.(open_root / "verify_signature") + Tezos_rpc.Path.(open_root /: Api.version_rpc_arg / "verify_signature") let get_preimage = Tezos_rpc.Service.get_service ~description:"Retrieves a page by its page hash and returns its contents" ~query:Tezos_rpc.Query.empty ~output:Data_encoding.bytes - Tezos_rpc.Path.(open_root / "preimage" /: Dac_plugin.raw_hash_rpc_arg) + Tezos_rpc.Path.( + open_root /: Api.version_rpc_arg / "preimage" + /: Dac_plugin.raw_hash_rpc_arg) let put_dac_member_signature = Tezos_rpc.Service.put_service @@ -95,7 +97,7 @@ let put_dac_member_signature = ~query:Tezos_rpc.Query.empty ~input:Signature_repr.encoding ~output:Data_encoding.empty - Tezos_rpc.Path.(open_root / "dac_member_signature") + Tezos_rpc.Path.(open_root /: Api.version_rpc_arg / "dac_member_signature") let get_certificate = Tezos_rpc.Service.get_service @@ -103,7 +105,9 @@ let get_certificate = "Retrieve the Dac certificate associated with the given root page hash" ~query:Tezos_rpc.Query.empty ~output:(Data_encoding.option Certificate_repr.encoding) - Tezos_rpc.Path.(open_root / "certificates" /: Dac_plugin.raw_hash_rpc_arg) + Tezos_rpc.Path.( + open_root /: Api.version_rpc_arg / "certificates" + /: Dac_plugin.raw_hash_rpc_arg) let get_missing_page = Tezos_rpc.Service.get_service @@ -114,7 +118,9 @@ let get_missing_page = Observer mode." ~query:Tezos_rpc.Query.empty ~output:Data_encoding.bytes - Tezos_rpc.Path.(open_root / "missing_page" /: Dac_plugin.raw_hash_rpc_arg) + Tezos_rpc.Path.( + open_root /: Api.version_rpc_arg / "missing_page" + /: Dac_plugin.raw_hash_rpc_arg) (* TODO: https://gitlab.com/tezos/tezos/-/issues/4935 Coordinator's "POST /preimage" endpoint should in addition to root page hash @@ -134,5 +140,5 @@ module Coordinator = struct ~query:Tezos_rpc.Query.empty ~input:Data_encoding.bytes ~output:Dac_plugin.raw_hash_encoding - Tezos_rpc.Path.(open_root / "preimage") + Tezos_rpc.Path.(open_root /: Api.version_rpc_arg / "preimage") end diff --git a/src/lib_dac/RPC_services.mli b/src/lib_dac/RPC_services.mli index 7b61ce5c30b1..e88e586dcd36 100644 --- a/src/lib_dac/RPC_services.mli +++ b/src/lib_dac/RPC_services.mli @@ -45,7 +45,7 @@ end val post_store_preimage : ( [`POST], unit, - unit, + unit * Api.version, unit, Bytes.t * Pagination_scheme.t, Dac_plugin.raw_hash * Bytes.t ) @@ -56,7 +56,13 @@ val post_store_preimage : of the DAL node must be the same that was used to produce the [external_message]. *) val get_verify_signature : - ([`GET], unit, unit, string option, unit, bool) Tezos_rpc.Service.service + ( [`GET], + unit, + unit * Api.version, + string option, + unit, + bool ) + Tezos_rpc.Service.service (** GET dac/preimage requests the preimage of hash, consisting of a single page, from cctxt. When the request succeeds, the raw page will be @@ -64,7 +70,7 @@ val get_verify_signature : val get_preimage : ( [`GET], unit, - unit * Dac_plugin.raw_hash, + (unit * Api.version) * Dac_plugin.raw_hash, unit, unit, Bytes.t ) @@ -73,14 +79,20 @@ val get_preimage : (** PUT dac/member_signature endpoint stores the [signature] generated from signing [hex_root_hash] by [dac_member_pkh]. *) val put_dac_member_signature : - ([`PUT], unit, unit, unit, Signature_repr.t, unit) Tezos_rpc.Service.service + ( [`PUT], + unit, + unit * Api.version, + unit, + Signature_repr.t, + unit ) + Tezos_rpc.Service.service (** GET dac/certificate endpoint returns the DAC certificate for the provided [root_page_hash]. *) val get_certificate : ( [`GET], unit, - unit * Dac_plugin.raw_hash, + (unit * Api.version) * Dac_plugin.raw_hash, unit, unit, Certificate_repr.t option ) @@ -92,7 +104,7 @@ val get_certificate : val get_missing_page : ( [`GET], unit, - unit * Dac_plugin.raw_hash, + (unit * Api.version) * Dac_plugin.raw_hash, unit, unit, Bytes.t ) @@ -107,7 +119,7 @@ module Coordinator : sig val post_preimage : ( [`POST], unit, - unit, + unit * Api.version, unit, Bytes.t, Dac_plugin.raw_hash ) diff --git a/src/lib_dac_client/dac_node_client.ml b/src/lib_dac_client/dac_node_client.ml index 7ecc19953eef..6335f2380d6f 100644 --- a/src/lib_dac_client/dac_node_client.ml +++ b/src/lib_dac_client/dac_node_client.ml @@ -50,26 +50,47 @@ let make_unix_cctxt ~scheme ~host ~port = hash of the protocol that the coordinator was using when the page hash was computed. *) -let get_preimage (cctxt : #cctxt) ~page_hash = - cctxt#call_service RPC_services.get_preimage ((), page_hash) () () +let get_preimage (cctxt : #cctxt) api_version ~page_hash = + cctxt#call_service + RPC_services.get_preimage + (((), api_version), page_hash) + () + () -let post_store_preimage (cctxt : #cctxt) ~payload ~pagination_scheme = +let post_store_preimage (cctxt : #cctxt) ~payload ~pagination_scheme api_version + = cctxt#call_service RPC_services.post_store_preimage - () + ((), api_version) () (payload, pagination_scheme) -let get_verify_signature (cctxt : #cctxt) ~external_message = - cctxt#call_service RPC_services.get_verify_signature () external_message () +let get_verify_signature (cctxt : #cctxt) ~external_message api_version = + cctxt#call_service + RPC_services.get_verify_signature + ((), api_version) + external_message + () -let put_dac_member_signature (cctxt : #cctxt) ~signature = - cctxt#call_service RPC_services.put_dac_member_signature () () signature +let put_dac_member_signature (cctxt : #cctxt) ~signature api_version = + cctxt#call_service + RPC_services.put_dac_member_signature + ((), api_version) + () + signature -let get_certificate (cctxt : #cctxt) ~root_page_hash = - cctxt#call_service RPC_services.get_certificate ((), root_page_hash) () () +let get_certificate (cctxt : #cctxt) ~root_page_hash api_version = + cctxt#call_service + RPC_services.get_certificate + (((), api_version), root_page_hash) + () + () module Coordinator = struct - let post_preimage (cctxt : #cctxt) ~payload = - cctxt#call_service RPC_services.Coordinator.post_preimage () () payload + let post_preimage (cctxt : #cctxt) ~payload api_version = + cctxt#call_service + RPC_services.Coordinator.post_preimage + ((), api_version) + () + payload end diff --git a/src/lib_dac_client/dac_node_client.mli b/src/lib_dac_client/dac_node_client.mli index ee2665283bfb..ce6ee72f55ff 100644 --- a/src/lib_dac_client/dac_node_client.mli +++ b/src/lib_dac_client/dac_node_client.mli @@ -39,48 +39,62 @@ class unix_cctxt : the client configuration parameters. *) val make_unix_cctxt : scheme:string -> host:string -> port:int -> cctxt -(** [get_preimage cctxt ~hash] requests the preimage of hash, consisting of a - single page, from cctxt. When the request succeeds, the raw page will be - returned as a sequence of bytes. *) +(** [get_preimage cctxt api_version ~hash] requests the preimage of hash, + consisting of a single page, from cctxt. When the request succeeds, + the raw page will be returned as a sequence of bytes. *) val get_preimage : - #cctxt -> page_hash:Dac_plugin.raw_hash -> bytes tzresult Lwt.t + #cctxt -> + RPC_services.Api.version -> + page_hash:Dac_plugin.raw_hash -> + bytes tzresult Lwt.t -(** [post_store_preimage cctxt ~payload ~pagination_scheme] posts a [payload] to dac/store_preimage - using a given [pagination_scheme]. It returns the base58 encoded root page hash - and the raw bytes. *) +(** [post_store_preimage cctxt ~payload ~pagination_scheme api_version] posts a + [payload] to dac/store_preimage using a given [pagination_scheme]. + It returns the base58 encoded root page hash and the raw bytes. *) val post_store_preimage : #cctxt -> payload:bytes -> pagination_scheme:Pagination_scheme.t -> + RPC_services.Api.version -> (Dac_plugin.raw_hash * bytes) tzresult Lwt.t -(** [get_verify_signature cctxt ~external_message] requests the DAL node to verify - the signature of the external message [external_message] via +(** [get_verify_signature cctxt ~external_message api_version] requests the DAL + node to verify the signature of the external message [external_message] via the plugin/dac/verify_signature endpoint. The DAC committee of the DAL node must be the same that was used to produce the [external_message]. *) val get_verify_signature : - #cctxt -> external_message:string option -> bool tzresult Lwt.t + #cctxt -> + external_message:string option -> + RPC_services.Api.version -> + bool tzresult Lwt.t -(** [put_dac_member_signature cctxt ~signature:Signature_repr.t] +(** [put_dac_member_signature cctxt ~signature:Signature_repr.t api_version] stores the [signature] generated from signing [hex_root_hash] by [dac_member_pkh]. *) val put_dac_member_signature : - #cctxt -> signature:Signature_repr.t -> unit tzresult Lwt.t + #cctxt -> + signature:Signature_repr.t -> + RPC_services.Api.version -> + unit tzresult Lwt.t -(** [get_certificate cctxt ~root_page_hash] fetches the DAC certificate for the +(** [get_certificate cctxt ~root_page_hash api_version] fetches the DAC certificate for the provided [root_page_hash]. *) val get_certificate : #cctxt -> root_page_hash:Dac_plugin.raw_hash -> + RPC_services.Api.version -> Certificate_repr.t option tzresult Lwt.t module Coordinator : sig - (** [post_preimage cctxt ~payload] sends a [payload] to the DAC + (** [post_preimage cctxt ~payload api_version] sends a [payload] to the DAC [Coordinator] via a POST RPC call to dac/preimage. It returns a hex encoded root page hash, produced by [Merkle_tree_V0] pagination scheme. On the backend side it also pushes root page hash of the preimage to all the subscribed DAC Members and Observers. *) val post_preimage : - #cctxt -> payload:bytes -> Dac_plugin.raw_hash tzresult Lwt.t + #cctxt -> + payload:bytes -> + RPC_services.Api.version -> + Dac_plugin.raw_hash tzresult Lwt.t end diff --git a/src/lib_dac_node/RPC_server.ml b/src/lib_dac_node/RPC_server.ml index e82cfc503cd2..4690a2ee9b89 100644 --- a/src/lib_dac_node/RPC_server.ml +++ b/src/lib_dac_node/RPC_server.ml @@ -58,101 +58,119 @@ let add_service registerer service handler directory = registerer directory service handler let handle_post_store_preimage dac_plugin cctxt dac_sk_uris page_store - hash_streamer (data, pagination_scheme) = - let open Lwt_result_syntax in - let open Pages_encoding in - let* root_hash = - match pagination_scheme with - | Pagination_scheme.Merkle_tree_V0 -> - (* FIXME: https://gitlab.com/tezos/tezos/-/issues/4897 - Once new "PUT /preimage" endpoint is implemented, pushing - a new root hash to the data streamer should be moved there. - Tezt for testing streaming of root hashes should also use - the new endpoint. *) - let* root_hash = - Merkle_tree.V0.Filesystem.serialize_payload - dac_plugin - ~page_store - data - in - let () = - Data_streamer.publish hash_streamer (Dac_plugin.hash_to_raw root_hash) - in - let*! () = - Event.emit_root_hash_pushed_to_data_streamer dac_plugin root_hash - in - return root_hash - | Pagination_scheme.Hash_chain_V0 -> - Hash_chain.V0.serialize_payload + hash_streamer (data, pagination_scheme) api_version = + match api_version with + | RPC_services.Api.V0 -> ( + let open Lwt_result_syntax in + let open Pages_encoding in + let* root_hash = + match pagination_scheme with + | Pagination_scheme.Merkle_tree_V0 -> + (* FIXME: https://gitlab.com/tezos/tezos/-/issues/4897 + Once new "PUT /preimage" endpoint is implemented, pushing + a new root hash to the data streamer should be moved there. + Tezt for testing streaming of root hashes should also use + the new endpoint. *) + let* root_hash = + Merkle_tree.V0.Filesystem.serialize_payload + dac_plugin + ~page_store + data + in + let () = + Data_streamer.publish + hash_streamer + (Dac_plugin.hash_to_raw root_hash) + in + let*! () = + Event.emit_root_hash_pushed_to_data_streamer dac_plugin root_hash + in + return root_hash + | Pagination_scheme.Hash_chain_V0 -> + Hash_chain.V0.serialize_payload + dac_plugin + ~for_each_page:(fun (hash, content) -> + Page_store.Filesystem.save dac_plugin page_store ~hash ~content) + data + in + let* signature, witnesses = + Signature_manager.Legacy.sign_root_hash dac_plugin - ~for_each_page:(fun (hash, content) -> - Page_store.Filesystem.save dac_plugin page_store ~hash ~content) - data - in - let* signature, witnesses = - Signature_manager.Legacy.sign_root_hash - dac_plugin - cctxt - dac_sk_uris - root_hash - in - let raw_root_hash = Dac_plugin.hash_to_raw root_hash in - let*! external_message = - External_message.Default.make dac_plugin root_hash signature witnesses - in - match external_message with - | Ok external_message -> return @@ (raw_root_hash, external_message) - | Error _ -> tzfail @@ Cannot_construct_external_message + cctxt + dac_sk_uris + root_hash + in + let raw_root_hash = Dac_plugin.hash_to_raw root_hash in + let*! external_message = + External_message.Default.make dac_plugin root_hash signature witnesses + in + match external_message with + | Ok external_message -> return @@ (raw_root_hash, external_message) + | Error _ -> tzfail @@ Cannot_construct_external_message) + | RPC_services.Api.V1 -> raise Not_found -let handle_get_verify_signature dac_plugin public_keys_opt encoded_l1_message = - let open Lwt_result_syntax in - let ((module Plugin) : Dac_plugin.t) = dac_plugin in - let external_message = - let open Option_syntax in - let* encoded_l1_message in - let* as_bytes = Hex.to_bytes @@ `Hex encoded_l1_message in - External_message.Default.of_bytes Plugin.encoding as_bytes - in - match external_message with - | None -> tzfail @@ Cannot_deserialize_external_message - | Some {root_hash; signature; witnesses} -> - Signature_manager.verify - dac_plugin - ~public_keys_opt - (Dac_plugin.hash_to_raw root_hash) - signature - witnesses +let handle_get_verify_signature dac_plugin public_keys_opt encoded_l1_message + api_version = + match api_version with + | RPC_services.Api.V0 -> ( + let open Lwt_result_syntax in + let ((module Plugin) : Dac_plugin.t) = dac_plugin in + let external_message = + let open Option_syntax in + let* encoded_l1_message in + let* as_bytes = Hex.to_bytes @@ `Hex encoded_l1_message in + External_message.Default.of_bytes Plugin.encoding as_bytes + in + match external_message with + | None -> tzfail @@ Cannot_deserialize_external_message + | Some {root_hash; signature; witnesses} -> + Signature_manager.verify + dac_plugin + ~public_keys_opt + (Dac_plugin.hash_to_raw root_hash) + signature + witnesses) + | RPC_services.Api.V1 -> raise Not_found -let handle_get_preimage dac_plugin page_store raw_hash = - let open Lwt_result_syntax in - let*? hash = Dac_plugin.raw_to_hash dac_plugin raw_hash in - Page_store.Filesystem.load dac_plugin page_store hash +let handle_get_preimage dac_plugin page_store api_version raw_hash = + match api_version with + | RPC_services.Api.V0 | RPC_services.Api.V1 -> + let open Lwt_result_syntax in + let*? hash = Dac_plugin.raw_to_hash dac_plugin raw_hash in + Page_store.Filesystem.load dac_plugin page_store hash (* Handler for subscribing to the streaming of root hashes via GET monitor/root_hashes RPC call. *) -let handle_monitor_root_hashes hash_streamer = - let open Lwt_syntax in - let stream, stopper = Data_streamer.handle_subscribe hash_streamer in - let shutdown () = Lwt_watcher.shutdown stopper in - let next () = Lwt_stream.get stream in - let* () = Event.(emit handle_new_subscription_to_hash_streamer ()) in - Tezos_rpc.Answer.return_stream {next; shutdown} +let handle_monitor_root_hashes hash_streamer api_version = + match api_version with + | RPC_services.Api.V0 | RPC_services.Api.V1 -> + let open Lwt_syntax in + let stream, stopper = Data_streamer.handle_subscribe hash_streamer in + let shutdown () = Lwt_watcher.shutdown stopper in + let next () = Lwt_stream.get stream in + let* () = Event.(emit handle_new_subscription_to_hash_streamer ()) in + Tezos_rpc.Answer.return_stream {next; shutdown} -let handle_get_certificate dac_plugin node_store raw_root_hash = - let open Lwt_result_syntax in - let*? root_hash = Dac_plugin.raw_to_hash dac_plugin raw_root_hash in +let handle_get_certificate dac_plugin node_store raw_root_hash api_version = + match api_version with + | RPC_services.Api.V0 | RPC_services.Api.V1 -> + let open Lwt_result_syntax in + let*? root_hash = Dac_plugin.raw_to_hash dac_plugin raw_root_hash in - let+ value_opt = Store.Certificate_store.find node_store root_hash in - Option.map - (fun Store.{aggregate_signature; witnesses} -> - Certificate_repr. - {aggregate_signature; witnesses; root_hash = raw_root_hash}) - value_opt + let+ value_opt = Store.Certificate_store.find node_store root_hash in + Option.map + (fun Store.{aggregate_signature; witnesses} -> + Certificate_repr. + {aggregate_signature; witnesses; root_hash = raw_root_hash}) + value_opt -let handle_get_missing_page cctxt page_store dac_plugin raw_root_hash = +let handle_get_missing_page cctxt page_store dac_plugin raw_root_hash + api_version = let open Lwt_result_syntax in let*? root_hash = Dac_plugin.raw_to_hash dac_plugin raw_root_hash in - let remote_store = Page_store.Remote.(init {cctxt; page_store}) in + let remote_store = + Page_store.Remote.(init {cctxt; page_store; api_version}) + in let* preimage = (* TODO: https://gitlab.com/tezos/tezos/-/issues/5142 Retrieve missing page from dac committee via "flooding". *) @@ -165,127 +183,145 @@ let register_post_store_preimage ctx cctxt dac_sk_uris page_store hash_streamer directory = directory |> add_service - Tezos_rpc.Directory.register0 + Tezos_rpc.Directory.register1 RPC_services.post_store_preimage - (fun () input -> + (fun api_version () input -> handle_post_store_preimage ctx cctxt dac_sk_uris page_store hash_streamer - input) + input + api_version) let register_get_verify_signature dac_plugin public_keys_opt directory = directory |> add_service - Tezos_rpc.Directory.register0 + Tezos_rpc.Directory.register1 RPC_services.get_verify_signature - (fun external_message () -> - handle_get_verify_signature dac_plugin public_keys_opt external_message) + (fun api_version external_message () -> + handle_get_verify_signature + dac_plugin + public_keys_opt + external_message + api_version) let register_get_preimage dac_plugin page_store = add_service - Tezos_rpc.Directory.register1 + Tezos_rpc.Directory.register2 RPC_services.get_preimage - (fun hash () () -> handle_get_preimage dac_plugin page_store hash) + (fun api_version hash () () -> + handle_get_preimage dac_plugin page_store api_version hash) let register_monitor_root_hashes hash_streamer dir = Tezos_rpc.Directory.gen_register dir Monitor_services.S.root_hashes - (fun () () () -> handle_monitor_root_hashes hash_streamer) + (fun ((), api_version) () () -> + handle_monitor_root_hashes hash_streamer api_version) let register_get_certificate node_store dac_plugin = add_service - Tezos_rpc.Directory.register1 + Tezos_rpc.Directory.register2 RPC_services.get_certificate - (fun root_hash () () -> - handle_get_certificate dac_plugin node_store root_hash) + (fun api_version root_hash () () -> + handle_get_certificate dac_plugin node_store root_hash api_version) let register_get_missing_page dac_plugin page_store cctxt = add_service - Tezos_rpc.Directory.register1 + Tezos_rpc.Directory.register2 RPC_services.get_missing_page - (fun root_hash () () -> - handle_get_missing_page cctxt page_store dac_plugin root_hash) + (fun api_version root_hash () () -> + handle_get_missing_page cctxt page_store dac_plugin root_hash api_version) module Coordinator = struct - let handle_post_preimage dac_plugin page_store hash_streamer payload = - let open Lwt_result_syntax in - let* root_hash = - Pages_encoding.Merkle_tree.V0.Filesystem.serialize_payload - dac_plugin - ~page_store - payload - in - let () = - Data_streamer.publish hash_streamer (Dac_plugin.hash_to_raw root_hash) - in - let*! () = - Event.emit_root_hash_pushed_to_data_streamer dac_plugin root_hash - in - return @@ Dac_plugin.hash_to_raw root_hash + let handle_post_preimage dac_plugin page_store hash_streamer payload + api_version = + match api_version with + | RPC_services.Api.V0 | RPC_services.Api.V1 -> + let open Lwt_result_syntax in + let* root_hash = + Pages_encoding.Merkle_tree.V0.Filesystem.serialize_payload + dac_plugin + ~page_store + payload + in + let () = + Data_streamer.publish hash_streamer (Dac_plugin.hash_to_raw root_hash) + in + let*! () = + Event.emit_root_hash_pushed_to_data_streamer dac_plugin root_hash + in + return @@ Dac_plugin.hash_to_raw root_hash let handle_monitor_certificate dac_plugin ro_node_store certificate_streamers - raw_root_hash committee_members = - let open Lwt_result_syntax in - let*? stream, stopper = - Certificate_streamers.handle_subscribe - dac_plugin - certificate_streamers - raw_root_hash - in - let*? root_hash = Dac_plugin.raw_to_hash dac_plugin raw_root_hash in - let*! () = Event.emit_new_subscription_to_certificate_updates root_hash in - let shutdown () = Lwt_watcher.shutdown stopper in - let next () = Lwt_stream.get stream in - (* Add the current certificate to the streamer, if any, to ensure that - a certificate is returned even in the case that no updates to the - certificate happen for a long time. *) - let*! current_certificate_store_value_res = - Store.Certificate_store.find ro_node_store root_hash - in - match current_certificate_store_value_res with - | Ok current_certificate_store_value -> - let () = - Option.iter - (fun Store.{aggregate_signature; witnesses} -> - let certificate = - Certificate_repr. - {root_hash = raw_root_hash; aggregate_signature; witnesses} - in - let _ = - Certificate_streamers.push - dac_plugin - certificate_streamers - raw_root_hash - certificate - in - if - Certificate_repr.all_committee_members_have_signed - committee_members - certificate - then - let _ = - Certificate_streamers.close - dac_plugin - certificate_streamers - raw_root_hash - in - () - else ()) - current_certificate_store_value + raw_root_hash committee_members api_version = + match api_version with + | RPC_services.Api.V0 | RPC_services.Api.V1 -> ( + let open Lwt_result_syntax in + let*? stream, stopper = + Certificate_streamers.handle_subscribe + dac_plugin + certificate_streamers + raw_root_hash + in + let*? root_hash = Dac_plugin.raw_to_hash dac_plugin raw_root_hash in + let*! () = + Event.emit_new_subscription_to_certificate_updates root_hash in - return (next, shutdown) - | Error e -> fail e + let shutdown () = Lwt_watcher.shutdown stopper in + let next () = Lwt_stream.get stream in + (* Add the current certificate to the streamer, if any, to ensure that + a certificate is returned even in the case that no updates to the + certificate happen for a long time. *) + let*! current_certificate_store_value_res = + Store.Certificate_store.find ro_node_store root_hash + in + match current_certificate_store_value_res with + | Ok current_certificate_store_value -> + let () = + Option.iter + (fun Store.{aggregate_signature; witnesses} -> + let certificate = + Certificate_repr. + { + root_hash = raw_root_hash; + aggregate_signature; + witnesses; + } + in + let _ = + Certificate_streamers.push + dac_plugin + certificate_streamers + raw_root_hash + certificate + in + if + Certificate_repr.all_committee_members_have_signed + committee_members + certificate + then + let _ = + Certificate_streamers.close + dac_plugin + certificate_streamers + raw_root_hash + in + () + else ()) + current_certificate_store_value + in + return (next, shutdown) + | Error e -> fail e) let register_monitor_certificate dac_plugin ro_node_store certificate_streamers committee_members dir = Tezos_rpc.Directory.gen_register dir Monitor_services.S.certificate - (fun ((), root_hash) () () -> + (fun (((), api_version), root_hash) () () -> let open Lwt_result_syntax in let*! handler = handle_monitor_certificate @@ -294,6 +330,7 @@ module Coordinator = struct certificate_streamers root_hash committee_members + api_version in match handler with | Ok (next, shutdown) -> Tezos_rpc.Answer.return_stream {next; shutdown} @@ -301,24 +338,30 @@ module Coordinator = struct let register_post_preimage dac_plugin hash_streamer page_store = add_service - Tezos_rpc.Directory.register0 + Tezos_rpc.Directory.register1 RPC_services.Coordinator.post_preimage - (fun () payload -> - handle_post_preimage dac_plugin page_store hash_streamer payload) + (fun api_version () payload -> + handle_post_preimage + dac_plugin + page_store + hash_streamer + payload + api_version) let register_put_dac_member_signature ctx dac_plugin rw_node_store page_store cctxt = add_service - Tezos_rpc.Directory.register0 + Tezos_rpc.Directory.register1 RPC_services.put_dac_member_signature - (fun () dac_member_signature -> + (fun api_version () dac_member_signature -> Signature_manager.Coordinator.handle_put_dac_member_signature ctx dac_plugin rw_node_store page_store cctxt - dac_member_signature) + dac_member_signature + api_version) let dynamic_rpc_dir dac_plugin rw_store page_store cctxt coordinator_node_ctxt = @@ -361,16 +404,17 @@ module Legacy = struct let register_put_dac_member_signature ctx dac_plugin rw_node_store page_store cctxt = add_service - Tezos_rpc.Directory.register0 + Tezos_rpc.Directory.register1 RPC_services.put_dac_member_signature - (fun () dac_member_signature -> + (fun api_version () dac_member_signature -> Signature_manager.Legacy.handle_put_dac_member_signature ctx dac_plugin rw_node_store page_store cctxt - dac_member_signature) + dac_member_signature + api_version) let dynamic_rpc_dir dac_plugin rw_store page_store cctxt legacy_node_ctxt = let hash_streamer = legacy_node_ctxt.Node_context.Legacy.hash_streamer in diff --git a/src/lib_dac_node/handler.ml b/src/lib_dac_node/handler.ml index 472fea25d6af..a4ba3d03a19f 100644 --- a/src/lib_dac_node/handler.ml +++ b/src/lib_dac_node/handler.ml @@ -157,6 +157,7 @@ module Committee_member = struct Dac_node_client.put_dac_member_signature coordinator_cctxt ~signature:signature_repr + RPC_services.Api.V1 in let*! () = Event.emit_signature_pushed_to_coordinator signature in return_unit @@ -197,12 +198,18 @@ module Committee_member = struct return () in let remote_store = - Page_store.(Remote.init {cctxt = coordinator_cctxt; page_store}) + Page_store.( + Remote.init + { + cctxt = coordinator_cctxt; + page_store; + api_version = RPC_services.Api.V1; + }) in let*! () = Event.(emit subscribed_to_root_hashes_stream ()) in make_stream_daemon (handler dac_plugin remote_store) - (Monitor_services.root_hashes coordinator_cctxt) + (Monitor_services.root_hashes coordinator_cctxt RPC_services.Api.V1) end (** Handlers specific to an [Observer]. An [Observer] is responsible for @@ -240,12 +247,18 @@ module Observer = struct return () in let remote_store = - Page_store.(Remote.init {cctxt = coordinator_cctxt; page_store}) + Page_store.( + Remote.init + { + cctxt = coordinator_cctxt; + page_store; + api_version = RPC_services.Api.V1; + }) in let*! () = Event.(emit subscribed_to_root_hashes_stream ()) in make_stream_daemon (handler dac_plugin remote_store) - (Monitor_services.root_hashes coordinator_cctxt) + (Monitor_services.root_hashes coordinator_cctxt RPC_services.Api.V1) end (** Handlers specific to a [Legacy] DAC node. If no @@ -298,6 +311,7 @@ module Legacy = struct Dac_node_client.put_dac_member_signature coordinator_cctxt ~signature:signature_repr + RPC_services.Api.V0 in let*! () = Event.emit_signature_pushed_to_coordinator signature in return_unit @@ -349,12 +363,18 @@ module Legacy = struct return () in let remote_store = - Page_store.(Remote.init {cctxt = coordinator_cctxt; page_store}) + Page_store.( + Remote.init + { + cctxt = coordinator_cctxt; + page_store; + api_version = RPC_services.Api.V0; + }) in let*! () = Event.(emit subscribed_to_root_hashes_stream ()) in make_stream_daemon (handler dac_plugin remote_store) - (Monitor_services.root_hashes coordinator_cctxt) + (Monitor_services.root_hashes coordinator_cctxt RPC_services.Api.V0) end let handlers node_ctxt = diff --git a/src/lib_dac_node/monitor_services.ml b/src/lib_dac_node/monitor_services.ml index 41a38017d90b..cc0e7d2a6974 100644 --- a/src/lib_dac_node/monitor_services.ml +++ b/src/lib_dac_node/monitor_services.ml @@ -31,7 +31,9 @@ module S = struct responsible for the serialization of the dac payload (coordinator). " ~query:Tezos_rpc.Query.empty ~output:Dac_plugin.raw_hash_encoding - Tezos_rpc.Path.(open_root / "monitor" / "root_hashes") + Tezos_rpc.Path.( + open_root /: RPC_services.Api.version_rpc_arg / "monitor" + / "root_hashes") let certificate = Tezos_rpc.Service.get_service @@ -45,16 +47,22 @@ module S = struct ~query:Tezos_rpc.Query.empty ~output:Certificate_repr.encoding Tezos_rpc.Path.( - open_root / "monitor" / "certificate" /: Dac_plugin.raw_hash_rpc_arg) + open_root /: RPC_services.Api.version_rpc_arg / "monitor" + / "certificate" /: Dac_plugin.raw_hash_rpc_arg) end -let root_hashes dac_node_cctxt = - Tezos_rpc.Context.make_streamed_call S.root_hashes dac_node_cctxt () () () +let root_hashes dac_node_cctxt api_version = + Tezos_rpc.Context.make_streamed_call + S.root_hashes + dac_node_cctxt + ((), api_version) + () + () -let certificate dac_node_cctxt root_hash = +let certificate dac_node_cctxt root_hash api_version = Tezos_rpc.Context.make_streamed_call S.certificate dac_node_cctxt - ((), root_hash) + (((), api_version), root_hash) () () diff --git a/src/lib_dac_node/monitor_services.mli b/src/lib_dac_node/monitor_services.mli index 63cc638a05fd..fbe9fa4d854a 100644 --- a/src/lib_dac_node/monitor_services.mli +++ b/src/lib_dac_node/monitor_services.mli @@ -28,7 +28,7 @@ module S : sig val root_hashes : ( [`GET], unit, - unit, + unit * RPC_services.Api.version, unit, unit, Dac_plugin.raw_hash ) @@ -38,7 +38,7 @@ module S : sig val certificate : ( [`GET], unit, - unit * Dac_plugin.raw_hash, + (unit * RPC_services.Api.version) * Dac_plugin.raw_hash, unit, unit, Certificate_repr.t ) @@ -52,6 +52,7 @@ end *) val root_hashes : #Tezos_rpc.Context.streamed -> + RPC_services.Api.version -> (Dac_plugin.raw_hash Lwt_stream.t * Tezos_rpc.Context.stopper) Error_monad.tzresult Lwt.t @@ -59,6 +60,7 @@ val root_hashes : val certificate : #Tezos_rpc.Context.streamed -> Dac_plugin.raw_hash -> + RPC_services.Api.version -> (Certificate_repr.t Lwt_stream.t * Tezos_rpc.Context.stopper) Error_monad.tzresult Lwt.t diff --git a/src/lib_dac_node/page_store.ml b/src/lib_dac_node/page_store.ml index a7045287d276..c5b9ab2da33d 100644 --- a/src/lib_dac_node/page_store.ml +++ b/src/lib_dac_node/page_store.ml @@ -230,29 +230,37 @@ module With_remote_fetch (R : sig type remote_context val fetch : - Dac_plugin.t -> remote_context -> Dac_plugin.hash -> bytes tzresult Lwt.t + Dac_plugin.t -> + remote_context -> + RPC_services.Api.version -> + Dac_plugin.hash -> + bytes tzresult Lwt.t end) (P : S) : S - with type configuration = R.remote_context * P.t - and type t = R.remote_context * P.t = struct - type t = R.remote_context * P.t + with type configuration = R.remote_context * P.t * RPC_services.Api.version + and type t = R.remote_context * P.t * RPC_services.Api.version = struct + type t = R.remote_context * P.t * RPC_services.Api.version - type configuration = R.remote_context * P.t + type configuration = R.remote_context * P.t * RPC_services.Api.version - let init (remote_ctxt, page_store) = (remote_ctxt, page_store) + let init (remote_ctxt, page_store, api_version) = + (remote_ctxt, page_store, api_version) - let save plugin (_remote_ctxt, page_store) ~hash ~content = + let save plugin (_remote_ctxt, page_store, _api_version) ~hash ~content = P.save plugin page_store ~hash ~content - let mem plugin (_remote_ctxt, page_store) hash = P.mem plugin page_store hash + let mem plugin (_remote_ctxt, page_store, _api_version) hash = + P.mem plugin page_store hash - let load plugin (remote_ctxt, page_store) hash = + let load plugin (remote_ctxt, page_store, api_version) hash = let open Lwt_result_syntax in - let* page_exists_in_store = mem plugin (remote_ctxt, page_store) hash in + let* page_exists_in_store = + mem plugin (remote_ctxt, page_store, api_version) hash + in if page_exists_in_store then P.load plugin page_store hash else - let* content = R.fetch plugin remote_ctxt hash in + let* content = R.fetch plugin remote_ctxt api_version hash in let+ () = P.save plugin page_store ~hash ~content in content end @@ -260,6 +268,7 @@ end type remote_configuration = { cctxt : Dac_node_client.cctxt; page_store : Filesystem.t; + api_version : RPC_services.Api.version; } module Remote : S with type configuration = remote_configuration = struct @@ -267,15 +276,18 @@ module Remote : S with type configuration = remote_configuration = struct module Internal : S - with type configuration = Dac_node_client.cctxt * Filesystem.t - and type t = Dac_node_client.cctxt * Filesystem.t = + with type configuration = + Dac_node_client.cctxt * Filesystem.t * RPC_services.Api.version + and type t = + Dac_node_client.cctxt * Filesystem.t * RPC_services.Api.version = With_remote_fetch (struct type remote_context = Dac_node_client.cctxt - let fetch _dac_plugin remote_context hash = + let fetch _dac_plugin remote_context api_version hash = Dac_node_client.get_preimage remote_context + api_version ~page_hash:(Dac_plugin.hash_to_raw hash) end) (F) @@ -286,7 +298,8 @@ module Remote : S with type configuration = remote_configuration = struct type configuration = remote_configuration - let init {cctxt; page_store} = Internal.init (cctxt, page_store) + let init {cctxt; page_store; api_version} = + Internal.init (cctxt, page_store, api_version) end module Internal_for_tests = struct @@ -298,11 +311,16 @@ module Internal_for_tests = struct type remote_context val fetch : - Dac_plugin.t -> remote_context -> Dac_plugin.hash -> bytes tzresult Lwt.t + Dac_plugin.t -> + remote_context -> + RPC_services.Api.version -> + Dac_plugin.hash -> + bytes tzresult Lwt.t end) (P : S) : S - with type configuration = R.remote_context * P.t - and type t = R.remote_context * P.t = + with type configuration = + R.remote_context * P.t * RPC_services.Api.version + and type t = R.remote_context * P.t * RPC_services.Api.version = With_remote_fetch (R) (P) end diff --git a/src/lib_dac_node/page_store.mli b/src/lib_dac_node/page_store.mli index 97ce6e12174e..e0255c24ac39 100644 --- a/src/lib_dac_node/page_store.mli +++ b/src/lib_dac_node/page_store.mli @@ -88,6 +88,7 @@ module Filesystem : S with type configuration = string type remote_configuration = { cctxt : Dac_node_client.cctxt; page_store : Filesystem.t; + api_version : RPC_services.Api.version; } (** A [Page_store] implementation backed by the local filesystem, which @@ -114,12 +115,17 @@ module Internal_for_tests : sig type remote_context val fetch : - Dac_plugin.t -> remote_context -> Dac_plugin.hash -> bytes tzresult Lwt.t + Dac_plugin.t -> + remote_context -> + RPC_services.Api.version -> + Dac_plugin.hash -> + bytes tzresult Lwt.t end) (P : S) : S - with type configuration = R.remote_context * P.t - and type t = R.remote_context * P.t + with type configuration = + R.remote_context * P.t * RPC_services.Api.version + and type t = R.remote_context * P.t * RPC_services.Api.version end (** [ensure_reveal_data_dir_exists reveal_data_dir] checks that the diff --git a/src/lib_dac_node/signature_manager.ml b/src/lib_dac_node/signature_manager.ml index 8b35e08a5299..71bb3cb0360a 100644 --- a/src/lib_dac_node/signature_manager.ml +++ b/src/lib_dac_node/signature_manager.ml @@ -353,47 +353,51 @@ let stream_certificate_update dac_plugin committee_members let handle_put_dac_member_signature dac_plugin certificate_streamers_opt rw_node_store page_store cctxt committee_members committee_member_signature - = - let open Lwt_result_syntax in - let ((module Plugin) : Dac_plugin.t) = dac_plugin in - let Signature_repr.{root_hash = raw_root_hash; _} = - committee_member_signature - in - let*? root_hash = Dac_plugin.raw_to_hash dac_plugin raw_root_hash in - let* () = check_coordinator_knows_root_hash dac_plugin page_store root_hash in - let* should_update_certificate = - should_update_certificate - dac_plugin - cctxt - rw_node_store - committee_members - committee_member_signature - in - if should_update_certificate then - let* () = - add_dac_member_signature - dac_plugin - rw_node_store + api_version = + match api_version with + | RPC_services.Api.V0 | RPC_services.Api.V1 -> + let open Lwt_result_syntax in + let ((module Plugin) : Dac_plugin.t) = dac_plugin in + let Signature_repr.{root_hash = raw_root_hash; _} = committee_member_signature - in - let* aggregate_signature, witnesses = - update_aggregate_sig_store rw_node_store committee_members root_hash - in - let*? () = - Option.iter_e - (stream_certificate_update - dac_plugin - committee_members - Certificate_repr. - {root_hash = raw_root_hash; aggregate_signature; witnesses}) - certificate_streamers_opt - in - return () - else return () + in + let*? root_hash = Dac_plugin.raw_to_hash dac_plugin raw_root_hash in + let* () = + check_coordinator_knows_root_hash dac_plugin page_store root_hash + in + let* should_update_certificate = + should_update_certificate + dac_plugin + cctxt + rw_node_store + committee_members + committee_member_signature + in + if should_update_certificate then + let* () = + add_dac_member_signature + dac_plugin + rw_node_store + committee_member_signature + in + let* aggregate_signature, witnesses = + update_aggregate_sig_store rw_node_store committee_members root_hash + in + let*? () = + Option.iter_e + (stream_certificate_update + dac_plugin + committee_members + Certificate_repr. + {root_hash = raw_root_hash; aggregate_signature; witnesses}) + certificate_streamers_opt + in + return () + else return () module Coordinator = struct let handle_put_dac_member_signature ctx dac_plugin rw_node_store page_store - cctxt dac_member_signature = + cctxt dac_member_signature api_version = let committee_members = Node_context.Coordinator.committee_members ctx in handle_put_dac_member_signature dac_plugin @@ -403,6 +407,7 @@ module Coordinator = struct cctxt committee_members dac_member_signature + api_version end module Legacy = struct @@ -428,7 +433,7 @@ module Legacy = struct | Some signature -> return @@ (signature, witnesses)) let handle_put_dac_member_signature ctx dac_plugin rw_node_store page_store - cctxt dac_member_signature = + cctxt dac_member_signature api_version = let committee_members = Node_context.Legacy.committee_members ctx in handle_put_dac_member_signature dac_plugin @@ -438,4 +443,5 @@ module Legacy = struct cctxt committee_members dac_member_signature + api_version end diff --git a/src/lib_dac_node/signature_manager.mli b/src/lib_dac_node/signature_manager.mli index 9afa9f5bb76a..90b7e51756f8 100644 --- a/src/lib_dac_node/signature_manager.mli +++ b/src/lib_dac_node/signature_manager.mli @@ -71,6 +71,7 @@ module Coordinator : sig Page_store.Filesystem.t -> #Client_context.wallet -> Signature_repr.t -> + RPC_services.Api.version -> unit tzresult Lwt.t end @@ -103,5 +104,6 @@ module Legacy : sig Page_store.Filesystem.t -> #Client_context.wallet -> Signature_repr.t -> + RPC_services.Api.version -> unit tzresult Lwt.t end diff --git a/src/proto_017_PtNairob/lib_dac_plugin/test/test_dac_pages_encoding.ml b/src/proto_017_PtNairob/lib_dac_plugin/test/test_dac_pages_encoding.ml index a9483a551538..096081c44996 100644 --- a/src/proto_017_PtNairob/lib_dac_plugin/test/test_dac_pages_encoding.ml +++ b/src/proto_017_PtNairob/lib_dac_plugin/test/test_dac_pages_encoding.ml @@ -233,12 +233,14 @@ module With_hash_check : module Double_hash_map_backend : Page_store.S - with type configuration = Hashes_map_backend.t * Hashes_map_backend.t = + with type configuration = + Hashes_map_backend.t * Hashes_map_backend.t * RPC_services.Api.version = Page_store.Internal_for_tests.With_remote_fetch (struct type remote_context = Hashes_map_backend.t - let fetch = Hashes_map_backend.load + let fetch dac_plugin remote_context _api_version hash = + Hashes_map_backend.load dac_plugin remote_context hash end) (With_hash_check) @@ -396,7 +398,8 @@ module Merkle_tree = struct let mock_remote_store = Hashes_map_backend.init () in let mock_local_store = Hashes_map_backend.init () in let page_store = - Double_hash_map_backend.init (mock_remote_store, mock_local_store) + Double_hash_map_backend.init + (mock_remote_store, mock_local_store, RPC_services.Api.V0) in let payload = Bytes.of_string "This is a payload that will be tampered later on" diff --git a/src/proto_alpha/lib_dac_plugin/test/test_dac_pages_encoding.ml b/src/proto_alpha/lib_dac_plugin/test/test_dac_pages_encoding.ml index 2a6018905dec..458ad72880bb 100644 --- a/src/proto_alpha/lib_dac_plugin/test/test_dac_pages_encoding.ml +++ b/src/proto_alpha/lib_dac_plugin/test/test_dac_pages_encoding.ml @@ -233,12 +233,14 @@ module With_hash_check : module Double_hash_map_backend : Page_store.S - with type configuration = Hashes_map_backend.t * Hashes_map_backend.t = + with type configuration = + Hashes_map_backend.t * Hashes_map_backend.t * RPC_services.Api.version = Page_store.Internal_for_tests.With_remote_fetch (struct type remote_context = Hashes_map_backend.t - let fetch = Hashes_map_backend.load + let fetch dac_plugin remote_context _api_version hash = + Hashes_map_backend.load dac_plugin remote_context hash end) (With_hash_check) @@ -396,7 +398,8 @@ module Merkle_tree = struct let mock_remote_store = Hashes_map_backend.init () in let mock_local_store = Hashes_map_backend.init () in let page_store = - Double_hash_map_backend.init (mock_remote_store, mock_local_store) + Double_hash_map_backend.init + (mock_remote_store, mock_local_store, RPC_services.Api.V0) in let payload = Bytes.of_string "This is a payload that will be tampered later on" diff --git a/tezt/lib_tezos/dac_rpc.ml b/tezt/lib_tezos/dac_rpc.ml index 6314faed3f95..59527cb8e913 100644 --- a/tezt/lib_tezos/dac_rpc.ml +++ b/tezt/lib_tezos/dac_rpc.ml @@ -23,6 +23,17 @@ (* *) (*****************************************************************************) +(** [Api] module is used for versioning DAC API. *) +module Api = struct + (** [v0] is a version that will get deprecated once old legacy tests are + refactored. Do not use it!. *) + let v0 = "v0" + + (** [v1] is a version that corresponds to the first public release of the DAC + API. *) + let v1 = "v1" +end + let make ?data ?query_string = RPC.make ?data @@ -40,9 +51,10 @@ let decode_hex_string_to_bytes s = Hex.to_string (`Hex s) let get_bytes_from_json_string_node json = JSON.as_string json |> decode_hex_string_to_bytes -let get_preimage page_hash = make GET ["preimage"; page_hash] JSON.as_string +let get_preimage page_hash ~api_version = + make GET [api_version; "preimage"; page_hash] JSON.as_string -let post_store_preimage ~payload ~pagination_scheme = +let post_store_preimage ~payload ~pagination_scheme ~api_version = let preimage = JSON.parse ~origin:"dal_node_dac_store_preimage_rpc" @@ -52,18 +64,19 @@ let post_store_preimage ~payload ~pagination_scheme = pagination_scheme) in let data : RPC_core.data = Data (JSON.unannotate preimage) in - make ~data POST ["store_preimage"] @@ fun json -> + make ~data POST [api_version; "store_preimage"] @@ fun json -> JSON. ( json |-> "root_hash" |> as_string, json |-> "external_message" |> get_bytes_from_json_string_node ) -let get_verify_signature external_msg = +let get_verify_signature external_msg ~api_version = let query_string = [("external_message", match Hex.of_string external_msg with `Hex s -> s)] in - make ~query_string GET ["verify_signature"] JSON.as_bool + make ~query_string GET [api_version; "verify_signature"] JSON.as_bool -let put_dac_member_signature ~hex_root_hash ~dac_member_pkh ~signature = +let put_dac_member_signature ~hex_root_hash ~dac_member_pkh ~signature + ~api_version = let (`Hex root_hash) = hex_root_hash in let payload = `O @@ -75,26 +88,26 @@ let put_dac_member_signature ~hex_root_hash ~dac_member_pkh ~signature = ] in let data : RPC_core.data = Data payload in - make ~data PUT ["dac_member_signature"] @@ fun _resp -> () + make ~data PUT [api_version; "dac_member_signature"] @@ fun _resp -> () -let get_certificate ~hex_root_hash = +let get_certificate ~hex_root_hash ~api_version = let (`Hex page_hash) = hex_root_hash in - make GET ["certificates"; page_hash] @@ fun json -> + make GET [api_version; "certificates"; page_hash] @@ fun json -> JSON. ( json |-> "witnesses" |> as_int, json |-> "aggregate_signature" |> as_string, json |-> "root_hash" |> as_string ) -let get_missing_page ~hex_root_hash = - make GET ["missing_page"; Hex.show hex_root_hash] JSON.as_string +let get_missing_page ~hex_root_hash ~api_version = + make GET [api_version; "missing_page"; Hex.show hex_root_hash] JSON.as_string module Coordinator = struct - let post_preimage ~payload = + let post_preimage ~payload ~api_version = let preimage = JSON.parse ~origin:"Rollup.DAC.RPC.coordinator_post_preimage" (encode_bytes_to_hex_string payload) in let data : RPC_core.data = Data (JSON.unannotate preimage) in - make ~data POST ["preimage"] JSON.as_string + make ~data POST [api_version; "preimage"] JSON.as_string end diff --git a/tezt/lib_tezos/dac_rpc.mli b/tezt/lib_tezos/dac_rpc.mli index 652f605a64f4..406491982da9 100644 --- a/tezt/lib_tezos/dac_rpc.mli +++ b/tezt/lib_tezos/dac_rpc.mli @@ -23,50 +23,70 @@ (* *) (*****************************************************************************) -(** [get_preimage hash] requests the preimage of hash, consisting of a +(** [Api] module is used for versioning DAC API. *) +module Api : sig + (** [v0] is a version that will get deprecated once old legacy tests are + refactored. Do not use it!. *) + val v0 : string + + (** [v1] is a version that corresponds to the first public release of the DAC + API. *) + val v1 : string +end + +(** [get_preimage hash ~api_version] requests the preimage of hash, consisting of a single page, from cctxt. When the request succeeds, the raw page will be returned as a sequence of bytes. *) -val get_preimage : string -> (Dac_node.t, string) RPC_core.t +val get_preimage : + string -> api_version:string -> (Dac_node.t, string) RPC_core.t -(** [post_store_preimage cctxt ~payload ~pagination_scheme] posts a [payload] to dac/store_preimage - using a given [pagination_scheme]. It returns the base58 encoded root page hash - and the raw bytes that can be used as contents of a rollup message to trigger the request of - the payload in a WASM rollup. *) +(** [post_store_preimage cctxt ~payload ~pagination_scheme ~api_version] posts + a [payload] to dac/store_preimage using a given [pagination_scheme]. + It returns the base58 encoded root page hash and the raw bytes that can be + used as contents of a rollup message to trigger the request of the payload + in a WASM rollup. *) val post_store_preimage : payload:string -> pagination_scheme:string -> + api_version:string -> (Dac_node.t, string * string) RPC_core.t -(** [get_verify_signature cctxt external_message] requests the DAL node to verify - the signature of the external message [external_message] via +(** [get_verify_signature cctxt external_message ~api_version] requests the DAL + node to verify the signature of the external message [external_message] via the plugin/dac/verify_signature endpoint. The DAC committee of the DAL node must be the same that was used to produce the [external_message]. *) -val get_verify_signature : string -> (Dac_node.t, bool) RPC_core.t +val get_verify_signature : + string -> api_version:string -> (Dac_node.t, bool) RPC_core.t -(** [put_dac_member_signature hex_root_hash dac_member_pkh signature] +(** [put_dac_member_signature hex_root_hash dac_member_pkh signature ~api_version] stores the [signature] generated from signing [hex_root_hash] by [dac_member_pkh]. *) val put_dac_member_signature : hex_root_hash:Hex.t -> dac_member_pkh:string -> signature:Tezos_crypto.Aggregate_signature.t -> + api_version:string -> (Dac_node.t, unit) RPC_core.t -(** [get_certificate ~hex_root_hash] fetches the DAC certificate for the - provided [hex_root_hash]. *) +(** [get_certificate ~hex_root_hash ~api_version] fetches the DAC certificate + for the provided [hex_root_hash]. *) val get_certificate : - hex_root_hash:Hex.t -> (Dac_node.t, int * string * string) RPC_core.t + hex_root_hash:Hex.t -> + api_version:string -> + (Dac_node.t, int * string * string) RPC_core.t -(** [get_missing_page ~hex_root_hash] calls GET missing_page/[page_hash] - endpoint. *) -val get_missing_page : hex_root_hash:Hex.t -> (Dac_node.t, string) RPC_core.t +(** [get_missing_page ~hex_root_hash ~api_version] calls GET missing_page/[page_hash] + endpoint. *) +val get_missing_page : + hex_root_hash:Hex.t -> api_version:string -> (Dac_node.t, string) RPC_core.t module Coordinator : sig - (** [post_preimage ~payload] sends a [payload] to the DAC + (** [post_preimage ~payload ~api_version] sends a [payload] to the DAC [Coordinator] via a POST RPC call to dac/preimage. It returns a hex encoded root page hash, produced by [Merkle_tree_V0] pagination scheme. On the backend side it also pushes root page hash of the preimage to all the subscribed DAC Members and Observers. *) - val post_preimage : payload:string -> (Dac_node.t, string) RPC_core.t + val post_preimage : + payload:string -> api_version:string -> (Dac_node.t, string) RPC_core.t end diff --git a/tezt/tests/dac.ml b/tezt/tests/dac.ml index d3afe4f6cad1..321dd4464f7d 100644 --- a/tezt/tests/dac.ml +++ b/tezt/tests/dac.ml @@ -82,7 +82,11 @@ let init_hex_root_hash ?payload coordinator_node = let* root_hash, _l1_op = RPC.call coordinator_node - (Dac_rpc.post_store_preimage ~payload ~pagination_scheme:"Merkle_tree_V0") + Dac_rpc.( + post_store_preimage + ~payload + ~pagination_scheme:"Merkle_tree_V0" + ~api_version:Api.v0) in let hex_root_hash = `Hex root_hash in return hex_root_hash @@ -129,9 +133,10 @@ let parse_certificate json = let streamed_certificates_client coordinator_node root_hash = let endpoint = Format.sprintf - "http://%s:%d/monitor/certificate/%s" + "http://%s:%d/%s/monitor/certificate/%s" (Dac_node.rpc_host coordinator_node) (Dac_node.rpc_port coordinator_node) + Dac_rpc.Api.v1 root_hash in RPC.Curl.get_raw endpoint @@ -600,13 +605,14 @@ let check_preimage expected_preimage actual_preimage = [page_hash], if the page corresponds to a hash page. Otherwise, it returns the empty list. *) let check_downloaded_page coordinator observer page_hash = + (* TODO this function is called by both legacy and non legacy tests. *) let* coordinator_hex_encoded_page = - RPC.call coordinator (Dac_rpc.get_preimage page_hash) + RPC.call coordinator Dac_rpc.(get_preimage page_hash ~api_version:Api.v1) in let coordinator_page = Hex.to_string (`Hex coordinator_hex_encoded_page) in (* Check that the page has been saved by the observer. *) let* observer_hex_encoded_page = - RPC.call observer (Dac_rpc.get_preimage page_hash) + RPC.call observer Dac_rpc.(get_preimage page_hash ~api_version:Api.v1) in let observer_page = Hex.to_string (`Hex observer_hex_encoded_page) in (* Check that the raw page for the root hash stored in the coordinator @@ -642,6 +648,7 @@ let check_downloaded_page coordinator observer page_hash = in return @@ split_hashes concatenated_hashes [] +(* TODO called by both legacy and non legacy API. *) let check_downloaded_preimage coordinator observer root_hash = let rec go hashes = match hashes with @@ -713,9 +720,11 @@ module Legacy = struct let* actual_rh, _l1_operation = RPC.call coordinator - (Dac_rpc.post_store_preimage - ~payload - ~pagination_scheme:"Merkle_tree_V0") + Dac_rpc.( + post_store_preimage + ~payload + ~pagination_scheme:"Merkle_tree_V0" + ~api_version:Api.v0) in return @@ check_valid_root_hash expected_rh actual_rh @@ -833,9 +842,11 @@ module Legacy = struct let* actual_rh, l1_operation = RPC.call dac_node - (Dac_rpc.post_store_preimage - ~payload - ~pagination_scheme:"Merkle_tree_V0") + Dac_rpc.( + post_store_preimage + ~payload + ~pagination_scheme:"Merkle_tree_V0" + ~api_version:Api.v0) in (* Expected reveal hash equals to the result of [Tezos_dac_alpha.Dac_pages_encoding.Merkle_tree.V0.serialize_payload "test"]. @@ -858,7 +869,9 @@ module Legacy = struct in check_preimage payload recovered_preimage ; let* is_signature_valid = - RPC.call dac_node (Dac_rpc.get_verify_signature l1_operation) + RPC.call + dac_node + Dac_rpc.(get_verify_signature l1_operation ~api_version:Api.v0) in Check.( (is_signature_valid = true) @@ -873,9 +886,11 @@ module Legacy = struct let* actual_rh, _l1_operation = RPC.call dac_node - (Dac_rpc.post_store_preimage - ~payload - ~pagination_scheme:"Hash_chain_V0") + Dac_rpc.( + post_store_preimage + ~payload + ~pagination_scheme:"Hash_chain_V0" + ~api_version:Api.v0) in (* Expected reveal hash equals to the result of [Tezos_dac_alpha.Dac_pages_encoding.Hash_chain.V0.serialize_payload "test"]. @@ -905,9 +920,11 @@ module Legacy = struct let* actual_rh, _l1_operation = RPC.call dac_node - (Dac_rpc.post_store_preimage - ~payload - ~pagination_scheme:"Merkle_tree_V0") + Dac_rpc.( + post_store_preimage + ~payload + ~pagination_scheme:"Merkle_tree_V0" + ~api_version:Api.v0) in (* Expected reveal hash equals to the result of [Tezos_dac_alpha.Dac_pages_encoding.Merkle_tree.V0.serialize_payload "test"]. @@ -925,7 +942,9 @@ module Legacy = struct let recovered_payload = really_input_string cin (in_channel_length cin) in let () = close_in cin in let recovered_preimage = Hex.of_string recovered_payload in - let* preimage = RPC.call dac_node (Dac_rpc.get_preimage expected_rh) in + let* preimage = + RPC.call dac_node Dac_rpc.(get_preimage expected_rh ~api_version:Api.v0) + in Check.( (preimage = Hex.show recovered_preimage) string @@ -963,9 +982,11 @@ module Legacy = struct let* actual_rh, _l1_operation = RPC.call dac_node - (Dac_rpc.post_store_preimage - ~payload - ~pagination_scheme:"Hash_chain_V0") + Dac_rpc.( + post_store_preimage + ~payload + ~pagination_scheme:"Hash_chain_V0" + ~api_version:Api.v0) in let expected_rh = "0027782d2a7020be332cc42c4e66592ec50305f559a4011981f1d5af81428e7aa3" @@ -1005,9 +1026,11 @@ module Legacy = struct let _actual_rh = RPC.call dac_node - (Dac_rpc.post_store_preimage - ~payload - ~pagination_scheme:"Hash_chain_V0") + Dac_rpc.( + post_store_preimage + ~payload + ~pagination_scheme:"Hash_chain_V0" + ~api_version:Api.v0) in let errorneous_hash = "0027782d2a7020be332cc42c4e66592ec50305f559a4011981f1d5af81428ecafe" @@ -1303,17 +1326,25 @@ module Legacy = struct assert_lwt_failure ~__LOC__ "Expected retrieve_preimage" - (RPC.call observer (Dac_rpc.get_preimage (Hex.show hex_root_hash))) + (RPC.call + observer + Dac_rpc.(get_preimage (Hex.show hex_root_hash) ~api_version:Api.v0)) in let* missing_page = - RPC.call observer (Dac_rpc.get_missing_page ~hex_root_hash) + RPC.call + observer + Dac_rpc.(get_missing_page ~hex_root_hash ~api_version:Api.v0) in let* coordinator_page = - RPC.call coordinator (Dac_rpc.get_preimage (Hex.show hex_root_hash)) + RPC.call + coordinator + Dac_rpc.(get_preimage (Hex.show hex_root_hash) ~api_version:Api.v0) in check_preimage coordinator_page missing_page ; let* observer_preimage = - RPC.call observer (Dac_rpc.get_preimage (Hex.show hex_root_hash)) + RPC.call + observer + Dac_rpc.(get_preimage (Hex.show hex_root_hash) ~api_version:Api.v0) in check_preimage coordinator_page observer_preimage ; unit @@ -1328,10 +1359,12 @@ module Legacy = struct let result = RPC.call coordinator_node - (Dac_rpc.put_dac_member_signature - ~hex_root_hash - ~dac_member_pkh:invalid_signer_key.aggregate_public_key_hash - ~signature) + Dac_rpc.( + put_dac_member_signature + ~hex_root_hash + ~dac_member_pkh:invalid_signer_key.aggregate_public_key_hash + ~signature + ~api_version:Api.v0) in assert_lwt_failure ~__LOC__ @@ -1353,10 +1386,12 @@ module Legacy = struct let result = RPC.call coordinator_node - (Dac_rpc.put_dac_member_signature - ~hex_root_hash - ~dac_member_pkh:memberj.aggregate_public_key_hash - ~signature) + Dac_rpc.( + put_dac_member_signature + ~hex_root_hash + ~dac_member_pkh:memberj.aggregate_public_key_hash + ~signature + ~api_version:Api.v0) in assert_lwt_failure ~__LOC__ @@ -1384,17 +1419,21 @@ module Legacy = struct let* () = RPC.call coordinator_node - (Dac_rpc.put_dac_member_signature - ~hex_root_hash - ~dac_member_pkh:member.aggregate_public_key_hash - ~signature) + Dac_rpc.( + put_dac_member_signature + ~hex_root_hash + ~dac_member_pkh:member.aggregate_public_key_hash + ~signature + ~api_version:Api.v0) in return (member :: keys)) (return []) members in let* witnesses, certificate, _root_hash = - RPC.call coordinator_node (Dac_rpc.get_certificate ~hex_root_hash) + RPC.call + coordinator_node + Dac_rpc.(get_certificate ~hex_root_hash ~api_version:Api.v0) in assert_witnesses ~__LOC__ 3 witnesses ; assert_verify_aggregate_signature members_keys hex_root_hash certificate ; @@ -1412,15 +1451,19 @@ module Legacy = struct let call () = RPC.call coordinator_node - (Dac_rpc.put_dac_member_signature - ~hex_root_hash - ~dac_member_pkh - ~signature) + Dac_rpc.( + put_dac_member_signature + ~hex_root_hash + ~dac_member_pkh + ~signature + ~api_version:Api.v0) in let* () = call () in let* () = call () in let* witnesses, certificate, _root_hash = - RPC.call coordinator_node (Dac_rpc.get_certificate ~hex_root_hash) + RPC.call + coordinator_node + Dac_rpc.(get_certificate ~hex_root_hash ~api_version:Api.v0) in assert_witnesses ~__LOC__ 4 witnesses ; assert_verify_aggregate_signature [member] hex_root_hash certificate ; @@ -1439,10 +1482,12 @@ module Legacy = struct let result = RPC.call coordinator_node - (Dac_rpc.put_dac_member_signature - ~hex_root_hash:false_root_hash - ~dac_member_pkh - ~signature) + Dac_rpc.( + put_dac_member_signature + ~hex_root_hash:false_root_hash + ~dac_member_pkh + ~signature + ~api_version:Api.v0) in assert_lwt_failure ~__LOC__ @@ -1480,13 +1525,17 @@ module Legacy = struct let* () = RPC.call coordinator - (Dac_rpc.put_dac_member_signature - ~hex_root_hash - ~dac_member_pkh:member.aggregate_public_key_hash - ~signature) + Dac_rpc.( + put_dac_member_signature + ~hex_root_hash + ~dac_member_pkh:member.aggregate_public_key_hash + ~signature + ~api_version:Api.v0) in let* witnesses, certificate, _root_hash = - RPC.call coordinator (Dac_rpc.get_certificate ~hex_root_hash) + RPC.call + coordinator + Dac_rpc.(get_certificate ~hex_root_hash ~api_version:Api.v0) in let expected_witnesses = Z.shift_left Z.one i in assert_witnesses ~__LOC__ (Z.to_int expected_witnesses) witnesses ; @@ -1498,7 +1547,9 @@ end module Full_infrastructure = struct let coordinator_serializes_payload coordinator ~payload ~expected_rh = let* actual_rh = - RPC.call coordinator (Dac_rpc.Coordinator.post_preimage ~payload) + RPC.call + coordinator + Dac_rpc.(Coordinator.post_preimage ~payload ~api_version:Api.v1) in return @@ check_valid_root_hash expected_rh actual_rh @@ -1515,7 +1566,9 @@ module Full_infrastructure = struct wait_for_root_hash_pushed_to_data_streamer coordinator_node expected_rh in let* actual_rh = - RPC.call coordinator_node (Dac_rpc.Coordinator.post_preimage ~payload) + RPC.call + coordinator_node + Dac_rpc.(Coordinator.post_preimage ~payload ~api_version:Api.v1) in let () = check_valid_root_hash expected_rh actual_rh in let* () = root_hash_pushed_to_data_streamer_promise in @@ -1696,7 +1749,8 @@ module Full_infrastructure = struct let* get_certificate = RPC.call coordinator_node - (Dac_rpc.get_certificate ~hex_root_hash:(`Hex expected_rh)) + Dac_rpc.( + get_certificate ~hex_root_hash:(`Hex expected_rh) ~api_version:Api.v1) in check_certificate get_certificate last_certificate_update ; (* 6. Request certificate via streamed endpoints again, check that one -- GitLab From 120a3e0172f70712b29a365d6d3d86eff392ddae Mon Sep 17 00:00:00 2001 From: Martin Tomazic Date: Tue, 25 Apr 2023 12:41:15 +0200 Subject: [PATCH 06/13] DAC/tezt: Consistent use of API version within same test suites --- tezt/tests/dac.ml | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/tezt/tests/dac.ml b/tezt/tests/dac.ml index 321dd4464f7d..ae11ac18b60f 100644 --- a/tezt/tests/dac.ml +++ b/tezt/tests/dac.ml @@ -598,21 +598,21 @@ let check_preimage expected_preimage actual_preimage = ~error_msg: "Preimage does not match expected value (Current: %L <> Expected: %R)") -(** [check_downloaded_page coordinator observer page_hash] checks that the - [observer] has downloaded a page with [page_hash] from the [coordinator], - that the contents of the page corresponds to the ones of the - [coordinator]. It returns the list of the hashes contained in the - [page_hash], if the page corresponds to a hash page. Otherwise, it returns - the empty list. *) -let check_downloaded_page coordinator observer page_hash = +(** [check_downloaded_page coordinator observer page_hash api_version] checks + that the [observer] has downloaded a page with [page_hash] from the + [coordinator], that the contents of the page corresponds to the ones of the + [coordinator]. It returns the list of the hashes contained in the + [page_hash], if the page corresponds to a hash page. Otherwise, it returns + the empty list. *) +let check_downloaded_page coordinator observer page_hash ~api_version = (* TODO this function is called by both legacy and non legacy tests. *) let* coordinator_hex_encoded_page = - RPC.call coordinator Dac_rpc.(get_preimage page_hash ~api_version:Api.v1) + RPC.call coordinator Dac_rpc.(get_preimage page_hash ~api_version) in let coordinator_page = Hex.to_string (`Hex coordinator_hex_encoded_page) in (* Check that the page has been saved by the observer. *) let* observer_hex_encoded_page = - RPC.call observer Dac_rpc.(get_preimage page_hash ~api_version:Api.v1) + RPC.call observer Dac_rpc.(get_preimage page_hash ~api_version) in let observer_page = Hex.to_string (`Hex observer_hex_encoded_page) in (* Check that the raw page for the root hash stored in the coordinator @@ -648,13 +648,14 @@ let check_downloaded_page coordinator observer page_hash = in return @@ split_hashes concatenated_hashes [] -(* TODO called by both legacy and non legacy API. *) -let check_downloaded_preimage coordinator observer root_hash = +let check_downloaded_preimage coordinator observer root_hash ~api_version = let rec go hashes = match hashes with | [] -> return () | hash :: hashes -> - let* next_hashes = check_downloaded_page coordinator observer hash in + let* next_hashes = + check_downloaded_page coordinator observer hash ~api_version + in go (hashes @ next_hashes) in go [root_hash] @@ -1288,7 +1289,11 @@ module Legacy = struct let* () = push_promise in (* Assert [observer] emitted event of received [expected_rh]. *) let* () = fetch_root_hash_promise in - check_downloaded_preimage coordinator observer expected_rh + check_downloaded_preimage + coordinator + observer + expected_rh + ~api_version:Dac_rpc.Api.v0 (* 1. Observer should fetch missing page from Coordinator when GET /missing_page/{hash} is called. @@ -1651,7 +1656,11 @@ module Full_infrastructure = struct This might be inefficient *) Lwt_list.iter_s (fun dac_node -> - check_downloaded_preimage coordinator_node dac_node expected_rh) + check_downloaded_preimage + coordinator_node + dac_node + expected_rh + ~api_version:Dac_rpc.Api.v1) (committee_members_nodes @ observer_nodes) let test_streaming_certificates -- GitLab From ffe63a7adfacdb77a4291a94ee2db271e94a9ff5 Mon Sep 17 00:00:00 2001 From: Martin Tomazic Date: Wed, 26 Apr 2023 11:40:53 +0200 Subject: [PATCH 07/13] DAC: Add api_version to Node_context.t --- src/lib_dac_node/handler.ml | 41 ++++++++++++++++--------------- src/lib_dac_node/node_context.ml | 4 +++ src/lib_dac_node/node_context.mli | 6 +++++ 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/src/lib_dac_node/handler.ml b/src/lib_dac_node/handler.ml index a4ba3d03a19f..3ff36e246d82 100644 --- a/src/lib_dac_node/handler.ml +++ b/src/lib_dac_node/handler.ml @@ -136,7 +136,7 @@ let new_head ctxt = } *) module Committee_member = struct let push_payload_signature coordinator_cctxt wallet_cctxt committee_member - root_hash = + root_hash api_version = let open Lwt_result_syntax in let signer_pkh = committee_member.Wallet_account.Committee_member.public_key_hash @@ -157,12 +157,12 @@ module Committee_member = struct Dac_node_client.put_dac_member_signature coordinator_cctxt ~signature:signature_repr - RPC_services.Api.V1 + api_version in let*! () = Event.emit_signature_pushed_to_coordinator signature in return_unit - let new_root_hash ctxt wallet_cctxt dac_plugin page_store = + let new_root_hash ctxt wallet_cctxt dac_plugin page_store api_version = let open Lwt_result_syntax in let coordinator_cctxt = ctxt.Node_context.Committee_member.coordinator_cctxt @@ -189,6 +189,7 @@ module Committee_member = struct wallet_cctxt committee_member root_hash + api_version | Error errs -> (* TODO: https://gitlab.com/tezos/tezos/-/issues/4930. Improve handling of errors. *) @@ -199,17 +200,12 @@ module Committee_member = struct in let remote_store = Page_store.( - Remote.init - { - cctxt = coordinator_cctxt; - page_store; - api_version = RPC_services.Api.V1; - }) + Remote.init {cctxt = coordinator_cctxt; page_store; api_version}) in let*! () = Event.(emit subscribed_to_root_hashes_stream ()) in make_stream_daemon (handler dac_plugin remote_store) - (Monitor_services.root_hashes coordinator_cctxt RPC_services.Api.V1) + (Monitor_services.root_hashes coordinator_cctxt api_version) end (** Handlers specific to an [Observer]. An [Observer] is responsible for @@ -220,7 +216,7 @@ end } *) module Observer = struct - let new_root_hash ctxt dac_plugin page_store = + let new_root_hash ctxt dac_plugin page_store api_version = let open Lwt_result_syntax in let coordinator_cctxt = ctxt.Node_context.Observer.coordinator_cctxt in let handler dac_plugin remote_store _stopper root_hash = @@ -248,17 +244,12 @@ module Observer = struct in let remote_store = Page_store.( - Remote.init - { - cctxt = coordinator_cctxt; - page_store; - api_version = RPC_services.Api.V1; - }) + Remote.init {cctxt = coordinator_cctxt; page_store; api_version}) in let*! () = Event.(emit subscribed_to_root_hashes_stream ()) in make_stream_daemon (handler dac_plugin remote_store) - (Monitor_services.root_hashes coordinator_cctxt RPC_services.Api.V1) + (Monitor_services.root_hashes coordinator_cctxt api_version) end (** Handlers specific to a [Legacy] DAC node. If no @@ -382,16 +373,26 @@ let handlers node_ctxt = let*? plugin = Node_context.get_dac_plugin node_ctxt in let page_store = Node_context.get_page_store node_ctxt in let wallet_cctxt = Node_context.get_tezos_node_cctxt node_ctxt in + let api_version = Node_context.get_api_version node_ctxt in match Node_context.get_mode node_ctxt with | Coordinator _ -> return [new_head node_ctxt] | Committee_member ctxt -> return [ new_head node_ctxt; - Committee_member.new_root_hash ctxt wallet_cctxt plugin page_store; + Committee_member.new_root_hash + ctxt + wallet_cctxt + plugin + page_store + api_version; ] | Observer ctxt -> - return [new_head node_ctxt; Observer.new_root_hash ctxt plugin page_store] + return + [ + new_head node_ctxt; + Observer.new_root_hash ctxt plugin page_store api_version; + ] | Legacy ctxt -> let coordinator_cctxt_opt = ctxt.Node_context.Legacy.coordinator_cctxt in let root_hash_handler = diff --git a/src/lib_dac_node/node_context.ml b/src/lib_dac_node/node_context.ml index 15a2306d9c6d..f3981e362afb 100644 --- a/src/lib_dac_node/node_context.ml +++ b/src/lib_dac_node/node_context.ml @@ -225,6 +225,7 @@ type t = { page_store : Page_store.Filesystem.t; node_store : Store_sigs.rw Store.Irmin_store.t; mode : mode; + api_version : RPC_services.Api.version; } let init_mode Configuration.{mode; _} cctxt = @@ -259,6 +260,7 @@ let init config cctxt = Page_store.Filesystem.init (Configuration.reveal_data_dir config); node_store; mode; + api_version = RPC_services.Api.V1; } let get_mode node_ctxt = node_ctxt.mode @@ -312,3 +314,5 @@ let get_node_store (type a) ctxt (access_mode : a Store_sigs.mode) : match access_mode with | Store_sigs.Read_only -> Store.Irmin_store.readonly ctxt.node_store | Store_sigs.Read_write -> ctxt.node_store + +let get_api_version ctxt = ctxt.api_version diff --git a/src/lib_dac_node/node_context.mli b/src/lib_dac_node/node_context.mli index bcd6a697c66d..f23cb569c536 100644 --- a/src/lib_dac_node/node_context.mli +++ b/src/lib_dac_node/node_context.mli @@ -180,3 +180,9 @@ val get_page_store : t -> Page_store.Filesystem.t (** [get_node_store ctxt access_mode] returns the [Store.Irmin_store.t] with [access_mode] used by Dac components. *) val get_node_store : t -> 'a Store_sigs.mode -> 'a Store.Irmin_store.t + +(** [get_api_version] returns the api version that the node is running in. + [api_version] corresponds to the latest api version supported by the + node and is also used for interacting with other nodes APIs. The exception + being [Legacy] mode, where the node always uses [API.V0]. *) +val get_api_version : t -> RPC_services.Api.version -- GitLab From 320c3a375fae9b142dd1a17c04fcbc5a07a86dcc Mon Sep 17 00:00:00 2001 From: Martin Tomazic Date: Thu, 27 Apr 2023 10:38:13 +0200 Subject: [PATCH 08/13] Fixup: Split Dac_node_client into V0 and V1 module --- src/lib_dac_client/dac_node_client.ml | 128 ++++++++++++++++--------- src/lib_dac_client/dac_node_client.mli | 125 ++++++++++++++---------- src/lib_dac_node/handler.ml | 8 +- src/lib_dac_node/page_store.ml | 5 +- 4 files changed, 162 insertions(+), 104 deletions(-) diff --git a/src/lib_dac_client/dac_node_client.ml b/src/lib_dac_client/dac_node_client.ml index 6335f2380d6f..ff502537e190 100644 --- a/src/lib_dac_client/dac_node_client.ml +++ b/src/lib_dac_client/dac_node_client.ml @@ -44,53 +44,89 @@ let make_unix_cctxt ~scheme ~host ~port = in new unix_cctxt ~rpc_config -(* FIXME: https://gitlab.com/tezos/tezos/-/issues/4895 - If the preimage was generated using a different plugin, the computation of - the hash might fail. In practice it would be better to retrieve the - hash of the protocol that the coordinator was using when the page hash - was computed. -*) -let get_preimage (cctxt : #cctxt) api_version ~page_hash = - cctxt#call_service - RPC_services.get_preimage - (((), api_version), page_hash) - () - () - -let post_store_preimage (cctxt : #cctxt) ~payload ~pagination_scheme api_version - = - cctxt#call_service - RPC_services.post_store_preimage - ((), api_version) - () - (payload, pagination_scheme) - -let get_verify_signature (cctxt : #cctxt) ~external_message api_version = - cctxt#call_service - RPC_services.get_verify_signature - ((), api_version) - external_message - () - -let put_dac_member_signature (cctxt : #cctxt) ~signature api_version = - cctxt#call_service - RPC_services.put_dac_member_signature - ((), api_version) - () - signature - -let get_certificate (cctxt : #cctxt) ~root_page_hash api_version = - cctxt#call_service - RPC_services.get_certificate - (((), api_version), root_page_hash) - () - () - -module Coordinator = struct - let post_preimage (cctxt : #cctxt) ~payload api_version = +module Shared = struct + (* FIXME: https://gitlab.com/tezos/tezos/-/issues/4895 + If the preimage was generated using a different plugin, the computation of + the hash might fail. In practice it would be better to retrieve the + hash of the protocol that the coordinator was using when the page hash + was computed. + *) + let get_preimage (cctxt : #cctxt) api_version ~page_hash = cctxt#call_service - RPC_services.Coordinator.post_preimage + RPC_services.get_preimage + (((), api_version), page_hash) + () + () + + let post_store_preimage (cctxt : #cctxt) ~payload ~pagination_scheme + api_version = + cctxt#call_service + RPC_services.post_store_preimage + ((), api_version) + () + (payload, pagination_scheme) + + let get_verify_signature (cctxt : #cctxt) ~external_message api_version = + cctxt#call_service + RPC_services.get_verify_signature ((), api_version) + external_message + () + + let put_dac_member_signature (cctxt : #cctxt) ~signature api_version = + cctxt#call_service + RPC_services.put_dac_member_signature + ((), api_version) + () + signature + + let get_certificate (cctxt : #cctxt) ~root_page_hash api_version = + cctxt#call_service + RPC_services.get_certificate + (((), api_version), root_page_hash) + () () - payload + + module Coordinator = struct + let post_preimage (cctxt : #cctxt) ~payload api_version = + cctxt#call_service + RPC_services.Coordinator.post_preimage + ((), api_version) + () + payload + end +end + +module V0 = struct + let get_preimage cctxt = Shared.get_preimage cctxt RPC_services.Api.V0 + + let put_dac_member_signature cctxt = + Shared.put_dac_member_signature cctxt RPC_services.Api.V0 + + let get_certificate cctxt = Shared.get_certificate cctxt RPC_services.Api.V0 + + module Coordinator = struct + let post_preimage cctxt = + Shared.Coordinator.post_preimage cctxt RPC_services.Api.V0 + end + + let post_store_preimage cctxt = + Shared.post_store_preimage cctxt RPC_services.Api.V0 + + let get_verify_signature cctxt = + Shared.get_verify_signature cctxt RPC_services.Api.V0 +end + +module V1 = struct + let get_preimage cctxt = Shared.get_preimage cctxt RPC_services.Api.V1 + + let put_dac_member_signature cctxt = + Shared.put_dac_member_signature cctxt RPC_services.Api.V1 + + let get_certificate cctxt = Shared.get_certificate cctxt RPC_services.Api.V1 + + module Coordinator = struct + let post_preimage cctxt = + Shared.Coordinator.post_preimage cctxt RPC_services.Api.V1 + end end diff --git a/src/lib_dac_client/dac_node_client.mli b/src/lib_dac_client/dac_node_client.mli index ce6ee72f55ff..20716e864cec 100644 --- a/src/lib_dac_client/dac_node_client.mli +++ b/src/lib_dac_client/dac_node_client.mli @@ -39,62 +39,83 @@ class unix_cctxt : the client configuration parameters. *) val make_unix_cctxt : scheme:string -> host:string -> port:int -> cctxt -(** [get_preimage cctxt api_version ~hash] requests the preimage of hash, - consisting of a single page, from cctxt. When the request succeeds, - the raw page will be returned as a sequence of bytes. *) -val get_preimage : - #cctxt -> - RPC_services.Api.version -> - page_hash:Dac_plugin.raw_hash -> - bytes tzresult Lwt.t +(** [V0] is a module that provides a client specification for interacting with + [Tezos_lib_dac.Rpc_services.Api.V0] API. *) +module V0 : sig + (** [get_preimage cctxt ~hash] requests the preimage of hash, consisting of a + single page, from cctxt. When the request succeeds, the raw page will be + returned as a sequence of bytes. *) + val get_preimage : + #cctxt -> page_hash:Dac_plugin.raw_hash -> bytes tzresult Lwt.t -(** [post_store_preimage cctxt ~payload ~pagination_scheme api_version] posts a - [payload] to dac/store_preimage using a given [pagination_scheme]. - It returns the base58 encoded root page hash and the raw bytes. *) -val post_store_preimage : - #cctxt -> - payload:bytes -> - pagination_scheme:Pagination_scheme.t -> - RPC_services.Api.version -> - (Dac_plugin.raw_hash * bytes) tzresult Lwt.t + (** [post_store_preimage cctxt ~payload ~pagination_scheme] posts + a [payload] to dac/store_preimage using a given [pagination_scheme]. + It returns the base58 encoded root page hash and the raw bytes. *) + val post_store_preimage : + #cctxt -> + payload:bytes -> + pagination_scheme:Pagination_scheme.t -> + (Dac_plugin.raw_hash * bytes) tzresult Lwt.t + + (** [get_verify_signature cctxt ~external_message] requests the + DAC node to verify the signature of the external message + [external_message] via the plugin/dac/verify_signature endpoint. + The DAC committee of the DAL node must be the same that was used to + produce the [external_message]. *) + val get_verify_signature : + #cctxt -> external_message:string option -> bool tzresult Lwt.t -(** [get_verify_signature cctxt ~external_message api_version] requests the DAL - node to verify the signature of the external message [external_message] via - the plugin/dac/verify_signature endpoint. The DAC committee - of the DAL node must be the same that was used to produce the - [external_message]. *) -val get_verify_signature : - #cctxt -> - external_message:string option -> - RPC_services.Api.version -> - bool tzresult Lwt.t + (** [put_dac_member_signature cctxt ~signature:Signature_repr.t] stores the + [signature] generated from signing [hex_root_hash] by [dac_member_pkh]. *) + val put_dac_member_signature : + #cctxt -> signature:Signature_repr.t -> unit tzresult Lwt.t -(** [put_dac_member_signature cctxt ~signature:Signature_repr.t api_version] - stores the [signature] generated from signing [hex_root_hash] by - [dac_member_pkh]. *) -val put_dac_member_signature : - #cctxt -> - signature:Signature_repr.t -> - RPC_services.Api.version -> - unit tzresult Lwt.t + (** [get_certificate cctxt ~root_page_hash] fetches the DAC certificate + for the provided [root_page_hash]. *) + val get_certificate : + #cctxt -> + root_page_hash:Dac_plugin.raw_hash -> + Certificate_repr.t option tzresult Lwt.t -(** [get_certificate cctxt ~root_page_hash api_version] fetches the DAC certificate for the - provided [root_page_hash]. *) -val get_certificate : - #cctxt -> - root_page_hash:Dac_plugin.raw_hash -> - RPC_services.Api.version -> - Certificate_repr.t option tzresult Lwt.t + module Coordinator : sig + (** [post_preimage cctxt ~payload] sends a [payload] to the DAC + [Coordinator] via a POST RPC call to dac/preimage. It returns a hex + encoded root page hash, produced by [Merkle_tree_V0] pagination scheme. + On the backend side it also pushes root page hash of the preimage to all + the subscribed DAC Members and Observers. *) + val post_preimage : + #cctxt -> payload:bytes -> Dac_plugin.raw_hash tzresult Lwt.t + end +end -module Coordinator : sig - (** [post_preimage cctxt ~payload api_version] sends a [payload] to the DAC - [Coordinator] via a POST RPC call to dac/preimage. It returns a hex - encoded root page hash, produced by [Merkle_tree_V0] pagination scheme. - On the backend side it also pushes root page hash of the preimage to all - the subscribed DAC Members and Observers. *) - val post_preimage : +(** [V1] is a module that provides a client specification for interacting with + [Tezos_lib_dac.Rpc_services.Api.V1] API. *) +module V1 : sig + (** [get_preimage cctxt ~hash] requests the preimage of hash, consisting of a + single page, from cctxt. When the request succeeds, the raw page will be + returned as a sequence of bytes. *) + val get_preimage : + #cctxt -> page_hash:Dac_plugin.raw_hash -> bytes tzresult Lwt.t + + (** [put_dac_member_signature cctxt ~signature:Signature_repr.t] stores the + [signature] generated from signing [hex_root_hash] by [dac_member_pkh]. *) + val put_dac_member_signature : + #cctxt -> signature:Signature_repr.t -> unit tzresult Lwt.t + + (** [get_certificate cctxt ~root_page_hash] fetches the DAC certificate + for the provided [root_page_hash]. *) + val get_certificate : #cctxt -> - payload:bytes -> - RPC_services.Api.version -> - Dac_plugin.raw_hash tzresult Lwt.t + root_page_hash:Dac_plugin.raw_hash -> + Certificate_repr.t option tzresult Lwt.t + + module Coordinator : sig + (** [post_preimage cctxt ~payload] sends a [payload] to the DAC + [Coordinator] via a POST RPC call to dac/preimage. It returns a hex + encoded root page hash, produced by [Merkle_tree_V0] pagination scheme. + On the backend side it also pushes root page hash of the preimage to all + the subscribed DAC Members and Observers. *) + val post_preimage : + #cctxt -> payload:bytes -> Dac_plugin.raw_hash tzresult Lwt.t + end end diff --git a/src/lib_dac_node/handler.ml b/src/lib_dac_node/handler.ml index 3ff36e246d82..953fd8aad7ba 100644 --- a/src/lib_dac_node/handler.ml +++ b/src/lib_dac_node/handler.ml @@ -154,10 +154,11 @@ module Committee_member = struct {root_hash = Dac_plugin.hash_to_raw root_hash; signature; signer_pkh} in let* () = - Dac_node_client.put_dac_member_signature + (match api_version with + | RPC_services.Api.V0 -> Dac_node_client.V0.put_dac_member_signature + | RPC_services.Api.V1 -> Dac_node_client.V1.put_dac_member_signature) coordinator_cctxt ~signature:signature_repr - api_version in let*! () = Event.emit_signature_pushed_to_coordinator signature in return_unit @@ -299,10 +300,9 @@ module Legacy = struct } in let* () = - Dac_node_client.put_dac_member_signature + Dac_node_client.V0.put_dac_member_signature coordinator_cctxt ~signature:signature_repr - RPC_services.Api.V0 in let*! () = Event.emit_signature_pushed_to_coordinator signature in return_unit diff --git a/src/lib_dac_node/page_store.ml b/src/lib_dac_node/page_store.ml index c5b9ab2da33d..2e781ca0fcf5 100644 --- a/src/lib_dac_node/page_store.ml +++ b/src/lib_dac_node/page_store.ml @@ -285,9 +285,10 @@ module Remote : S with type configuration = remote_configuration = struct type remote_context = Dac_node_client.cctxt let fetch _dac_plugin remote_context api_version hash = - Dac_node_client.get_preimage + (match api_version with + | RPC_services.Api.V0 -> Dac_node_client.V0.get_preimage + | RPC_services.Api.V1 -> Dac_node_client.V1.get_preimage) remote_context - api_version ~page_hash:(Dac_plugin.hash_to_raw hash) end) (F) -- GitLab From e235e8e06eaabeb7f45a8b2f853da83c05101d57 Mon Sep 17 00:00:00 2001 From: Martin Tomazic Date: Fri, 28 Apr 2023 15:29:30 +0200 Subject: [PATCH 09/13] DAC: Enforce `legacy` mode does not support `v1` endpoints --- src/lib_dac_node/RPC_server.ml | 124 +++++++++++++++++--------- src/lib_dac_node/signature_manager.ml | 7 ++ 2 files changed, 90 insertions(+), 41 deletions(-) diff --git a/src/lib_dac_node/RPC_server.ml b/src/lib_dac_node/RPC_server.ml index 4690a2ee9b89..4d399f5eaa06 100644 --- a/src/lib_dac_node/RPC_server.ml +++ b/src/lib_dac_node/RPC_server.ml @@ -132,7 +132,17 @@ let handle_get_verify_signature dac_plugin public_keys_opt encoded_l1_message witnesses) | RPC_services.Api.V1 -> raise Not_found -let handle_get_preimage dac_plugin page_store api_version raw_hash = +let assert_legacy_mode_cannot_run_v1 node_ctxt api_version = + match api_version with + | RPC_services.Api.V0 -> Result_syntax.return_unit + | RPC_services.Api.V1 -> ( + match Node_context.get_mode node_ctxt with + | Node_context.Legacy _ -> raise Not_found + | _ -> Result_syntax.return_unit) + +let handle_get_preimage node_ctxt dac_plugin page_store api_version raw_hash = + let open Lwt_result_syntax in + let*? () = assert_legacy_mode_cannot_run_v1 node_ctxt api_version in match api_version with | RPC_services.Api.V0 | RPC_services.Api.V1 -> let open Lwt_result_syntax in @@ -141,22 +151,25 @@ let handle_get_preimage dac_plugin page_store api_version raw_hash = (* Handler for subscribing to the streaming of root hashes via GET monitor/root_hashes RPC call. *) -let handle_monitor_root_hashes hash_streamer api_version = +let handle_monitor_root_hashes node_ctxt hash_streamer api_version = + let open Lwt_result_syntax in + let*? () = assert_legacy_mode_cannot_run_v1 node_ctxt api_version in match api_version with | RPC_services.Api.V0 | RPC_services.Api.V1 -> - let open Lwt_syntax in let stream, stopper = Data_streamer.handle_subscribe hash_streamer in let shutdown () = Lwt_watcher.shutdown stopper in let next () = Lwt_stream.get stream in - let* () = Event.(emit handle_new_subscription_to_hash_streamer ()) in - Tezos_rpc.Answer.return_stream {next; shutdown} + let*! () = Event.(emit handle_new_subscription_to_hash_streamer ()) in + return (next, shutdown) -let handle_get_certificate dac_plugin node_store raw_root_hash api_version = +let handle_get_certificate dac_plugin ctx raw_root_hash api_version = + let open Lwt_result_syntax in + let*? () = assert_legacy_mode_cannot_run_v1 ctx api_version in match api_version with | RPC_services.Api.V0 | RPC_services.Api.V1 -> let open Lwt_result_syntax in let*? root_hash = Dac_plugin.raw_to_hash dac_plugin raw_root_hash in - + let node_store = Node_context.get_node_store ctx Store_sigs.Read_write in let+ value_opt = Store.Certificate_store.find node_store root_hash in Option.map (fun Store.{aggregate_signature; witnesses} -> @@ -164,20 +177,23 @@ let handle_get_certificate dac_plugin node_store raw_root_hash api_version = {aggregate_signature; witnesses; root_hash = raw_root_hash}) value_opt -let handle_get_missing_page cctxt page_store dac_plugin raw_root_hash +let handle_get_missing_page node_ctxt cctxt page_store dac_plugin raw_root_hash api_version = let open Lwt_result_syntax in + let*? () = assert_legacy_mode_cannot_run_v1 node_ctxt api_version in let*? root_hash = Dac_plugin.raw_to_hash dac_plugin raw_root_hash in - let remote_store = - Page_store.Remote.(init {cctxt; page_store; api_version}) - in - let* preimage = - (* TODO: https://gitlab.com/tezos/tezos/-/issues/5142 - Retrieve missing page from dac committee via "flooding". *) - Page_store.Remote.load dac_plugin remote_store root_hash - in - let*! () = Event.(emit fetched_missing_page raw_root_hash) in - return preimage + match api_version with + | RPC_services.Api.V0 | RPC_services.Api.V1 -> + let remote_store = + Page_store.Remote.(init {cctxt; page_store; api_version}) + in + let* preimage = + (* TODO: https://gitlab.com/tezos/tezos/-/issues/5142 + Retrieve missing page from dac committee via "flooding". *) + Page_store.Remote.load dac_plugin remote_store root_hash + in + let*! () = Event.(emit fetched_missing_page raw_root_hash) in + return preimage let register_post_store_preimage ctx cctxt dac_sk_uris page_store hash_streamer directory = @@ -207,19 +223,25 @@ let register_get_verify_signature dac_plugin public_keys_opt directory = external_message api_version) -let register_get_preimage dac_plugin page_store = +let register_get_preimage node_ctxt dac_plugin page_store = add_service Tezos_rpc.Directory.register2 RPC_services.get_preimage (fun api_version hash () () -> - handle_get_preimage dac_plugin page_store api_version hash) + handle_get_preimage node_ctxt dac_plugin page_store api_version hash) -let register_monitor_root_hashes hash_streamer dir = +let register_monitor_root_hashes node_ctxt hash_streamer dir = Tezos_rpc.Directory.gen_register dir Monitor_services.S.root_hashes (fun ((), api_version) () () -> - handle_monitor_root_hashes hash_streamer api_version) + let open Lwt_result_syntax in + let*! handler = + handle_monitor_root_hashes node_ctxt hash_streamer api_version + in + match handler with + | Ok (next, shutdown) -> Tezos_rpc.Answer.return_stream {next; shutdown} + | Error e -> Tezos_rpc.Answer.fail e) let register_get_certificate node_store dac_plugin = add_service @@ -228,12 +250,18 @@ let register_get_certificate node_store dac_plugin = (fun api_version root_hash () () -> handle_get_certificate dac_plugin node_store root_hash api_version) -let register_get_missing_page dac_plugin page_store cctxt = +let register_get_missing_page node_ctxt dac_plugin page_store cctxt = add_service Tezos_rpc.Directory.register2 RPC_services.get_missing_page (fun api_version root_hash () () -> - handle_get_missing_page cctxt page_store dac_plugin root_hash api_version) + handle_get_missing_page + node_ctxt + cctxt + page_store + dac_plugin + root_hash + api_version) module Coordinator = struct let handle_post_preimage dac_plugin page_store hash_streamer payload @@ -363,8 +391,8 @@ module Coordinator = struct dac_member_signature api_version) - let dynamic_rpc_dir dac_plugin rw_store page_store cctxt coordinator_node_ctxt - = + let dynamic_rpc_dir node_ctxt dac_plugin rw_store page_store cctxt + coordinator_node_ctxt = let hash_streamer = coordinator_node_ctxt.Node_context.Coordinator.hash_streamer in @@ -372,8 +400,8 @@ module Coordinator = struct let committee_members = coordinator_node_ctxt.committee_members in Tezos_rpc.Directory.empty |> register_post_preimage dac_plugin hash_streamer page_store - |> register_get_preimage dac_plugin page_store - |> register_monitor_root_hashes hash_streamer + |> register_get_preimage node_ctxt dac_plugin page_store + |> register_monitor_root_hashes node_ctxt hash_streamer |> register_monitor_certificate dac_plugin rw_store @@ -385,19 +413,25 @@ module Coordinator = struct rw_store page_store cctxt - |> register_get_certificate rw_store dac_plugin + |> register_get_certificate node_ctxt dac_plugin end module Committee_member = struct - let dynamic_rpc_dir dac_plugin page_store = - Tezos_rpc.Directory.empty |> register_get_preimage dac_plugin page_store + let dynamic_rpc_dir dac_plugin node_ctxt = + let page_store = Node_context.get_page_store node_ctxt in + Tezos_rpc.Directory.empty + |> register_get_preimage node_ctxt dac_plugin page_store end module Observer = struct - let dynamic_rpc_dir dac_plugin coordinator_cctxt page_store = + let dynamic_rpc_dir node_ctxt dac_plugin coordinator_cctxt page_store = Tezos_rpc.Directory.empty - |> register_get_preimage dac_plugin page_store - |> register_get_missing_page dac_plugin page_store coordinator_cctxt + |> register_get_preimage node_ctxt dac_plugin page_store + |> register_get_missing_page + node_ctxt + dac_plugin + page_store + coordinator_cctxt end module Legacy = struct @@ -416,7 +450,8 @@ module Legacy = struct dac_member_signature api_version) - let dynamic_rpc_dir dac_plugin rw_store page_store cctxt legacy_node_ctxt = + let dynamic_rpc_dir node_ctxt dac_plugin rw_store page_store cctxt + legacy_node_ctxt = let hash_streamer = legacy_node_ctxt.Node_context.Legacy.hash_streamer in let public_keys_opt = Node_context.Legacy.public_keys_opt legacy_node_ctxt @@ -429,7 +464,8 @@ module Legacy = struct | None -> fun dir -> dir | Some cctxt -> fun dir -> - dir |> register_get_missing_page dac_plugin page_store cctxt + dir + |> register_get_missing_page node_ctxt dac_plugin page_store cctxt in Tezos_rpc.Directory.empty |> register_post_store_preimage @@ -439,15 +475,15 @@ module Legacy = struct page_store hash_streamer |> register_get_verify_signature dac_plugin public_keys_opt - |> register_get_preimage dac_plugin page_store - |> register_monitor_root_hashes hash_streamer + |> register_get_preimage node_ctxt dac_plugin page_store + |> register_monitor_root_hashes node_ctxt hash_streamer |> register_put_dac_member_signature legacy_node_ctxt dac_plugin rw_store page_store cctxt - |> register_get_certificate rw_store dac_plugin + |> register_get_certificate node_ctxt dac_plugin |> register_get_missing_page end @@ -460,17 +496,23 @@ let start ~rpc_address ~rpc_port node_ctxt = match Node_context.get_mode node_ctxt with | Coordinator coordinator_node_ctxt -> Coordinator.dynamic_rpc_dir + node_ctxt dac_plugin rw_store page_store cctxt coordinator_node_ctxt | Committee_member _committee_member_node_ctxt -> - Committee_member.dynamic_rpc_dir dac_plugin page_store + Committee_member.dynamic_rpc_dir dac_plugin node_ctxt | Observer {coordinator_cctxt; _} -> - Observer.dynamic_rpc_dir dac_plugin coordinator_cctxt page_store + Observer.dynamic_rpc_dir + node_ctxt + dac_plugin + coordinator_cctxt + page_store | Legacy legacy_node_ctxt -> Legacy.dynamic_rpc_dir + node_ctxt dac_plugin rw_store page_store diff --git a/src/lib_dac_node/signature_manager.ml b/src/lib_dac_node/signature_manager.ml index 71bb3cb0360a..5c119cb40b86 100644 --- a/src/lib_dac_node/signature_manager.ml +++ b/src/lib_dac_node/signature_manager.ml @@ -434,6 +434,13 @@ module Legacy = struct let handle_put_dac_member_signature ctx dac_plugin rw_node_store page_store cctxt dac_member_signature api_version = + let assert_legacy_mode_cannot_run_v1 api_version = + match api_version with + | RPC_services.Api.V0 -> Result_syntax.return_unit + | RPC_services.Api.V1 -> raise Not_found + in + let open Lwt_result_syntax in + let*? () = assert_legacy_mode_cannot_run_v1 api_version in let committee_members = Node_context.Legacy.committee_members ctx in handle_put_dac_member_signature dac_plugin -- GitLab From f0f9bb3d1d1bcd4a34d8b0f2e1ee9d923c32c52e Mon Sep 17 00:00:00 2001 From: Martin Tomazic Date: Fri, 28 Apr 2023 19:51:59 +0200 Subject: [PATCH 10/13] DAC/tezt: Test legacy mode does not support v1 API --- tezt/tests/dac.ml | 124 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 120 insertions(+), 4 deletions(-) diff --git a/tezt/tests/dac.ml b/tezt/tests/dac.ml index ae11ac18b60f..2264b32e957b 100644 --- a/tezt/tests/dac.ml +++ b/tezt/tests/dac.ml @@ -130,13 +130,13 @@ let parse_certificate json = (* Helper process that listens to certificate updates through a RPC request. Upon termination, the list of certificate updates is returned *) -let streamed_certificates_client coordinator_node root_hash = +let streamed_certificates_client coordinator_node root_hash api_version = let endpoint = Format.sprintf "http://%s:%d/%s/monitor/certificate/%s" (Dac_node.rpc_host coordinator_node) (Dac_node.rpc_port coordinator_node) - Dac_rpc.Api.v1 + api_version root_hash in RPC.Curl.get_raw endpoint @@ -1547,6 +1547,107 @@ module Legacy = struct assert_verify_aggregate_signature [member] hex_root_hash certificate ; unit end + + module Api_versioning = struct + (** [test_legacy_mode_does_not_support_v1] tests that dac node running in + the [Legacy] mode does not support [Api.v1]. *) + let test_legacy_mode_does_not_support_v1 _protocol node client coordinator + threshold dac_committee = + (* Unfortunatelly, validation that [Legacy] mode only supports [v0] + API happens as part of the handlers inside [RPC_server]. This means, + that it is not sufficient to simply ping the endpoints with dummy data, + as deserialization of RPC arguments and input parameters happens before + the actual API validation. *) + let expected_root_hash = + "00a3703854279d2f377d689163d1ec911a840d84b56c4c6f6cafdf0610394df7c6" + in + let root_hash_stream_promise = + wait_for_root_hash_pushed_to_data_streamer + coordinator + expected_root_hash + in + let* hex_root_hash = init_hex_root_hash ~payload:"test" coordinator in + assert (expected_root_hash = Hex.show hex_root_hash) ; + let* () = root_hash_stream_promise in + (* [observer] node is needeed to test "GET /v1/missing_page". The reason + is that this endpoint triggers validation that coordinator of the + current node is specified in the the configuration file. + Unfortunatelly, this validation happens before the actual validation + of the appropriate API version, thus the boilerplate inside this test. + *) + let observer = + Dac_node.create_legacy + ~name:"Observer" + ~threshold + ~committee_members: + (List.map + (fun (dc : Account.aggregate_key) -> + dc.aggregate_public_key_hash) + dac_committee) + ~node + ~client + () + in + let* _ = Dac_node.init_config observer in + let () = set_coordinator observer coordinator in + let* () = Dac_node.run observer in + assert (List.length dac_committee > 0) ; + let member_key : Account.aggregate_key = List.nth dac_committee 0 in + let dac_member_pkh = member_key.aggregate_public_key_hash in + let member_signature = bls_sign_hex_hash member_key hex_root_hash in + (* Test starts here.*) + (* We call endpoints supported by [Legacy] mode with [v1] API version. + Since [Legacy] mode does not support [v1] API, we assert that each one + of the responses returns 404 (not found) code. Additionally, we also + call endpoint only supported by [Coordinator] mode with both [v0] and + [v1] API version and assert both return not found response. *) + let invalid_legacy_mode_calls = + [ + (* GET /v1/preimage *) + RPC.call_raw + coordinator + Dac_rpc.(get_preimage (Hex.show hex_root_hash) ~api_version:Api.v1); + (* GET /v1/verify_signature *) + RPC.call_raw + coordinator + Dac_rpc.(get_verify_signature "dummy signature" ~api_version:Api.v1); + (* GET /v1/missing_page *) + RPC.call_raw + observer + Dac_rpc.(get_missing_page ~hex_root_hash ~api_version:Api.v1); + (* PUT /v1/dac_member_signature *) + RPC.call_raw + coordinator + Dac_rpc.( + put_dac_member_signature + ~hex_root_hash + ~dac_member_pkh + ~signature:member_signature + ~api_version:Api.v1); + (* GET /v1/certificates *) + RPC.call_raw + coordinator + Dac_rpc.(get_certificate ~hex_root_hash ~api_version:Api.v1); + (* [Legacy] mode fails calling [Coordinator]'s mode specific + PUT v0/preimage*) + RPC.call_raw + coordinator + Dac_rpc.( + Coordinator.post_preimage ~payload:"Test" ~api_version:Api.v0); + (* [Legacy] mode fails calling [Coordinator]'s mode specific + PUT v1/preimage*) + RPC.call_raw + coordinator + Dac_rpc.( + Coordinator.post_preimage ~payload:"Test" ~api_version:Api.v1); + ] + in + Lwt_list.iter_s + (fun invalid_response -> + let* response = invalid_response in + return @@ RPC.check_string_response ~code:404 response) + invalid_legacy_mode_calls + end end module Full_infrastructure = struct @@ -1689,7 +1790,11 @@ module Full_infrastructure = struct assert (List.length committee_members > 0) ; let payload, expected_rh = sample_payload "preimage" in let certificate_stream_client = - Runnable.run @@ streamed_certificates_client coordinator_node expected_rh + Runnable.run + @@ streamed_certificates_client + coordinator_node + expected_rh + Dac_rpc.Api.v1 in let push_promise = wait_for_root_hash_pushed_to_data_streamer coordinator_node expected_rh @@ -1766,7 +1871,11 @@ module Full_infrastructure = struct item is returned with the same certificate returned by the GET endpoint. *) let* second_certificates_stream = - Runnable.run @@ streamed_certificates_client coordinator_node expected_rh + Runnable.run + @@ streamed_certificates_client + coordinator_node + expected_rh + Dac_rpc.Api.v1 in Check.( (1 = List.length second_certificates_stream) @@ -1858,6 +1967,13 @@ let register ~protocols = "dac_store_member_signature" Legacy.Signature_manager.test_handle_store_signature protocols ; + scenario_with_layer1_and_legacy_dac_nodes + ~threshold:0 + ~committee_members:1 + ~tags:["dac"; "dac_node"] + "dac_legacy_mode_only_supports_v0" + Legacy.Api_versioning.test_legacy_mode_does_not_support_v1 + protocols ; scenario_with_full_dac_infrastructure ~observers:0 ~committee_members:0 -- GitLab From 93a741e83f464a209b7c946d504bab721f38c0fd Mon Sep 17 00:00:00 2001 From: Martin Tomazic Date: Wed, 3 May 2023 14:27:28 +0200 Subject: [PATCH 11/13] DAC/tezt: v0 specific endpoints are not supported in V1 api --- tezt/tests/dac.ml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tezt/tests/dac.ml b/tezt/tests/dac.ml index 2264b32e957b..b9826b049843 100644 --- a/tezt/tests/dac.ml +++ b/tezt/tests/dac.ml @@ -1890,6 +1890,30 @@ module Full_infrastructure = struct in check_certificate get_certificate certificate_update_from_second_stream ; return () + + module Api = struct + (** Test that [V0] API specific endpoints are not supported in + [V1] API. I.e. [V1] no longer support "POST /store_preimage" and + "GET /verify_signature". *) + let test_v1_does_not_support_v0_specific_endpoints + Scenarios.{coordinator_node; _} = + let* response = + RPC.call_raw + coordinator_node + Dac_rpc.( + post_store_preimage + ~payload:"test" + ~pagination_scheme:"Merkle_tree_V0" + ~api_version:Api.v1) + in + let () = RPC.check_string_response ~code:404 response in + let* response = + RPC.call_raw + coordinator_node + Dac_rpc.(get_verify_signature "test" ~api_version:Dac_rpc.Api.v1) + in + return @@ RPC.check_string_response ~code:404 response + end end let register ~protocols = @@ -2001,4 +2025,11 @@ let register ~protocols = ~tags:["dac"; "dac_node"] "certificates are updated in streaming endpoint" Full_infrastructure.test_streaming_certificates + protocols ; + scenario_with_full_dac_infrastructure + ~observers:0 + ~committee_members:0 + ~tags:["dac"; "dac_node"] + "v1 does not support v0 specific endpoints" + Full_infrastructure.Api.test_v1_does_not_support_v0_specific_endpoints protocols -- GitLab From a3e6d9981259a826354bb1644806e5efbe065855 Mon Sep 17 00:00:00 2001 From: Martin Tomazic Date: Thu, 4 May 2023 13:25:17 +0200 Subject: [PATCH 12/13] Fixup: Fetch function should not depend on the API version --- src/lib_dac_node/RPC_server.ml | 10 ++- src/lib_dac_node/handler.ml | 84 +++++++++++-------- src/lib_dac_node/page_store.ml | 76 +++++++++-------- src/lib_dac_node/page_store.mli | 25 +++--- src/lib_dac_node/pages_encoding.ml | 14 +++- src/lib_dac_node/pages_encoding.mli | 7 +- .../test/test_dac_pages_encoding.ml | 8 +- .../test/test_dac_pages_encoding.ml | 8 +- 8 files changed, 134 insertions(+), 98 deletions(-) diff --git a/src/lib_dac_node/RPC_server.ml b/src/lib_dac_node/RPC_server.ml index 4d399f5eaa06..8b8ed37c3043 100644 --- a/src/lib_dac_node/RPC_server.ml +++ b/src/lib_dac_node/RPC_server.ml @@ -182,15 +182,17 @@ let handle_get_missing_page node_ctxt cctxt page_store dac_plugin raw_root_hash let open Lwt_result_syntax in let*? () = assert_legacy_mode_cannot_run_v1 node_ctxt api_version in let*? root_hash = Dac_plugin.raw_to_hash dac_plugin raw_root_hash in + let (module Remote_store : Page_store.S + with type configuration = Page_store.remote_configuration) = + Page_store.remote_store api_version + in + let remote_store = Remote_store.init {cctxt; page_store} in match api_version with | RPC_services.Api.V0 | RPC_services.Api.V1 -> - let remote_store = - Page_store.Remote.(init {cctxt; page_store; api_version}) - in let* preimage = (* TODO: https://gitlab.com/tezos/tezos/-/issues/5142 Retrieve missing page from dac committee via "flooding". *) - Page_store.Remote.load dac_plugin remote_store root_hash + Remote_store.load dac_plugin remote_store root_hash in let*! () = Event.(emit fetched_missing_page raw_root_hash) in return preimage diff --git a/src/lib_dac_node/handler.ml b/src/lib_dac_node/handler.ml index 953fd8aad7ba..8b6515d07459 100644 --- a/src/lib_dac_node/handler.ml +++ b/src/lib_dac_node/handler.ml @@ -168,14 +168,11 @@ module Committee_member = struct let coordinator_cctxt = ctxt.Node_context.Committee_member.coordinator_cctxt in - let handler dac_plugin remote_store _stopper root_hash = + let handler dac_plugin remote_store deserialize_payload _stopper root_hash = let*? root_hash = Dac_plugin.raw_to_hash dac_plugin root_hash in let*! () = Event.emit_new_root_hash_received dac_plugin root_hash in let*! payload_result = - Pages_encoding.Merkle_tree.V0.Remote.deserialize_payload - dac_plugin - ~page_store:remote_store - root_hash + deserialize_payload dac_plugin ~page_store:remote_store root_hash in match payload_result with | Ok _ -> @@ -199,14 +196,29 @@ module Committee_member = struct in return () in - let remote_store = - Page_store.( - Remote.init {cctxt = coordinator_cctxt; page_store; api_version}) - in - let*! () = Event.(emit subscribed_to_root_hashes_stream ()) in - make_stream_daemon - (handler dac_plugin remote_store) - (Monitor_services.root_hashes coordinator_cctxt api_version) + match api_version with + | RPC_services.Api.V0 -> + let remote_store = + Page_store.Remote_V0.init {cctxt = coordinator_cctxt; page_store} + in + let*! () = Event.(emit subscribed_to_root_hashes_stream ()) in + make_stream_daemon + (handler + dac_plugin + remote_store + Pages_encoding.Merkle_tree.V0.Remote.deserialize_payload) + (Monitor_services.root_hashes coordinator_cctxt api_version) + | RPC_services.Api.V1 -> + let remote_store = + Page_store.Remote_V1.init {cctxt = coordinator_cctxt; page_store} + in + let*! () = Event.(emit subscribed_to_root_hashes_stream ()) in + make_stream_daemon + (handler + dac_plugin + remote_store + Pages_encoding.Merkle_tree.V1.Remote.deserialize_payload) + (Monitor_services.root_hashes coordinator_cctxt api_version) end (** Handlers specific to an [Observer]. An [Observer] is responsible for @@ -220,14 +232,11 @@ module Observer = struct let new_root_hash ctxt dac_plugin page_store api_version = let open Lwt_result_syntax in let coordinator_cctxt = ctxt.Node_context.Observer.coordinator_cctxt in - let handler dac_plugin remote_store _stopper root_hash = + let handler dac_plugin remote_store deserialize_payload _stopper root_hash = let*? root_hash = Dac_plugin.raw_to_hash dac_plugin root_hash in let*! () = Event.emit_new_root_hash_received dac_plugin root_hash in let*! payload_result = - Pages_encoding.Merkle_tree.V0.Remote.deserialize_payload - dac_plugin - ~page_store:remote_store - root_hash + deserialize_payload dac_plugin ~page_store:remote_store root_hash in match payload_result with | Ok _ -> @@ -243,14 +252,29 @@ module Observer = struct in return () in - let remote_store = - Page_store.( - Remote.init {cctxt = coordinator_cctxt; page_store; api_version}) - in - let*! () = Event.(emit subscribed_to_root_hashes_stream ()) in - make_stream_daemon - (handler dac_plugin remote_store) - (Monitor_services.root_hashes coordinator_cctxt api_version) + match api_version with + | RPC_services.Api.V0 -> + let remote_store = + Page_store.Remote_V0.init {cctxt = coordinator_cctxt; page_store} + in + let*! () = Event.(emit subscribed_to_root_hashes_stream ()) in + make_stream_daemon + (handler + dac_plugin + remote_store + Pages_encoding.Merkle_tree.V0.Remote.deserialize_payload) + (Monitor_services.root_hashes coordinator_cctxt api_version) + | RPC_services.Api.V1 -> + let remote_store = + Page_store.Remote_V1.init {cctxt = coordinator_cctxt; page_store} + in + let*! () = Event.(emit subscribed_to_root_hashes_stream ()) in + make_stream_daemon + (handler + dac_plugin + remote_store + Pages_encoding.Merkle_tree.V1.Remote.deserialize_payload) + (Monitor_services.root_hashes coordinator_cctxt api_version) end (** Handlers specific to a [Legacy] DAC node. If no @@ -354,13 +378,7 @@ module Legacy = struct return () in let remote_store = - Page_store.( - Remote.init - { - cctxt = coordinator_cctxt; - page_store; - api_version = RPC_services.Api.V0; - }) + Page_store.Remote_V0.init {cctxt = coordinator_cctxt; page_store} in let*! () = Event.(emit subscribed_to_root_hashes_stream ()) in make_stream_daemon diff --git a/src/lib_dac_node/page_store.ml b/src/lib_dac_node/page_store.ml index 2e781ca0fcf5..5ea321202578 100644 --- a/src/lib_dac_node/page_store.ml +++ b/src/lib_dac_node/page_store.ml @@ -230,62 +230,56 @@ module With_remote_fetch (R : sig type remote_context val fetch : - Dac_plugin.t -> - remote_context -> - RPC_services.Api.version -> - Dac_plugin.hash -> - bytes tzresult Lwt.t + Dac_plugin.t -> remote_context -> Dac_plugin.hash -> bytes tzresult Lwt.t end) (P : S) : S - with type configuration = R.remote_context * P.t * RPC_services.Api.version - and type t = R.remote_context * P.t * RPC_services.Api.version = struct - type t = R.remote_context * P.t * RPC_services.Api.version + with type configuration = R.remote_context * P.t + and type t = R.remote_context * P.t = struct + type t = R.remote_context * P.t - type configuration = R.remote_context * P.t * RPC_services.Api.version + type configuration = R.remote_context * P.t - let init (remote_ctxt, page_store, api_version) = - (remote_ctxt, page_store, api_version) + let init (remote_ctxt, page_store) = (remote_ctxt, page_store) - let save plugin (_remote_ctxt, page_store, _api_version) ~hash ~content = + let save plugin (_remote_ctxt, page_store) ~hash ~content = P.save plugin page_store ~hash ~content - let mem plugin (_remote_ctxt, page_store, _api_version) hash = - P.mem plugin page_store hash + let mem plugin (_remote_ctxt, page_store) hash = P.mem plugin page_store hash - let load plugin (remote_ctxt, page_store, api_version) hash = + let load plugin (remote_ctxt, page_store) hash = let open Lwt_result_syntax in - let* page_exists_in_store = - mem plugin (remote_ctxt, page_store, api_version) hash - in + let* page_exists_in_store = mem plugin (remote_ctxt, page_store) hash in if page_exists_in_store then P.load plugin page_store hash else - let* content = R.fetch plugin remote_ctxt api_version hash in + let* content = R.fetch plugin remote_ctxt hash in let+ () = P.save plugin page_store ~hash ~content in content end +module type API_VERSION = sig + val version : RPC_services.Api.version +end + type remote_configuration = { cctxt : Dac_node_client.cctxt; page_store : Filesystem.t; - api_version : RPC_services.Api.version; } -module Remote : S with type configuration = remote_configuration = struct +module Make_Remote (A : API_VERSION) : + S with type configuration = remote_configuration = struct module F = Filesystem_with_integrity_check module Internal : S - with type configuration = - Dac_node_client.cctxt * Filesystem.t * RPC_services.Api.version - and type t = - Dac_node_client.cctxt * Filesystem.t * RPC_services.Api.version = + with type configuration = Dac_node_client.cctxt * Filesystem.t + and type t = Dac_node_client.cctxt * Filesystem.t = With_remote_fetch (struct type remote_context = Dac_node_client.cctxt - let fetch _dac_plugin remote_context api_version hash = - (match api_version with + let fetch _dac_plugin remote_context hash = + (match A.version with | RPC_services.Api.V0 -> Dac_node_client.V0.get_preimage | RPC_services.Api.V1 -> Dac_node_client.V1.get_preimage) remote_context @@ -299,10 +293,23 @@ module Remote : S with type configuration = remote_configuration = struct type configuration = remote_configuration - let init {cctxt; page_store; api_version} = - Internal.init (cctxt, page_store, api_version) + let init {cctxt; page_store} = Internal.init (cctxt, page_store) end +module Remote_V0 = Make_Remote (struct + let version = RPC_services.Api.V0 +end) + +module Remote_V1 = Make_Remote (struct + let version = RPC_services.Api.V1 +end) + +let remote_store api_version = + (module Make_Remote (struct + let version = api_version + end) : S + with type configuration = remote_configuration) + module Internal_for_tests = struct module With_data_integrity_check (P : S) : S with type configuration = P.configuration and type t = P.t = @@ -312,16 +319,11 @@ module Internal_for_tests = struct type remote_context val fetch : - Dac_plugin.t -> - remote_context -> - RPC_services.Api.version -> - Dac_plugin.hash -> - bytes tzresult Lwt.t + Dac_plugin.t -> remote_context -> Dac_plugin.hash -> bytes tzresult Lwt.t end) (P : S) : S - with type configuration = - R.remote_context * P.t * RPC_services.Api.version - and type t = R.remote_context * P.t * RPC_services.Api.version = + with type configuration = R.remote_context * P.t + and type t = R.remote_context * P.t = With_remote_fetch (R) (P) end diff --git a/src/lib_dac_node/page_store.mli b/src/lib_dac_node/page_store.mli index e0255c24ac39..111740a15541 100644 --- a/src/lib_dac_node/page_store.mli +++ b/src/lib_dac_node/page_store.mli @@ -88,15 +88,21 @@ module Filesystem : S with type configuration = string type remote_configuration = { cctxt : Dac_node_client.cctxt; page_store : Filesystem.t; - api_version : RPC_services.Api.version; } (** A [Page_store] implementation backed by the local filesystem, which uses a connection to a Dac node to retrieve pages that are not - saved locally. *) -module Remote : S with type configuration = remote_configuration + saved locally. [V0] api is used in case of retrieving the pages remotely. *) +module Remote_V0 : S with type configuration = remote_configuration -(**/**) +(** A [Page_store] implementation backed by the local filesystem, which + uses a connection to a Dac node to retrieve pages that are not + saved locally. [V1] api is used in case of retrieving the pages remotely. *) +module Remote_V1 : S with type configuration = remote_configuration + +val remote_store : + RPC_services.Api.version -> + (module S with type configuration = remote_configuration) module Internal_for_tests : sig (** [With_data_integrity_check] tweaks a module [P] of type [Page_store] @@ -115,17 +121,12 @@ module Internal_for_tests : sig type remote_context val fetch : - Dac_plugin.t -> - remote_context -> - RPC_services.Api.version -> - Dac_plugin.hash -> - bytes tzresult Lwt.t + Dac_plugin.t -> remote_context -> Dac_plugin.hash -> bytes tzresult Lwt.t end) (P : S) : S - with type configuration = - R.remote_context * P.t * RPC_services.Api.version - and type t = R.remote_context * P.t * RPC_services.Api.version + with type configuration = R.remote_context * P.t + and type t = R.remote_context * P.t end (** [ensure_reveal_data_dir_exists reveal_data_dir] checks that the diff --git a/src/lib_dac_node/pages_encoding.ml b/src/lib_dac_node/pages_encoding.ml index 7c090bea980b..12e8036765b1 100644 --- a/src/lib_dac_node/pages_encoding.ml +++ b/src/lib_dac_node/pages_encoding.ml @@ -554,7 +554,7 @@ module Merkle_tree = struct module Filesystem = Make_buffered (Page_store.Filesystem) (V0_metadata) (V0_page_size) module Remote = - Make_buffered (Page_store.Remote) (V0_metadata) (V0_page_size) + Make_buffered (Page_store.Remote_V0) (V0_metadata) (V0_page_size) end module V0 = struct @@ -562,6 +562,18 @@ module Merkle_tree = struct module Remote = Make (V0_Buffered.Remote) end + module V1_Buffered = struct + module Filesystem = + Make_buffered (Page_store.Filesystem) (V0_metadata) (V0_page_size) + module Remote = + Make_buffered (Page_store.Remote_V1) (V0_metadata) (V0_page_size) + end + + module V1 = struct + module Filesystem = Make (V1_Buffered.Filesystem) + module Remote = Make (V1_Buffered.Remote) + end + module Internal_for_tests = struct module Make_buffered = Make_buffered module Make = Make diff --git a/src/lib_dac_node/pages_encoding.mli b/src/lib_dac_node/pages_encoding.mli index 662d27731ff8..e6d512cbbf69 100644 --- a/src/lib_dac_node/pages_encoding.mli +++ b/src/lib_dac_node/pages_encoding.mli @@ -157,9 +157,14 @@ module Merkle_tree : sig module V0 : sig module Filesystem : Dac_codec with type page_store = Page_store.Filesystem.t - module Remote : Dac_codec with type page_store = Page_store.Remote.t + module Remote : Dac_codec with type page_store = Page_store.Remote_V0.t end + module V1 : sig + module Filesystem : Dac_codec with type page_store = Page_store.Filesystem.t + + module Remote : Dac_codec with type page_store = Page_store.Remote_V1.t + end module Internal_for_tests : sig module Make_buffered (S : Page_store.S) (V : VERSION) (C : CONFIG) : Buffered_dac_codec with type page_store = S.t diff --git a/src/proto_017_PtNairob/lib_dac_plugin/test/test_dac_pages_encoding.ml b/src/proto_017_PtNairob/lib_dac_plugin/test/test_dac_pages_encoding.ml index 096081c44996..8a91291d16c9 100644 --- a/src/proto_017_PtNairob/lib_dac_plugin/test/test_dac_pages_encoding.ml +++ b/src/proto_017_PtNairob/lib_dac_plugin/test/test_dac_pages_encoding.ml @@ -233,13 +233,12 @@ module With_hash_check : module Double_hash_map_backend : Page_store.S - with type configuration = - Hashes_map_backend.t * Hashes_map_backend.t * RPC_services.Api.version = + with type configuration = Hashes_map_backend.t * Hashes_map_backend.t = Page_store.Internal_for_tests.With_remote_fetch (struct type remote_context = Hashes_map_backend.t - let fetch dac_plugin remote_context _api_version hash = + let fetch dac_plugin remote_context hash = Hashes_map_backend.load dac_plugin remote_context hash end) (With_hash_check) @@ -398,8 +397,7 @@ module Merkle_tree = struct let mock_remote_store = Hashes_map_backend.init () in let mock_local_store = Hashes_map_backend.init () in let page_store = - Double_hash_map_backend.init - (mock_remote_store, mock_local_store, RPC_services.Api.V0) + Double_hash_map_backend.init (mock_remote_store, mock_local_store) in let payload = Bytes.of_string "This is a payload that will be tampered later on" diff --git a/src/proto_alpha/lib_dac_plugin/test/test_dac_pages_encoding.ml b/src/proto_alpha/lib_dac_plugin/test/test_dac_pages_encoding.ml index 458ad72880bb..4bcca03fed5e 100644 --- a/src/proto_alpha/lib_dac_plugin/test/test_dac_pages_encoding.ml +++ b/src/proto_alpha/lib_dac_plugin/test/test_dac_pages_encoding.ml @@ -233,13 +233,12 @@ module With_hash_check : module Double_hash_map_backend : Page_store.S - with type configuration = - Hashes_map_backend.t * Hashes_map_backend.t * RPC_services.Api.version = + with type configuration = Hashes_map_backend.t * Hashes_map_backend.t = Page_store.Internal_for_tests.With_remote_fetch (struct type remote_context = Hashes_map_backend.t - let fetch dac_plugin remote_context _api_version hash = + let fetch dac_plugin remote_context hash = Hashes_map_backend.load dac_plugin remote_context hash end) (With_hash_check) @@ -398,8 +397,7 @@ module Merkle_tree = struct let mock_remote_store = Hashes_map_backend.init () in let mock_local_store = Hashes_map_backend.init () in let page_store = - Double_hash_map_backend.init - (mock_remote_store, mock_local_store, RPC_services.Api.V0) + Double_hash_map_backend.init (mock_remote_store, mock_local_store) in let payload = Bytes.of_string "This is a payload that will be tampered later on" -- GitLab From 6639bbd20a591eadd254e1ff150c6fe23fb149cf Mon Sep 17 00:00:00 2001 From: Martin Tomazic Date: Fri, 5 May 2023 13:50:27 +0200 Subject: [PATCH 13/13] Fixup: Update docstring with new api_version parameter --- src/lib_dac/RPC_services.mli | 40 +++++++++++++-------------- src/lib_dac_node/monitor_services.mli | 13 ++++++--- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/src/lib_dac/RPC_services.mli b/src/lib_dac/RPC_services.mli index e88e586dcd36..e1a5df0571c5 100644 --- a/src/lib_dac/RPC_services.mli +++ b/src/lib_dac/RPC_services.mli @@ -39,9 +39,9 @@ module Api : sig val version_rpc_arg : version Tezos_rpc.Arg.arg end -(** POST dac/store_preimage to post a payload using a given [pagination_scheme]. - It returns the base58 encoded root page hash - and the raw bytes. *) +(** "POST [api_version]/store_preimage" serializes a payload using a given + [pagination_scheme]. It returns the base58 encoded root page hash + and the raw bytes. *) val post_store_preimage : ( [`POST], unit, @@ -51,10 +51,10 @@ val post_store_preimage : Dac_plugin.raw_hash * Bytes.t ) Tezos_rpc.Service.service -(** GET dac/verify_signature endpoint requests the DAL node to verify - the signature of the external message [external_message]. The DAC committee - of the DAL node must be the same that was used to produce the - [external_message]. *) +(** "GET [api_version]/verify_signature" endpoint requests the DAL node to verify + the signature of the external message [external_message]. The DAC committee + of the DAL node must be the same that was used to produce the + [external_message]. *) val get_verify_signature : ( [`GET], unit, @@ -64,7 +64,7 @@ val get_verify_signature : bool ) Tezos_rpc.Service.service -(** GET dac/preimage requests the preimage of hash, consisting of a +(** "GET [api_version]/preimage" requests the preimage of hash, consisting of a single page, from cctxt. When the request succeeds, the raw page will be returned as a sequence of bytes. *) val get_preimage : @@ -76,8 +76,8 @@ val get_preimage : Bytes.t ) Tezos_rpc.Service.service -(** PUT dac/member_signature endpoint stores the [signature] - generated from signing [hex_root_hash] by [dac_member_pkh]. *) +(** "PUT [api_version]/member_signature" endpoint stores the [signature] + generated from signing [hex_root_hash] by [dac_member_pkh]. *) val put_dac_member_signature : ( [`PUT], unit, @@ -87,8 +87,8 @@ val put_dac_member_signature : unit ) Tezos_rpc.Service.service -(** GET dac/certificate endpoint returns the DAC certificate for the - provided [root_page_hash]. *) +(** "GET [api_version]/certificate" endpoint returns the DAC certificate for the + provided [root_page_hash]. *) val get_certificate : ( [`GET], unit, @@ -98,9 +98,9 @@ val get_certificate : Certificate_repr.t option ) Tezos_rpc.Service.service -(** GET dac/missing_page/[page_hash] Observer fetches the missing page - from a Coordinator node. The missing page is then saved to a - page store before returning the page as a response. *) +(** "GET [api_version]/missing_page/[page_hash]" fetches the missing page from + a Coordinator node. The missing page is then saved to a page store before + returning the page as a response. *) val get_missing_page : ( [`GET], unit, @@ -111,11 +111,11 @@ val get_missing_page : Tezos_rpc.Service.service module Coordinator : sig - (** POST dac/preimage sends a [payload] to the DAC - [Coordinator]. It returns a hex encoded root page hash, - produced by [Merkle_tree_V0] pagination scheme. - On the backend side it also pushes root page hash of the preimage to all - the subscribed DAC Members and Observers. *) + (** "POST [api_version]/preimage" sends a [payload] to the DAC + [Coordinator]. It returns a hex encoded root page hash, + produced by [Merkle_tree_V0] pagination scheme. + On the backend side it also pushes root page hash of the preimage to all + the subscribed DAC Members and Observers. *) val post_preimage : ( [`POST], unit, diff --git a/src/lib_dac_node/monitor_services.mli b/src/lib_dac_node/monitor_services.mli index fbe9fa4d854a..83e4450eb57e 100644 --- a/src/lib_dac_node/monitor_services.mli +++ b/src/lib_dac_node/monitor_services.mli @@ -24,7 +24,7 @@ (*****************************************************************************) module S : sig - (** Define RPC GET /monitor/root_hashes. *) + (** Define RPC GET [api_version]/monitor/root_hashes. *) val root_hashes : ( [`GET], unit, @@ -34,7 +34,7 @@ module S : sig Dac_plugin.raw_hash ) Tezos_rpc.Service.service - (** Define RPC GET /monitor/certificate/{hex_root_hash}. *) + (** Define RPC GET [api_version]/monitor/certificate/{hex_root_hash}. *) val certificate : ( [`GET], unit, @@ -45,10 +45,10 @@ module S : sig Tezos_rpc.Service.service end -(** [root_hashes streamed_cctxt dac_plugin] returns a stream of root hashes +(** [root_hashes streamed_cctxt api_version] returns a stream of root hashes and a stopper for it. - Stream is produced by calling RPC GET /monitor/root_hashes. + Stream is produced by calling RPC GET [api_version]/monitor/root_hashes. *) val root_hashes : #Tezos_rpc.Context.streamed -> @@ -57,6 +57,11 @@ val root_hashes : Error_monad.tzresult Lwt.t +(** [certificate streamed_cctxt raw_hash api_version] returns a stream of + updates to certificate of a given root hash and a stopper for it. + + Stream is produced by calling RPC GET [api_version]/monitor/certificate. +*) val certificate : #Tezos_rpc.Context.streamed -> Dac_plugin.raw_hash -> -- GitLab