From 3be629a1bda15b24207a19faf49065fcd1494355 Mon Sep 17 00:00:00 2001 From: Andrea Cerone Date: Mon, 9 Jan 2023 14:22:47 +0000 Subject: [PATCH 1/6] Manifest: add lib_dac --- .gitlab/ci/jobs/packaging/opam_package.yml | 2 + dune-project | 1 + manifest/main.ml | 57 +++++++++++++++++++++- opam/octez-dal-node.opam | 1 + opam/tezos-dac-alpha.opam | 30 ++++++++++++ src/bin_dal_node/dune | 11 ++++- src/proto_alpha/lib_dac/dune | 30 ++++++++++++ src/proto_alpha/lib_dac/test/dune | 25 ++++++++++ src/proto_alpha/lib_dac/test/main.ml | 47 ++++++++++++++++++ 9 files changed, 200 insertions(+), 4 deletions(-) create mode 100644 opam/tezos-dac-alpha.opam create mode 100644 src/proto_alpha/lib_dac/dune create mode 100644 src/proto_alpha/lib_dac/test/dune create mode 100644 src/proto_alpha/lib_dac/test/main.ml diff --git a/.gitlab/ci/jobs/packaging/opam_package.yml b/.gitlab/ci/jobs/packaging/opam_package.yml index 4f538e9bc66a..91ea225aac79 100644 --- a/.gitlab/ci/jobs/packaging/opam_package.yml +++ b/.gitlab/ci/jobs/packaging/opam_package.yml @@ -605,6 +605,8 @@ opam:tezos-crypto-dal: variables: package: tezos-crypto-dal +# Ignoring unreleased package tezos-dac-alpha. + # Ignoring unreleased package tezos-dal-016-PtMumbai. # Ignoring unreleased package tezos-dal-alpha. diff --git a/dune-project b/dune-project index cf43b111b5ca..cc119d876940 100644 --- a/dune-project +++ b/dune-project @@ -76,6 +76,7 @@ (package (name tezos-context-ops)) (package (name tezos-crypto)) (package (name tezos-crypto-dal)) +(package (name tezos-dac-alpha)) (package (name tezos-dal-016-PtMumbai)) (package (name tezos-dal-alpha)) (package (name tezos-dal-node-lib)(allow_empty)) diff --git a/manifest/main.ml b/manifest/main.ml index 649decdd5128..6150545ca4da 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -3421,6 +3421,8 @@ module Protocol : sig val dal : t -> target option + val dac : t -> target option + val parameters_exn : t -> target val benchmarks_proto_exn : t -> target @@ -3535,6 +3537,7 @@ end = struct plugin : target option; plugin_registerer : target option; dal : target option; + dac : target option; test_helpers : target option; parameters : target option; benchmarks_proto : target option; @@ -3542,7 +3545,7 @@ end = struct } let make ?client ?client_commands ?client_commands_registration - ?baking_commands_registration ?plugin ?plugin_registerer ?dal + ?baking_commands_registration ?plugin ?plugin_registerer ?dal ?dac ?test_helpers ?parameters ?benchmarks_proto ?baking ~status ~name ~main ~embedded () = { @@ -3557,6 +3560,7 @@ end = struct plugin; plugin_registerer; dal; + dac; test_helpers; parameters; benchmarks_proto; @@ -3611,6 +3615,8 @@ end = struct let dal p = p.dal + let dac p = p.dac + let parameters_exn p = mandatory "parameters" p p.parameters let benchmarks_proto_exn p = mandatory "benchmarks_proto" p p.benchmarks_proto @@ -5053,6 +5059,50 @@ module Protocol = Protocol alcotest_lwt; ] in + let dac = + (* [~link_all:true] is necessary to ensure that the dac plugin + registration happens when running the dal node. Removing this + option would cause DAL related tezts to fail because the DAC + plugin cannot be resolved. *) + only_if (active && N.(number >= 017)) @@ fun () -> + public_lib + (sf "tezos-dac-%s" name_dash) + ~path:(path // "lib_dac") + ~synopsis: + "Tezos/Protocol: protocol specific library for the Data availability \ + Committee" + ~deps: + [ + octez_base |> open_ ~m:"TzPervasives" + |> open_ ~m:"TzPervasives.Error_monad.Legacy_monad_globals"; + octez_protocol_compiler_registerer |> open_; + octez_stdlib_unix |> open_; + octez_dal_node_lib |> open_; + client |> if_some |> open_; + embedded |> open_; + layer2_utils |> if_some |> open_; + main |> open_; + ] + ~inline_tests:ppx_expect + ~linkall:true + in + let _dac_tests = + only_if (active && N.(number >= 017)) @@ fun () -> + test + "main" + ~path:(path // "lib_dac/test") + ~opam:(sf "tezos-dac-%s" name_dash) + ~deps: + [ + octez_base |> open_ ~m:"TzPervasives" + |> open_ ~m:"TzPervasives.Error_monad.Legacy_monad_globals"; + dac |> if_some |> open_; + main |> open_; + octez_base_test_helpers |> open_; + test_helpers |> if_some |> open_; + alcotest_lwt; + ] + in let benchmark_type_inference = only_if active @@ fun () -> public_lib @@ -5217,6 +5267,7 @@ module Protocol = Protocol ?plugin ?plugin_registerer ?dal + ?dac ?test_helpers ?parameters ?benchmarks_proto @@ -5969,7 +6020,9 @@ let _octez_dal_node = (* Other protocols are optional. *) true in - let targets = List.filter_map Fun.id [Protocol.dal protocol] in + let targets = + List.filter_map Fun.id [Protocol.dal protocol; Protocol.dac protocol] + in if is_optional then List.map optional targets else targets in List.map deps_for_protocol Protocol.all |> List.flatten diff --git a/opam/octez-dal-node.opam b/opam/octez-dal-node.opam index 726ae6a440c6..d7c2bc836635 100644 --- a/opam/octez-dal-node.opam +++ b/opam/octez-dal-node.opam @@ -30,6 +30,7 @@ depends: [ ] depopts: [ "tezos-dal-alpha" + "tezos-dac-alpha" ] build: [ ["rm" "-r" "vendors"] diff --git a/opam/tezos-dac-alpha.opam b/opam/tezos-dac-alpha.opam new file mode 100644 index 000000000000..d11ebbbb5c3e --- /dev/null +++ b/opam/tezos-dac-alpha.opam @@ -0,0 +1,30 @@ +# This file was automatically generated, do not edit. +# Edit file manifest/main.ml instead. +opam-version: "2.0" +maintainer: "contact@tezos.com" +authors: ["Tezos devteam"] +homepage: "https://www.tezos.com/" +bug-reports: "https://gitlab.com/tezos/tezos/issues" +dev-repo: "git+https://gitlab.com/tezos/tezos.git" +license: "MIT" +depends: [ + "dune" { >= "3.0" } + "ppx_expect" + "tezos-base" + "octez-protocol-compiler" + "tezos-stdlib-unix" + "tezos-dal-node-lib" + "tezos-client-alpha" + "tezos-embedded-protocol-alpha" + "tezos-layer2-utils-alpha" + "tezos-protocol-alpha" + "tezos-base-test-helpers" {with-test} + "tezos-alpha-test-helpers" {with-test} + "alcotest-lwt" { with-test & >= "1.5.0" } +] +build: [ + ["rm" "-r" "vendors"] + ["dune" "build" "-p" name "-j" jobs] + ["dune" "runtest" "-p" name "-j" jobs] {with-test} +] +synopsis: "Tezos/Protocol: protocol specific library for the Data availability Committee" diff --git a/src/bin_dal_node/dune b/src/bin_dal_node/dune index e37e55fe1146..6c96ca3234da 100644 --- a/src/bin_dal_node/dune +++ b/src/bin_dal_node/dune @@ -29,7 +29,10 @@ tezos-dal-016-PtMumbai (select void_for_linking-tezos-dal-alpha from (tezos-dal-alpha -> void_for_linking-tezos-dal-alpha.empty) - (-> void_for_linking-tezos-dal-alpha.empty))) + (-> void_for_linking-tezos-dal-alpha.empty)) + (select void_for_linking-tezos-dac-alpha from + (tezos-dac-alpha -> void_for_linking-tezos-dac-alpha.empty) + (-> void_for_linking-tezos-dac-alpha.empty))) (link_flags (:standard) (:include %{workspace_root}/static-link-flags.sexp)) @@ -47,4 +50,8 @@ -open Tezos_layer2_store -open Tezos_crypto_dal)) -(rule (action (progn (write-file void_for_linking-tezos-dal-alpha.empty "")))) +(rule + (action + (progn + (write-file void_for_linking-tezos-dal-alpha.empty "") + (write-file void_for_linking-tezos-dac-alpha.empty "")))) diff --git a/src/proto_alpha/lib_dac/dune b/src/proto_alpha/lib_dac/dune new file mode 100644 index 000000000000..fd122828bdf6 --- /dev/null +++ b/src/proto_alpha/lib_dac/dune @@ -0,0 +1,30 @@ +; This file was automatically generated, do not edit. +; Edit file manifest/main.ml instead. + +(library + (name tezos_dac_alpha) + (public_name tezos-dac-alpha) + (instrumentation (backend bisect_ppx)) + (libraries + tezos-base + octez-protocol-compiler.registerer + tezos-stdlib-unix + tezos_dal_node_lib + tezos-client-alpha + tezos-embedded-protocol-alpha + tezos-layer2-utils-alpha + tezos-protocol-alpha) + (inline_tests (flags -verbose) (modes native)) + (preprocess (pps ppx_expect)) + (library_flags (:standard -linkall)) + (flags + (:standard) + -open Tezos_base.TzPervasives + -open Tezos_base.TzPervasives.Error_monad.Legacy_monad_globals + -open Tezos_protocol_registerer + -open Tezos_stdlib_unix + -open Tezos_dal_node_lib + -open Tezos_client_alpha + -open Tezos_embedded_protocol_alpha + -open Tezos_layer2_utils_alpha + -open Tezos_protocol_alpha)) diff --git a/src/proto_alpha/lib_dac/test/dune b/src/proto_alpha/lib_dac/test/dune new file mode 100644 index 000000000000..0b4377db3166 --- /dev/null +++ b/src/proto_alpha/lib_dac/test/dune @@ -0,0 +1,25 @@ +; This file was automatically generated, do not edit. +; Edit file manifest/main.ml instead. + +(executable + (name main) + (libraries + tezos-base + tezos-dac-alpha + tezos-protocol-alpha + tezos-base-test-helpers + tezos-alpha-test-helpers + alcotest-lwt) + (flags + (:standard) + -open Tezos_base.TzPervasives + -open Tezos_base.TzPervasives.Error_monad.Legacy_monad_globals + -open Tezos_dac_alpha + -open Tezos_protocol_alpha + -open Tezos_base_test_helpers + -open Tezos_alpha_test_helpers)) + +(rule + (alias runtest) + (package tezos-dac-alpha) + (action (run %{dep:./main.exe}))) diff --git a/src/proto_alpha/lib_dac/test/main.ml b/src/proto_alpha/lib_dac/test/main.ml new file mode 100644 index 000000000000..9450a26495ca --- /dev/null +++ b/src/proto_alpha/lib_dac/test/main.ml @@ -0,0 +1,47 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2022 Trili Tech, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +module Unit_test : sig + (** + * Example: [spec "Dac_pages_encoding.ml" Test_dac_pages_encoding.tests] + * Unit tests needs tag in log (like "[UNIT] some test description here...") + * This function handles such meta data *) + val spec : + string -> + unit Alcotest_lwt.test_case list -> + string * unit Alcotest_lwt.test_case list + + (** Tests with description string without [Unit] are skipped *) + val _skip : + string -> + unit Alcotest_lwt.test_case list -> + string * unit Alcotest_lwt.test_case list +end = struct + let spec unit_name test_cases = ("[Unit] " ^ unit_name, test_cases) + + let _skip unit_name test_cases = ("[SKIPPED] " ^ unit_name, test_cases) +end + +let () = Alcotest_lwt.run "protocol > unit" [] |> Lwt_main.run -- GitLab From 83be828495427f94e0807307921d5ce59d0e2da4 Mon Sep 17 00:00:00 2001 From: Andrea Cerone Date: Mon, 9 Jan 2023 16:38:33 +0000 Subject: [PATCH 2/6] Dac: Plugin for Dal node --- src/lib_dal_node/dac_plugin.ml | 37 +++++++++++++++++++ src/lib_dal_node/dac_plugin.mli | 32 ++++++++++++++++ .../lib_dac/dac_plugin_registration.ml | 30 +++++++++++++++ 3 files changed, 99 insertions(+) create mode 100644 src/lib_dal_node/dac_plugin.ml create mode 100644 src/lib_dal_node/dac_plugin.mli create mode 100644 src/proto_alpha/lib_dac/dac_plugin_registration.ml diff --git a/src/lib_dal_node/dac_plugin.ml b/src/lib_dal_node/dac_plugin.ml new file mode 100644 index 000000000000..aec1365c3daa --- /dev/null +++ b/src/lib_dal_node/dac_plugin.ml @@ -0,0 +1,37 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Trili Tech, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +module type T = sig + module Proto : Registered_protocol.T +end + +let table : (module T) Tezos_crypto.Protocol_hash.Table.t = + Tezos_crypto.Protocol_hash.Table.create 5 + +let register (module Plugin : T) = + assert (not (Tezos_crypto.Protocol_hash.Table.mem table Plugin.Proto.hash)) ; + Tezos_crypto.Protocol_hash.Table.add table Plugin.Proto.hash (module Plugin) + +let get hash = Tezos_crypto.Protocol_hash.Table.find table hash diff --git a/src/lib_dal_node/dac_plugin.mli b/src/lib_dal_node/dac_plugin.mli new file mode 100644 index 000000000000..f59d55c3084c --- /dev/null +++ b/src/lib_dal_node/dac_plugin.mli @@ -0,0 +1,32 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Trili Tech, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +module type T = sig + module Proto : Registered_protocol.T +end + +val register : (module T) -> unit + +val get : Tezos_crypto.Protocol_hash.Table.key -> (module T) option diff --git a/src/proto_alpha/lib_dac/dac_plugin_registration.ml b/src/proto_alpha/lib_dac/dac_plugin_registration.ml new file mode 100644 index 000000000000..bb39651ba200 --- /dev/null +++ b/src/proto_alpha/lib_dac/dac_plugin_registration.ml @@ -0,0 +1,30 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2022 Nomadic Labs, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +module Plugin = struct + module Proto = Registerer.Registered +end + +let () = Dac_plugin.register (module Plugin) -- GitLab From df51a031f50501adf99ca2421f6374a588c928fe Mon Sep 17 00:00:00 2001 From: Andrea Cerone Date: Mon, 9 Jan 2023 16:38:50 +0000 Subject: [PATCH 3/6] Dal node: register both Dal and Dac plugins --- src/bin_dal_node/RPC_server.ml | 4 +- src/bin_dal_node/dac_manager.ml | 19 +++++++++ src/bin_dal_node/dac_manager.mli | 6 +++ src/bin_dal_node/daemon.ml | 65 ++++++++++++++++++++----------- src/bin_dal_node/event_legacy.ml | 9 +++-- src/bin_dal_node/node_context.ml | 10 +++-- src/bin_dal_node/node_context.mli | 16 ++++---- 7 files changed, 91 insertions(+), 38 deletions(-) diff --git a/src/bin_dal_node/RPC_server.ml b/src/bin_dal_node/RPC_server.ml index b75f506635d3..013937882e58 100644 --- a/src/bin_dal_node/RPC_server.ml +++ b/src/bin_dal_node/RPC_server.ml @@ -199,7 +199,9 @@ let start configuration cctxt ctxt dac_pks_opt dac_sk_uris = let dir = Tezos_rpc.Directory.register_dynamic_directory dir plugin_prefix (fun () -> match Node_context.get_status ctxt with - | Ready {plugin = (module Plugin); _} -> + (* TODO: Replace this with the Dac plugin once the RPC server has + been moved there. *) + | Ready {dal_plugin = (module Plugin); _} -> Lwt.return (Plugin.RPC.rpc_services ~reveal_data_dir diff --git a/src/bin_dal_node/dac_manager.ml b/src/bin_dal_node/dac_manager.ml index 99960e3a301f..01262a1ad73a 100644 --- a/src/bin_dal_node/dac_manager.ml +++ b/src/bin_dal_node/dac_manager.ml @@ -110,3 +110,22 @@ module Storage = struct else tzfail @@ Cannot_create_reveal_data_dir reveal_data_dir | _ -> tzfail @@ Cannot_create_reveal_data_dir reveal_data_dir) end + +let resolve_plugin + (protocols : Tezos_shell_services.Chain_services.Blocks.protocols) = + let open Lwt_syntax in + let plugin_opt = + Option.either + (Dac_plugin.get protocols.current_protocol) + (Dac_plugin.get protocols.next_protocol) + in + Option.map_s + (fun dac_plugin -> + let (module Dac_plugin : Dac_plugin.T) = dac_plugin in + let* () = + Event.emit_protocol_plugin_resolved + ~plugin_name:"dac" + Dac_plugin.Proto.hash + in + return dac_plugin) + plugin_opt diff --git a/src/bin_dal_node/dac_manager.mli b/src/bin_dal_node/dac_manager.mli index 03b597650438..c40c51fe6472 100644 --- a/src/bin_dal_node/dac_manager.mli +++ b/src/bin_dal_node/dac_manager.mli @@ -62,3 +62,9 @@ module Storage : sig *) val ensure_reveal_data_dir_exists : string -> unit tzresult Lwt.t end + +(* TODO: https://gitlab.com/tezos/tezos/-/issues/4626 + Unify this function and resolve_dal_plugin in Dal node daemon. *) +val resolve_plugin : + Tezos_shell_services.Chain_services.Blocks.protocols -> + (module Dac_plugin.T) option Lwt.t diff --git a/src/bin_dal_node/daemon.ml b/src/bin_dal_node/daemon.ml index 25f448fcbc0d..46ef4bda5a39 100644 --- a/src/bin_dal_node/daemon.ml +++ b/src/bin_dal_node/daemon.ml @@ -23,15 +23,24 @@ (* *) (*****************************************************************************) -let resolve_plugin cctxt = - let open Lwt_result_syntax in - let* protocols = - Tezos_shell_services.Chain_services.Blocks.protocols cctxt () +let resolve_dal_plugin + (protocols : Tezos_shell_services.Chain_services.Blocks.protocols) = + let open Lwt_syntax in + let plugin_opt = + Option.either + (Dal_plugin.get protocols.current_protocol) + (Dal_plugin.get protocols.next_protocol) in - return - @@ Option.either - (Dal_plugin.get protocols.current_protocol) - (Dal_plugin.get protocols.next_protocol) + Option.map_s + (fun dal_plugin -> + let (module Dal_plugin : Dal_plugin.T) = dal_plugin in + let* () = + Event.emit_protocol_plugin_resolved + ~plugin_name:"dal" + Dal_plugin.Proto.hash + in + return dal_plugin) + plugin_opt type error += | Cryptobox_initialisation_failed of string @@ -97,7 +106,7 @@ module Handler = struct in return (go (), stopper) - let resolve_plugin_and_set_ready config ctxt cctxt = + let resolve_plugins_and_set_ready config ctxt cctxt = (* Monitor heads and try resolve the DAL protocol plugin corresponding to the protocol of the targeted node. *) (* FIXME: https://gitlab.com/tezos/tezos/-/issues/3605 @@ -105,22 +114,32 @@ module Handler = struct let open Lwt_result_syntax in let handler stopper (_block_hash, (_block_header : Tezos_base.Block_header.t)) = - let* plugin = resolve_plugin cctxt in - match plugin with - | Some plugin -> - let (module Plugin : Dal_plugin.T) = plugin in - let*! () = Event.emit_protocol_plugin_resolved Plugin.Proto.hash in + let* protocols = + Tezos_shell_services.Chain_services.Blocks.protocols cctxt () + in + (* TODO: https://gitlab.com/tezos/tezos/-/issues/4627 + Register only one plugin according to mode of operation. *) + let*! dal_plugin = resolve_dal_plugin protocols in + let*! dac_plugin = Dac_manager.resolve_plugin protocols in + match (dal_plugin, dac_plugin) with + | Some dal_plugin, Some dac_plugin -> + let (module Dal_plugin : Dal_plugin.T) = dal_plugin in let* proto_parameters = - Plugin.get_constants cctxt#chain cctxt#block cctxt + Dal_plugin.get_constants cctxt#chain cctxt#block cctxt in let* cryptobox = init_cryptobox config.Configuration.use_unsafe_srs proto_parameters in - Node_context.set_ready ctxt (module Plugin) cryptobox proto_parameters ; + Node_context.set_ready + ctxt + ~dal_plugin + ~dac_plugin + cryptobox + proto_parameters ; let*! () = Event.(emit node_is_ready ()) in stopper () ; return_unit - | None -> return_unit + | _, _ -> return_unit in let handler stopper el = match Node_context.get_status ctxt with @@ -138,15 +157,17 @@ module Handler = struct let handler _stopper (block_hash, (header : Tezos_base.Block_header.t)) = match Node_context.get_status ctxt with | Starting -> return_unit - | Ready {plugin = (module Plugin); proto_parameters; _} -> + | Ready {dal_plugin = (module Dal_plugin); proto_parameters; _} -> let block_level = header.shell.level in let* block_info = - Plugin.block_info + Dal_plugin.block_info cctxt ~block:(`Hash (block_hash, 0)) ~metadata:`Always in - let* slot_headers = Plugin.get_published_slot_headers block_info in + let* slot_headers = + Dal_plugin.get_published_slot_headers block_info + in let*! () = Slot_manager.store_slot_headers ~block_level @@ -155,7 +176,7 @@ module Handler = struct (Node_context.get_store ctxt) in let*? attested_slots = - Plugin.attested_slot_headers + Dal_plugin.attested_slot_headers block_hash block_info ~number_of_slots:proto_parameters.number_of_slots @@ -264,7 +285,7 @@ let run ~data_dir cctxt = in (* Start daemon to resolve current protocol plugin *) let* () = - daemonize [Handler.resolve_plugin_and_set_ready config ctxt cctxt] + daemonize [Handler.resolve_plugins_and_set_ready config ctxt cctxt] in (* Start never-ending monitoring daemons *) daemonize (Handler.new_head ctxt cctxt :: Handler.new_slot_header ctxt) diff --git a/src/bin_dal_node/event_legacy.ml b/src/bin_dal_node/event_legacy.ml index 1c620c60e586..3fe531229427 100644 --- a/src/bin_dal_node/event_legacy.ml +++ b/src/bin_dal_node/event_legacy.ml @@ -122,11 +122,12 @@ let layer1_node_tracking_started = () let protocol_plugin_resolved = - declare_1 + declare_2 ~section ~name:"dal_node_plugin_resolved" - ~msg:"Resolved plugin on protocol {proto_hash}" + ~msg:"Resolved plugin for {plugin_name} on protocol {proto_hash}" ~level:Notice + ("plugin_name", Data_encoding.string) ("proto_hash", Data_encoding.string) let daemon_error = @@ -173,5 +174,5 @@ let dac_account_cannot_sign = let proto_short_hash_string hash = Format.asprintf "%a" Tezos_crypto.Protocol_hash.pp_short hash -let emit_protocol_plugin_resolved hash = - emit protocol_plugin_resolved (proto_short_hash_string hash) +let emit_protocol_plugin_resolved ~plugin_name hash = + emit protocol_plugin_resolved (plugin_name, proto_short_hash_string hash) diff --git a/src/bin_dal_node/node_context.ml b/src/bin_dal_node/node_context.ml index 1bdc63d5e9bd..a963c78bebc1 100644 --- a/src/bin_dal_node/node_context.ml +++ b/src/bin_dal_node/node_context.ml @@ -28,7 +28,8 @@ exception Status_already_ready type ready_ctxt = { cryptobox : Cryptobox.t; proto_parameters : Dal_plugin.proto_parameters; - plugin : (module Dal_plugin.T); + dal_plugin : (module Dal_plugin.T); + dac_plugin : (module Dac_plugin.T); } type status = Ready of ready_ctxt | Starting @@ -59,9 +60,10 @@ let init config store cctxt = Committee_cache.create ~max_size:Constants.committee_cache_size; } -let set_ready ctxt plugin cryptobox proto_parameters = +let set_ready ctxt ~dal_plugin ~dac_plugin cryptobox proto_parameters = match ctxt.status with - | Starting -> ctxt.status <- Ready {plugin; cryptobox; proto_parameters} + | Starting -> + ctxt.status <- Ready {dac_plugin; dal_plugin; cryptobox; proto_parameters} | Ready _ -> raise Status_already_ready type error += Node_not_ready @@ -103,7 +105,7 @@ let fetch_assigned_shard_indicies ctxt ~level ~pkh = match Committee_cache.find cache ~level with | Some committee -> return committee | None -> - let*? {plugin = (module Plugin); _} = get_ready ctxt in + let*? {dal_plugin = (module Plugin); _} = get_ready ctxt in let+ committee = Plugin.get_committee cctxt ~level in let committee = Tezos_crypto.Signature.Public_key_hash.Map.map diff --git a/src/bin_dal_node/node_context.mli b/src/bin_dal_node/node_context.mli index 04e7356ee43b..d8b75b332c98 100644 --- a/src/bin_dal_node/node_context.mli +++ b/src/bin_dal_node/node_context.mli @@ -24,12 +24,13 @@ (*****************************************************************************) (** A [ready_ctx] value contains globally needed informations for a running dal - node. It is available when both cryptobox is initialized and dal plugin is - loaded. *) + node. It is available when both cryptobox is initialized and the plugins + for dal and dac have been loaded. *) type ready_ctxt = { cryptobox : Cryptobox.t; proto_parameters : Dal_plugin.proto_parameters; - plugin : (module Dal_plugin.T); + dal_plugin : (module Dal_plugin.T); + dac_plugin : (module Dac_plugin.T); } (** The status of the dal node *) @@ -47,14 +48,15 @@ val init : Configuration.t -> Store.node_store -> Client_context.full -> t (** Raised by [set_ready] when the status is already [Ready _] *) exception Status_already_ready -(** [set_ready ctxt plugin cryptobox proto_parameters] updates in place the - status value to [Ready], and initializes the inner [ready_ctxt] value with - the given parameters. +(** [set_ready ctxt ~dal_plugin ~dac_plugin cryptobox proto_parameters] updates + in place the status value to [Ready], and initializes the inner + [ready_ctxt] value with the given parameters. @raise Status_already_ready when the status is already [Ready _] *) val set_ready : t -> - (module Tezos_dal_node_lib.Dal_plugin.T) -> + dal_plugin:(module Tezos_dal_node_lib.Dal_plugin.T) -> + dac_plugin:(module Tezos_dal_node_lib.Dac_plugin.T) -> Cryptobox.t -> Dal_plugin.proto_parameters -> unit -- GitLab From 7de44b7e11ffcaeee7d7f0478c74222c665a38a2 Mon Sep 17 00:00:00 2001 From: Andrea Cerone Date: Mon, 9 Jan 2023 16:49:08 +0000 Subject: [PATCH 4/6] Dac plugin: copy dac files from Dal plugin --- src/lib_dal_node/dac_plugin.ml | 10 + src/lib_dal_node/dac_plugin.mli | 10 + src/proto_alpha/lib_dac/RPC.ml | 199 ++++++ .../lib_dac/dac_external_message_manager.ml | 114 ++++ src/proto_alpha/lib_dac/dac_manager.ml | 30 + src/proto_alpha/lib_dac/dac_pages_encoding.ml | 636 ++++++++++++++++++ .../lib_dac/dac_plugin_registration.ml | 1 + .../lib_dac/dac_preimage_data_manager.ml | 63 ++ .../lib_dac/dac_signature_manager.ml | 180 +++++ src/proto_alpha/lib_dac/test/main.ml | 6 +- .../test/test_dac_pages_encoding.ml | 2 +- src/proto_alpha/lib_dal/test/main.ml | 1 - 12 files changed, 1249 insertions(+), 3 deletions(-) create mode 100644 src/proto_alpha/lib_dac/RPC.ml create mode 100644 src/proto_alpha/lib_dac/dac_external_message_manager.ml create mode 100644 src/proto_alpha/lib_dac/dac_manager.ml create mode 100644 src/proto_alpha/lib_dac/dac_pages_encoding.ml create mode 100644 src/proto_alpha/lib_dac/dac_preimage_data_manager.ml create mode 100644 src/proto_alpha/lib_dac/dac_signature_manager.ml rename src/proto_alpha/{lib_dal => lib_dac}/test/test_dac_pages_encoding.ml (99%) diff --git a/src/lib_dal_node/dac_plugin.ml b/src/lib_dal_node/dac_plugin.ml index aec1365c3daa..7b298b3f8a2a 100644 --- a/src/lib_dal_node/dac_plugin.ml +++ b/src/lib_dal_node/dac_plugin.ml @@ -25,6 +25,16 @@ module type T = sig module Proto : Registered_protocol.T + + module RPC : sig + val rpc_services : + reveal_data_dir:string -> + #Client_context.wallet -> + Tezos_crypto.Aggregate_signature.public_key option list -> + Client_keys.aggregate_sk_uri option list -> + int -> + unit Tezos_rpc.Directory.directory + end end let table : (module T) Tezos_crypto.Protocol_hash.Table.t = diff --git a/src/lib_dal_node/dac_plugin.mli b/src/lib_dal_node/dac_plugin.mli index f59d55c3084c..4de3ee0faeb5 100644 --- a/src/lib_dal_node/dac_plugin.mli +++ b/src/lib_dal_node/dac_plugin.mli @@ -25,6 +25,16 @@ module type T = sig module Proto : Registered_protocol.T + + module RPC : sig + val rpc_services : + reveal_data_dir:string -> + #Client_context.wallet -> + Tezos_crypto.Aggregate_signature.public_key option list -> + Client_keys.aggregate_sk_uri option list -> + int -> + unit Tezos_rpc.Directory.directory + end end val register : (module T) -> unit diff --git a/src/proto_alpha/lib_dac/RPC.ml b/src/proto_alpha/lib_dac/RPC.ml new file mode 100644 index 000000000000..1cda639b25ef --- /dev/null +++ b/src/proto_alpha/lib_dac/RPC.ml @@ -0,0 +1,199 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2022 Trili Tech *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +open Environment +open Error_monad + +type error += + | Cannot_construct_external_message + | Cannot_deserialize_external_message + +let () = + register_error_kind + `Permanent + ~id:"dac_cannot_construct_external_message" + ~title:"External rollup message could not be constructed" + ~description:"External rollup message could not be constructed" + ~pp:(fun ppf () -> + Format.fprintf ppf "External rollup message could not be constructed") + Data_encoding.unit + (function Cannot_construct_external_message -> Some () | _ -> None) + (fun () -> Cannot_construct_external_message) ; + register_error_kind + `Permanent + ~id:"dac_cannot_deserialize_rollup_external_message" + ~title:"External rollup message could not be deserialized" + ~description:"External rollup message could not be deserialized" + ~pp:(fun ppf () -> + Format.fprintf ppf "External rollup message could not be deserialized") + Data_encoding.unit + (function Cannot_deserialize_external_message -> Some () | _ -> None) + (fun () -> Cannot_deserialize_external_message) + +module Registration = struct + let register0_noctxt ~chunked s f dir = + RPC_directory.register ~chunked dir s (fun _rpc_ctxt q i -> f q i) +end + +module DAC = struct + module Hash_storage = Dac_preimage_data_manager.Reveal_hash + + let store_preimage_request_encoding = + Data_encoding.( + obj2 + (req "payload" Data_encoding.(bytes Hex)) + (req "pagination_scheme" Dac_pages_encoding.pagination_scheme_encoding)) + + (* A variant of [Sc_rollup_reveal_hash.encoding] that prefers hex + encoding over b58check encoding for JSON. *) + let root_hash_encoding = + let binary = Protocol.Sc_rollup_reveal_hash.encoding in + Data_encoding.( + splitted + ~binary + ~json: + (conv_with_guard + Protocol.Sc_rollup_reveal_hash.to_hex + (fun str -> + Result.of_option + ~error:"Not a valid hash" + (Protocol.Sc_rollup_reveal_hash.of_hex str)) + (string Plain))) + + let store_preimage_response_encoding = + Data_encoding.( + obj2 + (req "root_hash" root_hash_encoding) + (req "external_message" (bytes Hex))) + + let external_message_query = + let open RPC_query in + query (fun hex_string -> hex_string) + |+ opt_field "external_message" RPC_arg.string (fun s -> s) + |> seal + + module S = struct + let dac_store_preimage = + RPC_service.put_service + ~description:"Split DAC reveal data" + ~query:RPC_query.empty + ~input:store_preimage_request_encoding + ~output:store_preimage_response_encoding + RPC_path.(open_root / "dac" / "store_preimage") + + (* DAC/FIXME: https://gitlab.com/tezos/tezos/-/issues/4263 + remove this endpoint once end-to-end tests are in place. *) + let verify_external_message_signature = + RPC_service.get_service + ~description:"Verify signature of an external message to inject in L1" + ~query:external_message_query + ~output:Data_encoding.bool + RPC_path.(open_root / "dac" / "verify_signature") + end + + let handle_serialize_dac_store_preimage cctxt dac_sk_uris reveal_data_dir + (data, pagination_scheme) = + let open Lwt_result_syntax in + let open Dac_pages_encoding in + let for_each_page (hash, page_contents) = + Dac_manager.Reveal_hash.Storage.save_bytes + reveal_data_dir + hash + page_contents + in + let* root_hash = + match pagination_scheme with + | Merkle_tree_V0 -> + let size = + Protocol.Alpha_context.Constants.sc_rollup_message_size_limit + in + Merkle_tree.V0.serialize_payload + ~max_page_size:size + data + ~for_each_page + | Hash_chain_V0 -> Hash_chain.V0.serialize_payload ~for_each_page data + in + let* signature, witnesses = + Dac_manager.Reveal_hash.Signatures.sign_root_hash + cctxt + dac_sk_uris + root_hash + in + let*? external_message = + match + Dac_manager.Reveal_hash.External_message.make + root_hash + signature + witnesses + with + | Ok external_message -> Ok external_message + | Error _ -> Error_monad.error Cannot_construct_external_message + in + return (root_hash, external_message) + + let handle_verify_external_message_signature public_keys_opt + encoded_l1_message = + let open Lwt_result_syntax in + let open Dac_manager.Reveal_hash in + let external_message = + let open Option_syntax in + let* encoded_l1_message = encoded_l1_message in + let* as_bytes = Hex.to_bytes @@ `Hex encoded_l1_message in + External_message.of_bytes as_bytes + in + match external_message with + | None -> tzfail @@ Cannot_deserialize_external_message + | Some (External_message.Dac_message {root_hash; signature; witnesses}) -> + Signatures.verify ~public_keys_opt root_hash signature witnesses + + let register_serialize_dac_store_preimage cctxt dac_sk_uris reveal_data_dir = + Registration.register0_noctxt + ~chunked:false + S.dac_store_preimage + (fun () input -> + handle_serialize_dac_store_preimage + cctxt + dac_sk_uris + reveal_data_dir + input) + + let register_verify_external_message_signature public_keys_opt = + Registration.register0_noctxt + ~chunked:false + S.verify_external_message_signature + (fun external_message () -> + handle_verify_external_message_signature + public_keys_opt + external_message) + + let register reveal_data_dir cctxt dac_public_keys_opt dac_sk_uris = + (RPC_directory.empty : unit RPC_directory.t) + |> register_serialize_dac_store_preimage cctxt dac_sk_uris reveal_data_dir + |> register_verify_external_message_signature dac_public_keys_opt +end + +let rpc_services ~reveal_data_dir cctxt dac_public_keys_opt dac_sk_uris + _threshold = + DAC.register reveal_data_dir cctxt dac_public_keys_opt dac_sk_uris diff --git a/src/proto_alpha/lib_dac/dac_external_message_manager.ml b/src/proto_alpha/lib_dac/dac_external_message_manager.ml new file mode 100644 index 000000000000..56b6557e296d --- /dev/null +++ b/src/proto_alpha/lib_dac/dac_external_message_manager.ml @@ -0,0 +1,114 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2022 Trili Tech *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +open Protocol +open Environment.Error_monad + +type error += Could_not_serialize_rollup_external_message of string + +let () = + register_error_kind + `Permanent + ~id:"dac_could_not_serialize_rollup_external_message" + ~title:"Could not serialize rollup external message" + ~description: + "Serialization of rollup external message containing Dac root page hash \ + failed" + ~pp:(fun ppf b58_hash -> + Format.fprintf + ppf + "Serialization of rollup external message containing Dac root page \ + hash %sfailed" + b58_hash) + Data_encoding.(obj1 (req "hash" string)) + (function + | Could_not_serialize_rollup_external_message b58_hash -> Some b58_hash + | _ -> None) + (fun b58_hash -> Could_not_serialize_rollup_external_message b58_hash) + +module type REVEAL_HASH = module type of Sc_rollup_reveal_hash + +module Make + (Hashing_scheme : REVEAL_HASH) (Encoding_metadata : sig + val tag : int + + val title : string + end) = +struct + type dac_message = + | Dac_message of { + root_hash : Hashing_scheme.t; + signature : Tezos_crypto.Aggregate_signature.t; + witnesses : Bitset.t; + } + + let untagged_encoding = + Data_encoding.( + conv + (function + | Dac_message {root_hash; signature; witnesses} -> + (root_hash, signature, witnesses)) + (fun (root_hash, signature, witnesses) -> + Dac_message {root_hash; signature; witnesses}) + (obj3 + (req "root_hash" Hashing_scheme.encoding) + (req "signature" Tezos_crypto.Aggregate_signature.encoding) + (req "witnesses" Bitset.encoding))) + + let dac_message_encoding = + Data_encoding.( + union + ~tag_size:`Uint8 + [ + case + ~title:("dac_message_" ^ Encoding_metadata.title) + (Tag Encoding_metadata.tag) + untagged_encoding + (fun msg -> Some msg) + (fun msg -> msg); + ]) + + let make root_hash signature witnesses = + let message = Dac_message {root_hash; signature; witnesses} in + let res = Data_encoding.Binary.to_bytes dac_message_encoding message in + match res with + | Ok bytes -> Ok bytes + | Error _ -> + error + @@ Could_not_serialize_rollup_external_message + (Hashing_scheme.to_hex root_hash) + + let of_bytes encoded_message = + Data_encoding.Binary.of_bytes_opt dac_message_encoding encoded_message +end + +module Reveal_hash = + Make + (Sc_rollup_reveal_hash) + (struct + let tag = 42 + + let title = "reveal_hash_v0" + end) diff --git a/src/proto_alpha/lib_dac/dac_manager.ml b/src/proto_alpha/lib_dac/dac_manager.ml new file mode 100644 index 000000000000..cbf068e1ffdd --- /dev/null +++ b/src/proto_alpha/lib_dac/dac_manager.ml @@ -0,0 +1,30 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2022 Trili Tech *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +module Reveal_hash = struct + module Storage = Dac_preimage_data_manager.Reveal_hash + module Signatures = Dac_signature_manager.Reveal_hash + module External_message = Dac_external_message_manager.Reveal_hash +end diff --git a/src/proto_alpha/lib_dac/dac_pages_encoding.ml b/src/proto_alpha/lib_dac/dac_pages_encoding.ml new file mode 100644 index 000000000000..99a7ed2ec3d2 --- /dev/null +++ b/src/proto_alpha/lib_dac/dac_pages_encoding.ml @@ -0,0 +1,636 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2022 Trili Tech *) +(* Copyright (c) 2022 Marigold *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(* DAC/FIXME: https://gitlab.com/tezos/tezos/-/issues/4088 + Add .mli file. *) + +(** Library for encoding payloads of arbitrary size in formats that can be + decoded by the Sc-rollup kernels. + *) + +open Protocol +open Environment.Error_monad + +type error += + | Payload_cannot_be_empty + | Cannot_serialize_page_payload + | Cannot_deserialize_page + | Non_positive_size_of_payload + | Merkle_tree_branching_factor_not_high_enough + | Cannot_combine_pages_data_of_different_type + | Hashes_page_repr_expected_single_element + +type pagination_scheme = Merkle_tree_V0 | Hash_chain_V0 + +let pagination_scheme_encoding = + Data_encoding.string_enum + [("Merkle_tree_V0", Merkle_tree_V0); ("Hash_chain_V0", Hash_chain_V0)] + +let () = + register_error_kind + `Permanent + ~id:"cannot_deserialize_dac_page_payload" + ~title:"DAC payload could not be deserialized" + ~description:"Error when recovering DAC payload payload from binary" + ~pp:(fun ppf () -> + Format.fprintf + ppf + "Error when recovering DAC payload from list of data chunks") + Data_encoding.(unit) + (function Cannot_deserialize_page -> Some () | _ -> None) + (fun () -> Cannot_deserialize_page) ; + register_error_kind + `Permanent + ~id:"cannot_serialize_dac_page" + ~title:"DAC page could not be serialized" + ~description:"Error when serializing DAC page" + ~pp:(fun ppf () -> Format.fprintf ppf "Error when serializing DAC page") + Data_encoding.(unit) + (function Cannot_serialize_page_payload -> Some () | _ -> None) + (fun () -> Cannot_serialize_page_payload) ; + register_error_kind + `Permanent + ~id:"non_positive_payload_size" + ~title:"Non positive size for dac payload" + ~description:"Dac page payload (excluded preamble) are non positive" + ~pp:(fun ppf () -> + Format.fprintf ppf "Dac page payload (excluded preamble) are non positive") + Data_encoding.(unit) + (function Non_positive_size_of_payload -> Some () | _ -> None) + (fun () -> Non_positive_size_of_payload) ; + register_error_kind + `Permanent + ~id:"dac_payload_cannot_be_empty" + ~title:"Cannot serialize empty DAC payload" + ~description:"Cannot serialize empty DAC payload" + ~pp:(fun ppf () -> Format.fprintf ppf "Cannot serialize empty DAC payload") + Data_encoding.(unit) + (function Payload_cannot_be_empty -> Some () | _ -> None) + (fun () -> Payload_cannot_be_empty) ; + register_error_kind + `Permanent + ~id:"merkle_tree_branching_factor_not_high_enough" + ~title:"Merkle tree branching factor must be at least 2" + ~description:"Merkle tree branching factor must be at least 2" + ~pp:(fun ppf () -> + Format.fprintf + ppf + "Cannot serialize DAC payload: pages must be able to contain at least \ + two hashes") + Data_encoding.(unit) + (function + | Merkle_tree_branching_factor_not_high_enough -> Some () | _ -> None) + (fun () -> Merkle_tree_branching_factor_not_high_enough) ; + register_error_kind + `Permanent + ~id:"cannot_combine_pages_data_of_different_type" + ~title:"Cannot combine pages data of different type" + ~description:"Cannot combine pages data of different type" + ~pp:(fun ppf () -> + Format.fprintf + ppf + "Merkle level serizalization: when adding page data to a given level \ + repr., the type of page data received is different then the one \ + inside a level repr.") + Data_encoding.(unit) + (function + | Cannot_combine_pages_data_of_different_type -> Some () | _ -> None) + (fun () -> Cannot_combine_pages_data_of_different_type) ; + register_error_kind + `Permanent + ~id:"hashes_page_repr_expected_single_element" + ~title:"Hashes page representation expected a single element" + ~description:"Hashes page representation expected a single element" + ~pp:(fun ppf () -> + Format.fprintf ppf "Hashes page representation expected a single element") + Data_encoding.unit + (function Hashes_page_repr_expected_single_element -> Some () | _ -> None) + (fun () -> Hashes_page_repr_expected_single_element) + +(** Encoding of DAC payload as a Merkle tree with an arbitrary branching + factor greater or equal to 2. The serialization process works as follows: + {ul + {li A large sequence of bytes, the payload, is split into several pages + of fixed size, each of which is prefixed with a small sequence + of bytes (also of fixed size), which is referred to as the preamble + of the page. Pages obtained directly from the original payload + are referred to as `Contents pages`. Contents pages constitute the + leaves of the Merkle tree being built, + } + {li Each contents page (each of which is a sequence of bytes consisting + of the preamble followed by the actual contents from the original + payload) is then hashed. The size of each hash is fixed. The hashes are + concatenated together, and the resulting sequence of bytes is split + into pages of the same size of `Hashes pages`, each of which is + prefixed with a preamble whose size is the same as in Contents pages. + Hashes pages correspond to nodes of the Merkle tree being built, and + the children of a hash page are the (either Payload or Hashes) pages + whose hash appear into the former, + } + {li Hashes pages are hashed using the same process described above, leading + to a smaller list of hashes pages. To guarantee that the list of hashes + pages is actually smaller than the original list of pages being hashed, + we require the size of pages to be large enough to contain at least two + hashes. + } + } + + Merkle tree encodings of DAC pages are versioned, to allow for multiple + hashing schemes to be used. + *) +module Merkle_tree = struct + type version = int + + (** A page is either a `Contents page`, containing a chunk of the payload + that needs to be serialized, or a `Hashes page`, containing a list + of hashes. The maximum size of bytes inside [Contents] page, or number + of hashes inside [Hashes] page is such, that when serializing a page + using [page_encoding], it does not exceed [max_page_size] bytes. + *) + type 'a page = Contents of bytes | Hashes of 'a list + + let max_version = 127 + + module type VERSION = sig + val contents_version_tag : version + + val hashes_version_tag : version + end + + (* Even numbers are used for versioning Contents pages, odd numbers are used + for versioning Hashes pages. *) + module Make_version (V : sig + val contents_version : int + + val hashes_version : int + end) = + struct + let contents_version_tag = 2 * V.contents_version + + let hashes_version_tag = (2 * V.hashes_version) + 1 + end + + module Make (Hashing_scheme : sig + include Dac_preimage_data_manager.REVEAL_HASH + + val scheme : supported_hashes + end) + (V : VERSION) = + struct + let hash bytes = + Hashing_scheme.hash_bytes [bytes] ~scheme:Hashing_scheme.scheme + + let hash_encoding = Hashing_scheme.encoding + + let hashes_encoding = Data_encoding.list hash_encoding + + (* The preamble of a serialized page contains 1 byte denoting the version, + and 4 bytes encoding the size of the rest of the page. In total, 5 + bytes. *) + let page_preamble_size = 5 + + let hash_bytes_size = Hashing_scheme.size ~scheme:Hashing_scheme.scheme + + (** Payload pages are encoded as follows: the first byte is an integer, + which is corresponds to either `payload_version` (for payload pages) or + `hashes_version` (for hashes pages). The next four bytes will contain + the size of the rest of the page; the remainder of the page is either a + list of raw bytes (in the case of a payload page), or a list of hashes, + which occupy 32 bytes each. *) + let page_encoding = + Data_encoding.( + union + ~tag_size:`Uint8 + [ + case + ~title:"contents" + (Tag V.contents_version_tag) + bytes + (function Contents payload -> Some payload | _ -> None) + (fun payload -> Contents payload); + case + ~title:"hashes" + (Tag V.hashes_version_tag) + hashes_encoding + (function Hashes hashes -> Some hashes | _ -> None) + (fun hashes -> Hashes hashes); + ]) + + (** Serialization function for a single page. It converts a page to a + sequence of bytes using [page_encoding]. It also checks that the + serialized page does not exceed [page_size] bytes. *) + let serialize_page ~max_page_size page = + match + Data_encoding.Binary.to_bytes + (Data_encoding.check_size max_page_size page_encoding) + page + with + | Ok raw_page -> Ok raw_page + | Error _ -> error Cannot_serialize_page_payload + + (* Splits payload into bytes chunks whose size does not exceed [page_size] bytes. *) + let split_payload ~max_page_size payload = + let open Result_syntax in + (* 1 byte for the version size, 4 bytes for the size of the payload. *) + let actual_page_size = max_page_size - page_preamble_size in + if actual_page_size <= 0 then error Non_positive_size_of_payload + else + let+ splitted_payload = String.chunk_bytes actual_page_size payload in + List.map (fun cont -> Contents (String.to_bytes cont)) splitted_payload + + let max_hashes_per_page ~max_page_size = + (max_page_size - page_preamble_size) / hash_bytes_size + + let split_hashes ~max_page_size hashes = + let number_of_hashes = max_hashes_per_page ~max_page_size in + (* Requiring a branching factor of at least 2 is necessary to ensure that + the serialization process terminates. If only one hash were stored per page, then + the number of pages at height `n-1` could be potentially equal to the number of + pages at height `n`. *) + if number_of_hashes < 2 then + error Merkle_tree_branching_factor_not_high_enough + else + let rec go aux list = + match list with + | [] -> List.rev aux + | list -> + let chunk, rest = List.split_n number_of_hashes list in + (go [@tailcall]) (Hashes chunk :: aux) rest + in + Ok (go [] hashes) + + let store_page ~max_page_size ~for_each_page page = + let open Lwt_result_syntax in + let*? serialized_page = serialize_page ~max_page_size page in + (* Hashes are computed from raw pages, each of which consists of a + preamble of 5 bytes followed by a page payload - a raw sequence + of bytes from the original payload for Contents pages, and a + a sequence of serialized hashes for hashes pages. The preamble + bytes is part of the sequence of bytes which is hashed. + *) + let hash = hash serialized_page in + let* () = for_each_page (hash, serialized_page) in + return hash + + (** [Payload_handler] is in-memory data structure that enables aggregating + DAC messages/payload. It serializes the data by respecting [page_encoding], + by storing the full payload in the shape of a k-ary merkle tree onto the disk. + During the proccess of partial serialization, the minimum amount of data + is kept in memory, by eagerly persisting the full pages of a given merkle + level whenever possible, to ease the load on the memory. + + The serializer respects the following invariant: + Starting with an [empty] serializer, splitting arbitrary [payload] into + chunks of arbitrary size, and adding them to the serializer from left to + right, should result in same root hash as adding all the payload data in + one chunk, provided it could fit into the memory. + *) + module Payload_handler = struct + (** A [page_data] is either [Cont_data] or [Hash_data] where each represents + data not bound in size for [Contents] or [Hashes] page respectively. *) + type page_data = Cont_data of bytes | Hash_data of Hashing_scheme.t list + + let is_empty_page_data = function + | Cont_data cont -> Bytes.empty = cont + | Hash_data ls -> List.is_empty ls + + (** [Payload_handler] serializes DAC data in the shape of k-ary Merkle tree + onto the disk. For every existing level written onto to the disk, handler + holds a corresponding [Merkle_level.t] in-memory representation. This + representation holds the data yet to be persisted to the disk - effectively + acting as buffer, making sure that all [pages] of the given level that + are written to the disk are full (with the exception of last one upon + finalization). + *) + module Merkle_level = struct + type t = page_data + + let init_level page_data = page_data + + let is_empty level = is_empty_page_data level + + let has_single_hash = function Hash_data [_] -> true | _ -> false + + let get_single_hash level = + let open Lwt_result_syntax in + match level with + | Hash_data [x] -> return x + | _ -> tzfail Hashes_page_repr_expected_single_element + + let split ~max_page_size page_data = + let leftover_with_full_pages ls = + let open Lwt_result_syntax in + match List.rev ls with + | [] -> + tzfail Payload_cannot_be_empty (* We don't expect to get here *) + | Contents h :: xs -> return (Cont_data h, List.rev xs) + | Hashes h :: xs -> return (Hash_data h, List.rev xs) + in + let open Lwt_result_syntax in + let*? pages = + match page_data with + | Cont_data cont -> split_payload ~max_page_size cont + | Hash_data ls -> split_hashes ~max_page_size ls + in + leftover_with_full_pages pages + + let add_page_data ~level ~page_data = + let open Lwt_result_syntax in + match (level, page_data) with + | Cont_data level, Cont_data page_data -> + return @@ Cont_data (Bytes.cat level page_data) + | Hash_data level, Hash_data page_data -> + return @@ Hash_data (List.append level page_data) + | Hash_data _, Cont_data _ | Cont_data _, Hash_data _ -> + (* We dont expect to get here *) + tzfail Cannot_combine_pages_data_of_different_type + + let process ~max_page_size ~for_each_page level = + let open Lwt_result_syntax in + let* level, pages = split ~max_page_size level in + let* rev_hashes = + List.fold_left_es + (fun accm page -> + let* hash = store_page ~max_page_size ~for_each_page page in + return @@ (hash :: accm)) + [] + pages + in + return (level, Hash_data (List.rev rev_hashes)) + end + + let empty : Merkle_level.t Stack.t = Stack.create () + + (** Adds a [page_data] to the bottom level of the in-memory merkle tree repr. + If [page_data] inside level exceeds its [max_page_size], then we split it + into valid full [page]/s and a leftover that is smaller or equal to + [max_page_size]. + + Full pages are eagerly stored to disk and parent hashes are added + in batch to next level. The leftover represents the content + of current level. Procedure repeats recursively if necessarily. + + Throws [Payload_cannot_be_empty] if [page_data] is empty. + *) + let rec add_rec ~max_page_size ~for_each_page stack page_data = + let open Lwt_result_syntax in + let open Merkle_level in + let* merkle_level = + if Stack.is_empty stack then return (init_level page_data) + else add_page_data ~level:(Stack.pop stack) ~page_data + in + let* merkle_level, page_data = + process ~max_page_size ~for_each_page merkle_level + in + let* () = + if not @@ is_empty_page_data page_data then + add_rec ~max_page_size ~for_each_page stack page_data + else return () + in + return @@ Stack.push merkle_level stack + + (** Adding [payload] to the serializer does not guarantee its persistance + to disk. The data is fully serialized after the call to [finalize]. + This guarantees that only the last [page] in the given level could be + partially filled. + + Throws [Payload_cannot_be_empty] in case of empty payload *) + let add ~max_page_size ~for_each_page stack payload = + let open Lwt_result_syntax in + let* () = + fail_unless (Bytes.length payload > 0) Payload_cannot_be_empty + in + let number_of_hashes = max_hashes_per_page ~max_page_size in + (* Requiring a branching factor of at least 2 is necessary to ensure that + the serialization process terminates. If only one hash were stored per page, then + the number of pages at height `n-1` could be potentially equal to the number of + pages at height `n`. *) + let* () = + fail_unless + (number_of_hashes > 1) + Merkle_tree_branching_factor_not_high_enough + in + add_rec ~max_page_size ~for_each_page stack (Cont_data payload) + + (** [finalize] returns a root hash of agggregated data, by eagerly storing + every partially filled level on the stack. Starting at the bottom and + adding stored page hash to next level, repeating this procedure until + left with a single (root) hash. *) + let rec finalize ~max_page_size ~for_each_page stack = + let open Merkle_level in + let finalize_level page_data = + let store_page = store_page ~max_page_size ~for_each_page in + match page_data with + | Cont_data cont -> store_page (Contents cont) + | Hash_data ls -> store_page (Hashes ls) + in + let open Lwt_result_syntax in + let* () = + (* Empty stack means no payload added in the first place *) + fail_unless (not @@ Stack.is_empty stack) Payload_cannot_be_empty + in + let merkle_level = Stack.pop stack in + if is_empty merkle_level then + (* If level is empty, there is nothing to hash *) + finalize ~max_page_size ~for_each_page stack + else if Stack.is_empty stack && has_single_hash merkle_level then + (* If top level of the tree and only one hash, then this is a root hash *) + get_single_hash merkle_level + else + let* hash = finalize_level merkle_level in + let* () = + add_rec ~max_page_size ~for_each_page stack (Hash_data [hash]) + in + finalize ~max_page_size ~for_each_page stack + end + + (** Main function for computing the pages of a Merkle tree from a sequence + of bytes. Each page is processed using the function [for_each_page] + provided in input, which is responsible for ensuring that the original + payload can be reconstructed from the Merkle tree root; this can be + achieved, for example, by letting [for_each_page] persist a serialized + page to disk using the page hash as its filename. + The function [serialize_payload] returns the root hash of the Merkle + tree constructed. *) + let serialize_payload ~max_page_size payload ~for_each_page = + let open Lwt_result_syntax in + let open Payload_handler in + let stack = empty in + let* () = add ~max_page_size ~for_each_page stack payload in + finalize ~max_page_size ~for_each_page stack + + (** Deserialization function for a single page. A sequence of bytes is + converted to a page using [page_encoding]. *) + let deserialize_page raw_page = + match Data_encoding.Binary.of_bytes page_encoding raw_page with + | Ok page -> Ok page + | Error _ -> error Cannot_deserialize_page + + (** Deserialization function for reconstructing the original payload from + its Merkle tree root hash. The function [retrieve_page_from_hash] + passed in input is responsible for determining how to retrieve the + serialized page from its hash. For example, if the page has been + persisted to disk using the page hash as its filename, + [retrieve_page_from_hash] simply loads the corresponding file from + disk to memory. The function [deserialize_payload] returns the + original payload that was used to compute the Merkle tree root hash. + This function is guaranteed to terminate if the directed graph induced + by the retrieved pages (that is, the graph where there is an edge + from one page to another if and only if the former contains the hash + of the latter) is acyclic. This property is guaranteed if the root + hash and pages are computed using the serialized_payload function + outlined above, but it is not guaranteed in more general cases. + *) + let deserialize_payload root_hash ~retrieve_page_from_hash = + let rec go retrieved_hashes retrieved_contents = + let open Lwt_result_syntax in + match retrieved_hashes with + | [] -> return @@ Bytes.concat Bytes.empty retrieved_contents + | hash :: hashes -> ( + let* serialized_page = retrieve_page_from_hash hash in + let*? page = deserialize_page serialized_page in + match page with + | Hashes page_hashes -> + (* Hashes are saved in reverse order. *) + (go [@tailcall]) + (List.rev_append page_hashes hashes) + retrieved_contents + | Contents contents -> + (* Because hashes are saved in reversed order, content pages + will be retrieved in reverse order. By always appending a + conetent page to the list of retrieved content pages, + we ensure that pages are saved in `retrieved_contents` in + their original order. *) + (go [@tailcall]) hashes (contents :: retrieved_contents)) + in + go [root_hash] [] + end + + module V0 = + Make + (struct + include Sc_rollup_reveal_hash + + let scheme = Sc_rollup_reveal_hash.Blake2B + end) + (Make_version (struct + (* Cntents_version_tag used in contents pages is 0. *) + let contents_version = 0 + + (* Hashes_version_tag used in hashes pages is 1. *) + let hashes_version = 0 + end)) +end + +module Hash_chain = struct + module type PAGE_FMT = sig + type h + + type page = {succ_hash : h; content : string} + + val content_limit : int + + val serialize_hash : h -> string + + (** Serializes a single page (an element in the hash link). *) + val serialize_page : page -> string + end + + module Make (Hashing_scheme : sig + include Dac_preimage_data_manager.REVEAL_HASH + + val scheme : supported_hashes + end) + (P : PAGE_FMT with type h = Hashing_scheme.t) = + struct + let hash bytes = + Hashing_scheme.hash_bytes ~scheme:Hashing_scheme.scheme [bytes] + + let to_hex = Hashing_scheme.to_hex + + let link_chunks chunks : (Hashing_scheme.t * bytes) list = + let rec link_chunks_rev linked_pages rev_pages = + match rev_pages with + | [] -> linked_pages + | chunk :: rev_chunks -> + let page = + match linked_pages with + | [] -> chunk + | (succ_hash, _) :: _ -> + P.serialize_page {succ_hash; content = chunk} + in + let page = Bytes.of_string page in + let hash = hash page in + (link_chunks_rev [@tailcall]) + ((hash, page) :: linked_pages) + rev_chunks + in + let rev_chunks = List.rev chunks in + link_chunks_rev [] rev_chunks + + let make_hash_chain data = + let open Result_syntax in + let+ chunks = String.chunk_bytes P.content_limit data in + link_chunks chunks + + (** Main function for computing a hash chain from a byte sequence. Returns the + chain head hash.[for_each_page] may be supplied to run post processing + tasks on each page, for example, to persisit a serialized page to disk. + *) + let serialize_payload ~for_each_page payload = + let open Lwt_result_syntax in + let* () = + fail_unless (Bytes.length payload > 0) Payload_cannot_be_empty + in + let*? hash_chain = make_hash_chain payload in + let+ () = List.iter_es for_each_page hash_chain in + Stdlib.List.hd hash_chain |> fst + end + + module V0 = + Make + (struct + include Sc_rollup_reveal_hash + + let scheme = Sc_rollup_reveal_hash.Blake2B + end) + (struct + type h = Sc_rollup_reveal_hash.t + + type page = {succ_hash : h; content : string} + + let content_limit = + (4 * 1024) - 100 (* We reserve 100 bytes for the continuation hash. *) + + let serialize_hash = Sc_rollup_reveal_hash.to_hex + + let serialize_page page = + Format.asprintf + "%s hash:%s" + page.content + (serialize_hash page.succ_hash) + end) +end diff --git a/src/proto_alpha/lib_dac/dac_plugin_registration.ml b/src/proto_alpha/lib_dac/dac_plugin_registration.ml index bb39651ba200..9cb87ec6180a 100644 --- a/src/proto_alpha/lib_dac/dac_plugin_registration.ml +++ b/src/proto_alpha/lib_dac/dac_plugin_registration.ml @@ -25,6 +25,7 @@ module Plugin = struct module Proto = Registerer.Registered + module RPC = RPC end let () = Dac_plugin.register (module Plugin) diff --git a/src/proto_alpha/lib_dac/dac_preimage_data_manager.ml b/src/proto_alpha/lib_dac/dac_preimage_data_manager.ml new file mode 100644 index 000000000000..70711afc5ab3 --- /dev/null +++ b/src/proto_alpha/lib_dac/dac_preimage_data_manager.ml @@ -0,0 +1,63 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2022 Trili Tech *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +open Protocol +open Environment.Error_monad + +type error += Cannot_write_dac_page_to_disk of string + +let () = + register_error_kind + `Permanent + ~id:"cannot_write_dac_page_to_disk" + ~title:"Cannot write Dac page to disk" + ~description:"Cannot write Dac page to disk" + ~pp:(fun ppf b58_hash -> + Format.fprintf ppf "Could not write dac page for hash %s" b58_hash) + Data_encoding.(obj1 (req "hash" string)) + (function + | Cannot_write_dac_page_to_disk b58_hash -> Some b58_hash | _ -> None) + (fun b58_hash -> Cannot_write_dac_page_to_disk b58_hash) + +module type REVEAL_HASH = module type of Sc_rollup_reveal_hash + +module Make (Hash : REVEAL_HASH) = struct + let path data_dir hash = + let hash = Hash.to_hex hash in + Filename.(concat data_dir hash) + + let save_bytes data_dir hash page_contents = + let open Lwt_result_syntax in + let path = path data_dir hash in + let*! result = + Lwt_utils_unix.with_atomic_open_out path @@ fun chan -> + Lwt_utils_unix.write_bytes chan page_contents + in + match result with + | Ok () -> return () + | Error _ -> tzfail @@ Cannot_write_dac_page_to_disk (Hash.to_hex hash) +end + +module Reveal_hash = Make (Sc_rollup_reveal_hash) diff --git a/src/proto_alpha/lib_dac/dac_signature_manager.ml b/src/proto_alpha/lib_dac/dac_signature_manager.ml new file mode 100644 index 000000000000..f0009dc71e87 --- /dev/null +++ b/src/proto_alpha/lib_dac/dac_signature_manager.ml @@ -0,0 +1,180 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2022 Trili Tech *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +open Protocol +open Environment +open Error_monad + +type error += + | Cannot_convert_root_page_hash_to_bytes of string + | Cannot_compute_aggregate_signature of string + | Public_key_for_witness_not_available of int * string + +let () = + register_error_kind + `Permanent + ~id:"cannot_extract_root_page_hash_to_sign" + ~title:"Cannot convert root hash page to byte sequence" + ~description:"Cannot convert root hash page to byte sequence" + ~pp:(fun ppf b58_hash -> + Format.fprintf + ppf + "Cannot convert root hash page to byte sequence: %s" + b58_hash) + Data_encoding.(obj1 (req "hash" (string Plain))) + (function + | Cannot_convert_root_page_hash_to_bytes b58_hash -> Some b58_hash + | _ -> None) + (fun b58_hash -> Cannot_convert_root_page_hash_to_bytes b58_hash) ; + register_error_kind + `Permanent + ~id:"cannot_compute_root_hash_aggregate_signature" + ~title:"Cannot compute aggregate signature of root page hash" + ~description:"Cannot compute aggregate signature of root page hash" + ~pp:(fun ppf b58_hash -> + Format.fprintf + ppf + "Cannot compute aggregate signature of root page hash: %s" + b58_hash) + Data_encoding.(obj1 (req "hash" (string Plain))) + (function + | Cannot_compute_aggregate_signature b58_hash -> Some b58_hash | _ -> None) + (fun b58_hash -> Cannot_compute_aggregate_signature b58_hash) ; + register_error_kind + `Permanent + ~id:"public_key_of_witness_not_available" + ~title: + "Public key of witness dac member not available for verifying signature" + ~description: + "Public key of witness dac member not available for verifying signature" + ~pp:(fun ppf (witness_index, b58_hash) -> + Format.fprintf + ppf + "Public key of dac member %d not available for verifying signature of \ + root page hash %s" + witness_index + b58_hash) + Data_encoding.(obj2 (req "witness_index" int31) (req "hash" (string Plain))) + (function + | Public_key_for_witness_not_available (index, hash) -> Some (index, hash) + | _ -> None) + (fun (index, hash) -> Public_key_for_witness_not_available (index, hash)) + +module type REVEAL_HASH = module type of Sc_rollup_reveal_hash + +module Make (Hashing_scheme : REVEAL_HASH) = struct + let bind_es (f : 'a -> 'b option tzresult Lwt.t) v_opt = + let open Lwt_result_syntax in + match v_opt with None -> return None | Some v -> f v + + let rev_collect_indexed_signatures cctxt dac_sk_uris bytes_to_sign = + let open Lwt_result_syntax in + List.rev_mapi_es + (fun index sk_uri_opt -> + (* TODO: https://gitlab.com/tezos/tezos/-/issues/4306 + Implement Option.bind_es and revisit this. *) + bind_es + (fun sk_uri -> + let*! signature_res = + Tezos_client_base.Client_keys.aggregate_sign + cctxt + sk_uri + bytes_to_sign + in + let signature_opt = Result.to_option signature_res in + return + @@ Option.map (fun signature -> (index, signature)) signature_opt) + sk_uri_opt) + dac_sk_uris + + let compute_signatures_with_witnesses rev_indexed_signatures = + let open Lwt_result_syntax in + List.fold_left_es + (fun (signatures, witnesses) signature_opt -> + match signature_opt with + | None -> return (signatures, witnesses) + | Some (index, signature) -> + let*? bitmap = Bitset.add witnesses index in + return (signature :: signatures, bitmap)) + ([], Bitset.empty) + rev_indexed_signatures + + let sign_root_hash cctxt dac_sk_uris root_hash = + let open Lwt_result_syntax in + let bytes_to_sign = + Data_encoding.Binary.to_bytes_opt Hashing_scheme.encoding root_hash + in + let root_hash = Hashing_scheme.to_hex root_hash in + match bytes_to_sign with + | None -> tzfail @@ Cannot_convert_root_page_hash_to_bytes root_hash + | Some bytes_to_sign -> ( + let* rev_indexed_signatures = + rev_collect_indexed_signatures cctxt dac_sk_uris bytes_to_sign + in + let* signatures, witnesses = + compute_signatures_with_witnesses rev_indexed_signatures + in + let final_signature = + Tezos_crypto.Aggregate_signature.aggregate_signature_opt signatures + in + match final_signature with + | None -> tzfail @@ Cannot_compute_aggregate_signature root_hash + | Some signature -> return @@ (signature, witnesses)) + + let verify ~public_keys_opt root_page_hash signature witnesses = + let open Lwt_result_syntax in + let hash_as_bytes = + Data_encoding.Binary.to_bytes_opt Hashing_scheme.encoding root_page_hash + in + match hash_as_bytes with + | None -> + tzfail + @@ Cannot_convert_root_page_hash_to_bytes + (Hashing_scheme.to_hex root_page_hash) + | Some bytes -> + let* pk_msg_list = + public_keys_opt + |> List.mapi (fun i public_key_opt -> (i, public_key_opt)) + |> List.filter_map_es (fun (i, public_key_opt) -> + let*? is_witness = Bitset.mem witnesses i in + + match public_key_opt with + | None -> + if is_witness then + tzfail + @@ Public_key_for_witness_not_available + (i, Hashing_scheme.to_hex root_page_hash) + else return None + | Some public_key -> + if is_witness then return @@ Some (public_key, None, bytes) + else return None) + in + return + @@ Tezos_crypto.Aggregate_signature.aggregate_check + pk_msg_list + signature +end + +module Reveal_hash = Make (Sc_rollup_reveal_hash) diff --git a/src/proto_alpha/lib_dac/test/main.ml b/src/proto_alpha/lib_dac/test/main.ml index 9450a26495ca..11bbabc8c5bb 100644 --- a/src/proto_alpha/lib_dac/test/main.ml +++ b/src/proto_alpha/lib_dac/test/main.ml @@ -44,4 +44,8 @@ end = struct let _skip unit_name test_cases = ("[SKIPPED] " ^ unit_name, test_cases) end -let () = Alcotest_lwt.run "protocol > unit" [] |> Lwt_main.run +let () = + Alcotest_lwt.run + "protocol > unit" + [Unit_test.spec "Dac_pages_encoding.ml" Test_dac_pages_encoding.tests] + |> Lwt_main.run diff --git a/src/proto_alpha/lib_dal/test/test_dac_pages_encoding.ml b/src/proto_alpha/lib_dac/test/test_dac_pages_encoding.ml similarity index 99% rename from src/proto_alpha/lib_dal/test/test_dac_pages_encoding.ml rename to src/proto_alpha/lib_dac/test/test_dac_pages_encoding.ml index c167e60f3d55..9460eb4b0600 100644 --- a/src/proto_alpha/lib_dal/test/test_dac_pages_encoding.ml +++ b/src/proto_alpha/lib_dac/test/test_dac_pages_encoding.ml @@ -26,7 +26,7 @@ (** Testing ------- Component: Dal_node Slot_frame_encoding - Invocation: dune exec src/proto_alpha/lib_dal/test/main.exe \ + Invocation: dune exec src/proto_alpha/lib_dac/test/main.exe \ -- test "^\[Unit\] Dac_pages_encoding.ml$" Subject: Tests for the SCORU storage module *) diff --git a/src/proto_alpha/lib_dal/test/main.ml b/src/proto_alpha/lib_dal/test/main.ml index e79b23050e17..16eba46a8df5 100644 --- a/src/proto_alpha/lib_dal/test/main.ml +++ b/src/proto_alpha/lib_dal/test/main.ml @@ -51,6 +51,5 @@ let () = Unit_test.spec "Slot_framing_protocol.ml" Test_dal_slot_frame_encoding.tests; - Unit_test.spec "Dac_pages_encoding.ml" Test_dac_pages_encoding.tests; ] |> Lwt_main.run -- GitLab From bfee51e194830185d8ceb86b85437b30848251d3 Mon Sep 17 00:00:00 2001 From: Andrea Cerone Date: Mon, 9 Jan 2023 16:54:14 +0000 Subject: [PATCH 5/6] Dal node: switch RPC server to Dac Plugin --- src/bin_dal_node/RPC_server.ml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/bin_dal_node/RPC_server.ml b/src/bin_dal_node/RPC_server.ml index 013937882e58..bc5d8b04d4a1 100644 --- a/src/bin_dal_node/RPC_server.ml +++ b/src/bin_dal_node/RPC_server.ml @@ -195,15 +195,16 @@ let start configuration cctxt ctxt dac_pks_opt dac_sk_uris = configuration in let dir = register ctxt in + (* TODO: https://gitlab.com/tezos/tezos/-/issues/4558 + Rename "plugin" prefix to "dac". + *) let plugin_prefix = Tezos_rpc.Path.(open_root / "plugin") in let dir = Tezos_rpc.Directory.register_dynamic_directory dir plugin_prefix (fun () -> match Node_context.get_status ctxt with - (* TODO: Replace this with the Dac plugin once the RPC server has - been moved there. *) - | Ready {dal_plugin = (module Plugin); _} -> + | Ready {dac_plugin = (module Dac_plugin); _} -> Lwt.return - (Plugin.RPC.rpc_services + (Dac_plugin.RPC.rpc_services ~reveal_data_dir cctxt dac_pks_opt -- GitLab From 0058dff6bbffd2981b8f97bb90d615c9f89e471f Mon Sep 17 00:00:00 2001 From: Andrea Cerone Date: Mon, 9 Jan 2023 17:01:43 +0000 Subject: [PATCH 6/6] Dal plugin: remove Dac RPC Server --- src/lib_dal_node/dal_plugin.ml | 10 - src/lib_dal_node/dal_plugin.mli | 10 - src/proto_alpha/lib_dal/RPC.ml | 199 ------ .../lib_dal/dac_external_message_manager.ml | 114 ---- src/proto_alpha/lib_dal/dac_manager.ml | 30 - src/proto_alpha/lib_dal/dac_pages_encoding.ml | 636 ------------------ .../lib_dal/dac_preimage_data_manager.ml | 63 -- .../lib_dal/dac_signature_manager.ml | 180 ----- .../lib_dal/dal_plugin_registration.ml | 2 - 9 files changed, 1244 deletions(-) delete mode 100644 src/proto_alpha/lib_dal/RPC.ml delete mode 100644 src/proto_alpha/lib_dal/dac_external_message_manager.ml delete mode 100644 src/proto_alpha/lib_dal/dac_manager.ml delete mode 100644 src/proto_alpha/lib_dal/dac_pages_encoding.ml delete mode 100644 src/proto_alpha/lib_dal/dac_preimage_data_manager.ml delete mode 100644 src/proto_alpha/lib_dal/dac_signature_manager.ml diff --git a/src/lib_dal_node/dal_plugin.ml b/src/lib_dal_node/dal_plugin.ml index db134b0522e0..8319ceca61a1 100644 --- a/src/lib_dal_node/dal_plugin.ml +++ b/src/lib_dal_node/dal_plugin.ml @@ -74,16 +74,6 @@ module type T = sig block_info -> number_of_slots:int -> slot_index list tzresult - - module RPC : sig - val rpc_services : - reveal_data_dir:string -> - #Client_context.wallet -> - Tezos_crypto.Aggregate_signature.public_key option list -> - Client_keys.aggregate_sk_uri option list -> - int -> - unit Tezos_rpc.Directory.directory - end end let table : (module T) Tezos_crypto.Protocol_hash.Table.t = diff --git a/src/lib_dal_node/dal_plugin.mli b/src/lib_dal_node/dal_plugin.mli index 230294a2c4d7..31b430af3f00 100644 --- a/src/lib_dal_node/dal_plugin.mli +++ b/src/lib_dal_node/dal_plugin.mli @@ -100,16 +100,6 @@ module type T = sig block_info -> number_of_slots:int -> slot_index list tzresult - - module RPC : sig - val rpc_services : - reveal_data_dir:string -> - #Client_context.wallet -> - Tezos_crypto.Aggregate_signature.public_key option list -> - Client_keys.aggregate_sk_uri option list -> - int -> - unit Tezos_rpc.Directory.directory - end end val register : (module T) -> unit diff --git a/src/proto_alpha/lib_dal/RPC.ml b/src/proto_alpha/lib_dal/RPC.ml deleted file mode 100644 index 1cda639b25ef..000000000000 --- a/src/proto_alpha/lib_dal/RPC.ml +++ /dev/null @@ -1,199 +0,0 @@ -(*****************************************************************************) -(* *) -(* Open Source License *) -(* Copyright (c) 2022 Trili Tech *) -(* *) -(* Permission is hereby granted, free of charge, to any person obtaining a *) -(* copy of this software and associated documentation files (the "Software"),*) -(* to deal in the Software without restriction, including without limitation *) -(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) -(* and/or sell copies of the Software, and to permit persons to whom the *) -(* Software is furnished to do so, subject to the following conditions: *) -(* *) -(* The above copyright notice and this permission notice shall be included *) -(* in all copies or substantial portions of the Software. *) -(* *) -(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) -(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) -(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) -(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) -(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) -(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) -(* DEALINGS IN THE SOFTWARE. *) -(* *) -(*****************************************************************************) - -open Environment -open Error_monad - -type error += - | Cannot_construct_external_message - | Cannot_deserialize_external_message - -let () = - register_error_kind - `Permanent - ~id:"dac_cannot_construct_external_message" - ~title:"External rollup message could not be constructed" - ~description:"External rollup message could not be constructed" - ~pp:(fun ppf () -> - Format.fprintf ppf "External rollup message could not be constructed") - Data_encoding.unit - (function Cannot_construct_external_message -> Some () | _ -> None) - (fun () -> Cannot_construct_external_message) ; - register_error_kind - `Permanent - ~id:"dac_cannot_deserialize_rollup_external_message" - ~title:"External rollup message could not be deserialized" - ~description:"External rollup message could not be deserialized" - ~pp:(fun ppf () -> - Format.fprintf ppf "External rollup message could not be deserialized") - Data_encoding.unit - (function Cannot_deserialize_external_message -> Some () | _ -> None) - (fun () -> Cannot_deserialize_external_message) - -module Registration = struct - let register0_noctxt ~chunked s f dir = - RPC_directory.register ~chunked dir s (fun _rpc_ctxt q i -> f q i) -end - -module DAC = struct - module Hash_storage = Dac_preimage_data_manager.Reveal_hash - - let store_preimage_request_encoding = - Data_encoding.( - obj2 - (req "payload" Data_encoding.(bytes Hex)) - (req "pagination_scheme" Dac_pages_encoding.pagination_scheme_encoding)) - - (* A variant of [Sc_rollup_reveal_hash.encoding] that prefers hex - encoding over b58check encoding for JSON. *) - let root_hash_encoding = - let binary = Protocol.Sc_rollup_reveal_hash.encoding in - Data_encoding.( - splitted - ~binary - ~json: - (conv_with_guard - Protocol.Sc_rollup_reveal_hash.to_hex - (fun str -> - Result.of_option - ~error:"Not a valid hash" - (Protocol.Sc_rollup_reveal_hash.of_hex str)) - (string Plain))) - - let store_preimage_response_encoding = - Data_encoding.( - obj2 - (req "root_hash" root_hash_encoding) - (req "external_message" (bytes Hex))) - - let external_message_query = - let open RPC_query in - query (fun hex_string -> hex_string) - |+ opt_field "external_message" RPC_arg.string (fun s -> s) - |> seal - - module S = struct - let dac_store_preimage = - RPC_service.put_service - ~description:"Split DAC reveal data" - ~query:RPC_query.empty - ~input:store_preimage_request_encoding - ~output:store_preimage_response_encoding - RPC_path.(open_root / "dac" / "store_preimage") - - (* DAC/FIXME: https://gitlab.com/tezos/tezos/-/issues/4263 - remove this endpoint once end-to-end tests are in place. *) - let verify_external_message_signature = - RPC_service.get_service - ~description:"Verify signature of an external message to inject in L1" - ~query:external_message_query - ~output:Data_encoding.bool - RPC_path.(open_root / "dac" / "verify_signature") - end - - let handle_serialize_dac_store_preimage cctxt dac_sk_uris reveal_data_dir - (data, pagination_scheme) = - let open Lwt_result_syntax in - let open Dac_pages_encoding in - let for_each_page (hash, page_contents) = - Dac_manager.Reveal_hash.Storage.save_bytes - reveal_data_dir - hash - page_contents - in - let* root_hash = - match pagination_scheme with - | Merkle_tree_V0 -> - let size = - Protocol.Alpha_context.Constants.sc_rollup_message_size_limit - in - Merkle_tree.V0.serialize_payload - ~max_page_size:size - data - ~for_each_page - | Hash_chain_V0 -> Hash_chain.V0.serialize_payload ~for_each_page data - in - let* signature, witnesses = - Dac_manager.Reveal_hash.Signatures.sign_root_hash - cctxt - dac_sk_uris - root_hash - in - let*? external_message = - match - Dac_manager.Reveal_hash.External_message.make - root_hash - signature - witnesses - with - | Ok external_message -> Ok external_message - | Error _ -> Error_monad.error Cannot_construct_external_message - in - return (root_hash, external_message) - - let handle_verify_external_message_signature public_keys_opt - encoded_l1_message = - let open Lwt_result_syntax in - let open Dac_manager.Reveal_hash in - let external_message = - let open Option_syntax in - let* encoded_l1_message = encoded_l1_message in - let* as_bytes = Hex.to_bytes @@ `Hex encoded_l1_message in - External_message.of_bytes as_bytes - in - match external_message with - | None -> tzfail @@ Cannot_deserialize_external_message - | Some (External_message.Dac_message {root_hash; signature; witnesses}) -> - Signatures.verify ~public_keys_opt root_hash signature witnesses - - let register_serialize_dac_store_preimage cctxt dac_sk_uris reveal_data_dir = - Registration.register0_noctxt - ~chunked:false - S.dac_store_preimage - (fun () input -> - handle_serialize_dac_store_preimage - cctxt - dac_sk_uris - reveal_data_dir - input) - - let register_verify_external_message_signature public_keys_opt = - Registration.register0_noctxt - ~chunked:false - S.verify_external_message_signature - (fun external_message () -> - handle_verify_external_message_signature - public_keys_opt - external_message) - - let register reveal_data_dir cctxt dac_public_keys_opt dac_sk_uris = - (RPC_directory.empty : unit RPC_directory.t) - |> register_serialize_dac_store_preimage cctxt dac_sk_uris reveal_data_dir - |> register_verify_external_message_signature dac_public_keys_opt -end - -let rpc_services ~reveal_data_dir cctxt dac_public_keys_opt dac_sk_uris - _threshold = - DAC.register reveal_data_dir cctxt dac_public_keys_opt dac_sk_uris diff --git a/src/proto_alpha/lib_dal/dac_external_message_manager.ml b/src/proto_alpha/lib_dal/dac_external_message_manager.ml deleted file mode 100644 index 56b6557e296d..000000000000 --- a/src/proto_alpha/lib_dal/dac_external_message_manager.ml +++ /dev/null @@ -1,114 +0,0 @@ -(*****************************************************************************) -(* *) -(* Open Source License *) -(* Copyright (c) 2022 Trili Tech *) -(* *) -(* Permission is hereby granted, free of charge, to any person obtaining a *) -(* copy of this software and associated documentation files (the "Software"),*) -(* to deal in the Software without restriction, including without limitation *) -(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) -(* and/or sell copies of the Software, and to permit persons to whom the *) -(* Software is furnished to do so, subject to the following conditions: *) -(* *) -(* The above copyright notice and this permission notice shall be included *) -(* in all copies or substantial portions of the Software. *) -(* *) -(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) -(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) -(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) -(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) -(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) -(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) -(* DEALINGS IN THE SOFTWARE. *) -(* *) -(*****************************************************************************) - -open Protocol -open Environment.Error_monad - -type error += Could_not_serialize_rollup_external_message of string - -let () = - register_error_kind - `Permanent - ~id:"dac_could_not_serialize_rollup_external_message" - ~title:"Could not serialize rollup external message" - ~description: - "Serialization of rollup external message containing Dac root page hash \ - failed" - ~pp:(fun ppf b58_hash -> - Format.fprintf - ppf - "Serialization of rollup external message containing Dac root page \ - hash %sfailed" - b58_hash) - Data_encoding.(obj1 (req "hash" string)) - (function - | Could_not_serialize_rollup_external_message b58_hash -> Some b58_hash - | _ -> None) - (fun b58_hash -> Could_not_serialize_rollup_external_message b58_hash) - -module type REVEAL_HASH = module type of Sc_rollup_reveal_hash - -module Make - (Hashing_scheme : REVEAL_HASH) (Encoding_metadata : sig - val tag : int - - val title : string - end) = -struct - type dac_message = - | Dac_message of { - root_hash : Hashing_scheme.t; - signature : Tezos_crypto.Aggregate_signature.t; - witnesses : Bitset.t; - } - - let untagged_encoding = - Data_encoding.( - conv - (function - | Dac_message {root_hash; signature; witnesses} -> - (root_hash, signature, witnesses)) - (fun (root_hash, signature, witnesses) -> - Dac_message {root_hash; signature; witnesses}) - (obj3 - (req "root_hash" Hashing_scheme.encoding) - (req "signature" Tezos_crypto.Aggregate_signature.encoding) - (req "witnesses" Bitset.encoding))) - - let dac_message_encoding = - Data_encoding.( - union - ~tag_size:`Uint8 - [ - case - ~title:("dac_message_" ^ Encoding_metadata.title) - (Tag Encoding_metadata.tag) - untagged_encoding - (fun msg -> Some msg) - (fun msg -> msg); - ]) - - let make root_hash signature witnesses = - let message = Dac_message {root_hash; signature; witnesses} in - let res = Data_encoding.Binary.to_bytes dac_message_encoding message in - match res with - | Ok bytes -> Ok bytes - | Error _ -> - error - @@ Could_not_serialize_rollup_external_message - (Hashing_scheme.to_hex root_hash) - - let of_bytes encoded_message = - Data_encoding.Binary.of_bytes_opt dac_message_encoding encoded_message -end - -module Reveal_hash = - Make - (Sc_rollup_reveal_hash) - (struct - let tag = 42 - - let title = "reveal_hash_v0" - end) diff --git a/src/proto_alpha/lib_dal/dac_manager.ml b/src/proto_alpha/lib_dal/dac_manager.ml deleted file mode 100644 index cbf068e1ffdd..000000000000 --- a/src/proto_alpha/lib_dal/dac_manager.ml +++ /dev/null @@ -1,30 +0,0 @@ -(*****************************************************************************) -(* *) -(* Open Source License *) -(* Copyright (c) 2022 Trili Tech *) -(* *) -(* Permission is hereby granted, free of charge, to any person obtaining a *) -(* copy of this software and associated documentation files (the "Software"),*) -(* to deal in the Software without restriction, including without limitation *) -(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) -(* and/or sell copies of the Software, and to permit persons to whom the *) -(* Software is furnished to do so, subject to the following conditions: *) -(* *) -(* The above copyright notice and this permission notice shall be included *) -(* in all copies or substantial portions of the Software. *) -(* *) -(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) -(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) -(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) -(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) -(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) -(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) -(* DEALINGS IN THE SOFTWARE. *) -(* *) -(*****************************************************************************) - -module Reveal_hash = struct - module Storage = Dac_preimage_data_manager.Reveal_hash - module Signatures = Dac_signature_manager.Reveal_hash - module External_message = Dac_external_message_manager.Reveal_hash -end diff --git a/src/proto_alpha/lib_dal/dac_pages_encoding.ml b/src/proto_alpha/lib_dal/dac_pages_encoding.ml deleted file mode 100644 index 99a7ed2ec3d2..000000000000 --- a/src/proto_alpha/lib_dal/dac_pages_encoding.ml +++ /dev/null @@ -1,636 +0,0 @@ -(*****************************************************************************) -(* *) -(* Open Source License *) -(* Copyright (c) 2022 Trili Tech *) -(* Copyright (c) 2022 Marigold *) -(* *) -(* Permission is hereby granted, free of charge, to any person obtaining a *) -(* copy of this software and associated documentation files (the "Software"),*) -(* to deal in the Software without restriction, including without limitation *) -(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) -(* and/or sell copies of the Software, and to permit persons to whom the *) -(* Software is furnished to do so, subject to the following conditions: *) -(* *) -(* The above copyright notice and this permission notice shall be included *) -(* in all copies or substantial portions of the Software. *) -(* *) -(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) -(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) -(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) -(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) -(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) -(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) -(* DEALINGS IN THE SOFTWARE. *) -(* *) -(*****************************************************************************) - -(* DAC/FIXME: https://gitlab.com/tezos/tezos/-/issues/4088 - Add .mli file. *) - -(** Library for encoding payloads of arbitrary size in formats that can be - decoded by the Sc-rollup kernels. - *) - -open Protocol -open Environment.Error_monad - -type error += - | Payload_cannot_be_empty - | Cannot_serialize_page_payload - | Cannot_deserialize_page - | Non_positive_size_of_payload - | Merkle_tree_branching_factor_not_high_enough - | Cannot_combine_pages_data_of_different_type - | Hashes_page_repr_expected_single_element - -type pagination_scheme = Merkle_tree_V0 | Hash_chain_V0 - -let pagination_scheme_encoding = - Data_encoding.string_enum - [("Merkle_tree_V0", Merkle_tree_V0); ("Hash_chain_V0", Hash_chain_V0)] - -let () = - register_error_kind - `Permanent - ~id:"cannot_deserialize_dac_page_payload" - ~title:"DAC payload could not be deserialized" - ~description:"Error when recovering DAC payload payload from binary" - ~pp:(fun ppf () -> - Format.fprintf - ppf - "Error when recovering DAC payload from list of data chunks") - Data_encoding.(unit) - (function Cannot_deserialize_page -> Some () | _ -> None) - (fun () -> Cannot_deserialize_page) ; - register_error_kind - `Permanent - ~id:"cannot_serialize_dac_page" - ~title:"DAC page could not be serialized" - ~description:"Error when serializing DAC page" - ~pp:(fun ppf () -> Format.fprintf ppf "Error when serializing DAC page") - Data_encoding.(unit) - (function Cannot_serialize_page_payload -> Some () | _ -> None) - (fun () -> Cannot_serialize_page_payload) ; - register_error_kind - `Permanent - ~id:"non_positive_payload_size" - ~title:"Non positive size for dac payload" - ~description:"Dac page payload (excluded preamble) are non positive" - ~pp:(fun ppf () -> - Format.fprintf ppf "Dac page payload (excluded preamble) are non positive") - Data_encoding.(unit) - (function Non_positive_size_of_payload -> Some () | _ -> None) - (fun () -> Non_positive_size_of_payload) ; - register_error_kind - `Permanent - ~id:"dac_payload_cannot_be_empty" - ~title:"Cannot serialize empty DAC payload" - ~description:"Cannot serialize empty DAC payload" - ~pp:(fun ppf () -> Format.fprintf ppf "Cannot serialize empty DAC payload") - Data_encoding.(unit) - (function Payload_cannot_be_empty -> Some () | _ -> None) - (fun () -> Payload_cannot_be_empty) ; - register_error_kind - `Permanent - ~id:"merkle_tree_branching_factor_not_high_enough" - ~title:"Merkle tree branching factor must be at least 2" - ~description:"Merkle tree branching factor must be at least 2" - ~pp:(fun ppf () -> - Format.fprintf - ppf - "Cannot serialize DAC payload: pages must be able to contain at least \ - two hashes") - Data_encoding.(unit) - (function - | Merkle_tree_branching_factor_not_high_enough -> Some () | _ -> None) - (fun () -> Merkle_tree_branching_factor_not_high_enough) ; - register_error_kind - `Permanent - ~id:"cannot_combine_pages_data_of_different_type" - ~title:"Cannot combine pages data of different type" - ~description:"Cannot combine pages data of different type" - ~pp:(fun ppf () -> - Format.fprintf - ppf - "Merkle level serizalization: when adding page data to a given level \ - repr., the type of page data received is different then the one \ - inside a level repr.") - Data_encoding.(unit) - (function - | Cannot_combine_pages_data_of_different_type -> Some () | _ -> None) - (fun () -> Cannot_combine_pages_data_of_different_type) ; - register_error_kind - `Permanent - ~id:"hashes_page_repr_expected_single_element" - ~title:"Hashes page representation expected a single element" - ~description:"Hashes page representation expected a single element" - ~pp:(fun ppf () -> - Format.fprintf ppf "Hashes page representation expected a single element") - Data_encoding.unit - (function Hashes_page_repr_expected_single_element -> Some () | _ -> None) - (fun () -> Hashes_page_repr_expected_single_element) - -(** Encoding of DAC payload as a Merkle tree with an arbitrary branching - factor greater or equal to 2. The serialization process works as follows: - {ul - {li A large sequence of bytes, the payload, is split into several pages - of fixed size, each of which is prefixed with a small sequence - of bytes (also of fixed size), which is referred to as the preamble - of the page. Pages obtained directly from the original payload - are referred to as `Contents pages`. Contents pages constitute the - leaves of the Merkle tree being built, - } - {li Each contents page (each of which is a sequence of bytes consisting - of the preamble followed by the actual contents from the original - payload) is then hashed. The size of each hash is fixed. The hashes are - concatenated together, and the resulting sequence of bytes is split - into pages of the same size of `Hashes pages`, each of which is - prefixed with a preamble whose size is the same as in Contents pages. - Hashes pages correspond to nodes of the Merkle tree being built, and - the children of a hash page are the (either Payload or Hashes) pages - whose hash appear into the former, - } - {li Hashes pages are hashed using the same process described above, leading - to a smaller list of hashes pages. To guarantee that the list of hashes - pages is actually smaller than the original list of pages being hashed, - we require the size of pages to be large enough to contain at least two - hashes. - } - } - - Merkle tree encodings of DAC pages are versioned, to allow for multiple - hashing schemes to be used. - *) -module Merkle_tree = struct - type version = int - - (** A page is either a `Contents page`, containing a chunk of the payload - that needs to be serialized, or a `Hashes page`, containing a list - of hashes. The maximum size of bytes inside [Contents] page, or number - of hashes inside [Hashes] page is such, that when serializing a page - using [page_encoding], it does not exceed [max_page_size] bytes. - *) - type 'a page = Contents of bytes | Hashes of 'a list - - let max_version = 127 - - module type VERSION = sig - val contents_version_tag : version - - val hashes_version_tag : version - end - - (* Even numbers are used for versioning Contents pages, odd numbers are used - for versioning Hashes pages. *) - module Make_version (V : sig - val contents_version : int - - val hashes_version : int - end) = - struct - let contents_version_tag = 2 * V.contents_version - - let hashes_version_tag = (2 * V.hashes_version) + 1 - end - - module Make (Hashing_scheme : sig - include Dac_preimage_data_manager.REVEAL_HASH - - val scheme : supported_hashes - end) - (V : VERSION) = - struct - let hash bytes = - Hashing_scheme.hash_bytes [bytes] ~scheme:Hashing_scheme.scheme - - let hash_encoding = Hashing_scheme.encoding - - let hashes_encoding = Data_encoding.list hash_encoding - - (* The preamble of a serialized page contains 1 byte denoting the version, - and 4 bytes encoding the size of the rest of the page. In total, 5 - bytes. *) - let page_preamble_size = 5 - - let hash_bytes_size = Hashing_scheme.size ~scheme:Hashing_scheme.scheme - - (** Payload pages are encoded as follows: the first byte is an integer, - which is corresponds to either `payload_version` (for payload pages) or - `hashes_version` (for hashes pages). The next four bytes will contain - the size of the rest of the page; the remainder of the page is either a - list of raw bytes (in the case of a payload page), or a list of hashes, - which occupy 32 bytes each. *) - let page_encoding = - Data_encoding.( - union - ~tag_size:`Uint8 - [ - case - ~title:"contents" - (Tag V.contents_version_tag) - bytes - (function Contents payload -> Some payload | _ -> None) - (fun payload -> Contents payload); - case - ~title:"hashes" - (Tag V.hashes_version_tag) - hashes_encoding - (function Hashes hashes -> Some hashes | _ -> None) - (fun hashes -> Hashes hashes); - ]) - - (** Serialization function for a single page. It converts a page to a - sequence of bytes using [page_encoding]. It also checks that the - serialized page does not exceed [page_size] bytes. *) - let serialize_page ~max_page_size page = - match - Data_encoding.Binary.to_bytes - (Data_encoding.check_size max_page_size page_encoding) - page - with - | Ok raw_page -> Ok raw_page - | Error _ -> error Cannot_serialize_page_payload - - (* Splits payload into bytes chunks whose size does not exceed [page_size] bytes. *) - let split_payload ~max_page_size payload = - let open Result_syntax in - (* 1 byte for the version size, 4 bytes for the size of the payload. *) - let actual_page_size = max_page_size - page_preamble_size in - if actual_page_size <= 0 then error Non_positive_size_of_payload - else - let+ splitted_payload = String.chunk_bytes actual_page_size payload in - List.map (fun cont -> Contents (String.to_bytes cont)) splitted_payload - - let max_hashes_per_page ~max_page_size = - (max_page_size - page_preamble_size) / hash_bytes_size - - let split_hashes ~max_page_size hashes = - let number_of_hashes = max_hashes_per_page ~max_page_size in - (* Requiring a branching factor of at least 2 is necessary to ensure that - the serialization process terminates. If only one hash were stored per page, then - the number of pages at height `n-1` could be potentially equal to the number of - pages at height `n`. *) - if number_of_hashes < 2 then - error Merkle_tree_branching_factor_not_high_enough - else - let rec go aux list = - match list with - | [] -> List.rev aux - | list -> - let chunk, rest = List.split_n number_of_hashes list in - (go [@tailcall]) (Hashes chunk :: aux) rest - in - Ok (go [] hashes) - - let store_page ~max_page_size ~for_each_page page = - let open Lwt_result_syntax in - let*? serialized_page = serialize_page ~max_page_size page in - (* Hashes are computed from raw pages, each of which consists of a - preamble of 5 bytes followed by a page payload - a raw sequence - of bytes from the original payload for Contents pages, and a - a sequence of serialized hashes for hashes pages. The preamble - bytes is part of the sequence of bytes which is hashed. - *) - let hash = hash serialized_page in - let* () = for_each_page (hash, serialized_page) in - return hash - - (** [Payload_handler] is in-memory data structure that enables aggregating - DAC messages/payload. It serializes the data by respecting [page_encoding], - by storing the full payload in the shape of a k-ary merkle tree onto the disk. - During the proccess of partial serialization, the minimum amount of data - is kept in memory, by eagerly persisting the full pages of a given merkle - level whenever possible, to ease the load on the memory. - - The serializer respects the following invariant: - Starting with an [empty] serializer, splitting arbitrary [payload] into - chunks of arbitrary size, and adding them to the serializer from left to - right, should result in same root hash as adding all the payload data in - one chunk, provided it could fit into the memory. - *) - module Payload_handler = struct - (** A [page_data] is either [Cont_data] or [Hash_data] where each represents - data not bound in size for [Contents] or [Hashes] page respectively. *) - type page_data = Cont_data of bytes | Hash_data of Hashing_scheme.t list - - let is_empty_page_data = function - | Cont_data cont -> Bytes.empty = cont - | Hash_data ls -> List.is_empty ls - - (** [Payload_handler] serializes DAC data in the shape of k-ary Merkle tree - onto the disk. For every existing level written onto to the disk, handler - holds a corresponding [Merkle_level.t] in-memory representation. This - representation holds the data yet to be persisted to the disk - effectively - acting as buffer, making sure that all [pages] of the given level that - are written to the disk are full (with the exception of last one upon - finalization). - *) - module Merkle_level = struct - type t = page_data - - let init_level page_data = page_data - - let is_empty level = is_empty_page_data level - - let has_single_hash = function Hash_data [_] -> true | _ -> false - - let get_single_hash level = - let open Lwt_result_syntax in - match level with - | Hash_data [x] -> return x - | _ -> tzfail Hashes_page_repr_expected_single_element - - let split ~max_page_size page_data = - let leftover_with_full_pages ls = - let open Lwt_result_syntax in - match List.rev ls with - | [] -> - tzfail Payload_cannot_be_empty (* We don't expect to get here *) - | Contents h :: xs -> return (Cont_data h, List.rev xs) - | Hashes h :: xs -> return (Hash_data h, List.rev xs) - in - let open Lwt_result_syntax in - let*? pages = - match page_data with - | Cont_data cont -> split_payload ~max_page_size cont - | Hash_data ls -> split_hashes ~max_page_size ls - in - leftover_with_full_pages pages - - let add_page_data ~level ~page_data = - let open Lwt_result_syntax in - match (level, page_data) with - | Cont_data level, Cont_data page_data -> - return @@ Cont_data (Bytes.cat level page_data) - | Hash_data level, Hash_data page_data -> - return @@ Hash_data (List.append level page_data) - | Hash_data _, Cont_data _ | Cont_data _, Hash_data _ -> - (* We dont expect to get here *) - tzfail Cannot_combine_pages_data_of_different_type - - let process ~max_page_size ~for_each_page level = - let open Lwt_result_syntax in - let* level, pages = split ~max_page_size level in - let* rev_hashes = - List.fold_left_es - (fun accm page -> - let* hash = store_page ~max_page_size ~for_each_page page in - return @@ (hash :: accm)) - [] - pages - in - return (level, Hash_data (List.rev rev_hashes)) - end - - let empty : Merkle_level.t Stack.t = Stack.create () - - (** Adds a [page_data] to the bottom level of the in-memory merkle tree repr. - If [page_data] inside level exceeds its [max_page_size], then we split it - into valid full [page]/s and a leftover that is smaller or equal to - [max_page_size]. - - Full pages are eagerly stored to disk and parent hashes are added - in batch to next level. The leftover represents the content - of current level. Procedure repeats recursively if necessarily. - - Throws [Payload_cannot_be_empty] if [page_data] is empty. - *) - let rec add_rec ~max_page_size ~for_each_page stack page_data = - let open Lwt_result_syntax in - let open Merkle_level in - let* merkle_level = - if Stack.is_empty stack then return (init_level page_data) - else add_page_data ~level:(Stack.pop stack) ~page_data - in - let* merkle_level, page_data = - process ~max_page_size ~for_each_page merkle_level - in - let* () = - if not @@ is_empty_page_data page_data then - add_rec ~max_page_size ~for_each_page stack page_data - else return () - in - return @@ Stack.push merkle_level stack - - (** Adding [payload] to the serializer does not guarantee its persistance - to disk. The data is fully serialized after the call to [finalize]. - This guarantees that only the last [page] in the given level could be - partially filled. - - Throws [Payload_cannot_be_empty] in case of empty payload *) - let add ~max_page_size ~for_each_page stack payload = - let open Lwt_result_syntax in - let* () = - fail_unless (Bytes.length payload > 0) Payload_cannot_be_empty - in - let number_of_hashes = max_hashes_per_page ~max_page_size in - (* Requiring a branching factor of at least 2 is necessary to ensure that - the serialization process terminates. If only one hash were stored per page, then - the number of pages at height `n-1` could be potentially equal to the number of - pages at height `n`. *) - let* () = - fail_unless - (number_of_hashes > 1) - Merkle_tree_branching_factor_not_high_enough - in - add_rec ~max_page_size ~for_each_page stack (Cont_data payload) - - (** [finalize] returns a root hash of agggregated data, by eagerly storing - every partially filled level on the stack. Starting at the bottom and - adding stored page hash to next level, repeating this procedure until - left with a single (root) hash. *) - let rec finalize ~max_page_size ~for_each_page stack = - let open Merkle_level in - let finalize_level page_data = - let store_page = store_page ~max_page_size ~for_each_page in - match page_data with - | Cont_data cont -> store_page (Contents cont) - | Hash_data ls -> store_page (Hashes ls) - in - let open Lwt_result_syntax in - let* () = - (* Empty stack means no payload added in the first place *) - fail_unless (not @@ Stack.is_empty stack) Payload_cannot_be_empty - in - let merkle_level = Stack.pop stack in - if is_empty merkle_level then - (* If level is empty, there is nothing to hash *) - finalize ~max_page_size ~for_each_page stack - else if Stack.is_empty stack && has_single_hash merkle_level then - (* If top level of the tree and only one hash, then this is a root hash *) - get_single_hash merkle_level - else - let* hash = finalize_level merkle_level in - let* () = - add_rec ~max_page_size ~for_each_page stack (Hash_data [hash]) - in - finalize ~max_page_size ~for_each_page stack - end - - (** Main function for computing the pages of a Merkle tree from a sequence - of bytes. Each page is processed using the function [for_each_page] - provided in input, which is responsible for ensuring that the original - payload can be reconstructed from the Merkle tree root; this can be - achieved, for example, by letting [for_each_page] persist a serialized - page to disk using the page hash as its filename. - The function [serialize_payload] returns the root hash of the Merkle - tree constructed. *) - let serialize_payload ~max_page_size payload ~for_each_page = - let open Lwt_result_syntax in - let open Payload_handler in - let stack = empty in - let* () = add ~max_page_size ~for_each_page stack payload in - finalize ~max_page_size ~for_each_page stack - - (** Deserialization function for a single page. A sequence of bytes is - converted to a page using [page_encoding]. *) - let deserialize_page raw_page = - match Data_encoding.Binary.of_bytes page_encoding raw_page with - | Ok page -> Ok page - | Error _ -> error Cannot_deserialize_page - - (** Deserialization function for reconstructing the original payload from - its Merkle tree root hash. The function [retrieve_page_from_hash] - passed in input is responsible for determining how to retrieve the - serialized page from its hash. For example, if the page has been - persisted to disk using the page hash as its filename, - [retrieve_page_from_hash] simply loads the corresponding file from - disk to memory. The function [deserialize_payload] returns the - original payload that was used to compute the Merkle tree root hash. - This function is guaranteed to terminate if the directed graph induced - by the retrieved pages (that is, the graph where there is an edge - from one page to another if and only if the former contains the hash - of the latter) is acyclic. This property is guaranteed if the root - hash and pages are computed using the serialized_payload function - outlined above, but it is not guaranteed in more general cases. - *) - let deserialize_payload root_hash ~retrieve_page_from_hash = - let rec go retrieved_hashes retrieved_contents = - let open Lwt_result_syntax in - match retrieved_hashes with - | [] -> return @@ Bytes.concat Bytes.empty retrieved_contents - | hash :: hashes -> ( - let* serialized_page = retrieve_page_from_hash hash in - let*? page = deserialize_page serialized_page in - match page with - | Hashes page_hashes -> - (* Hashes are saved in reverse order. *) - (go [@tailcall]) - (List.rev_append page_hashes hashes) - retrieved_contents - | Contents contents -> - (* Because hashes are saved in reversed order, content pages - will be retrieved in reverse order. By always appending a - conetent page to the list of retrieved content pages, - we ensure that pages are saved in `retrieved_contents` in - their original order. *) - (go [@tailcall]) hashes (contents :: retrieved_contents)) - in - go [root_hash] [] - end - - module V0 = - Make - (struct - include Sc_rollup_reveal_hash - - let scheme = Sc_rollup_reveal_hash.Blake2B - end) - (Make_version (struct - (* Cntents_version_tag used in contents pages is 0. *) - let contents_version = 0 - - (* Hashes_version_tag used in hashes pages is 1. *) - let hashes_version = 0 - end)) -end - -module Hash_chain = struct - module type PAGE_FMT = sig - type h - - type page = {succ_hash : h; content : string} - - val content_limit : int - - val serialize_hash : h -> string - - (** Serializes a single page (an element in the hash link). *) - val serialize_page : page -> string - end - - module Make (Hashing_scheme : sig - include Dac_preimage_data_manager.REVEAL_HASH - - val scheme : supported_hashes - end) - (P : PAGE_FMT with type h = Hashing_scheme.t) = - struct - let hash bytes = - Hashing_scheme.hash_bytes ~scheme:Hashing_scheme.scheme [bytes] - - let to_hex = Hashing_scheme.to_hex - - let link_chunks chunks : (Hashing_scheme.t * bytes) list = - let rec link_chunks_rev linked_pages rev_pages = - match rev_pages with - | [] -> linked_pages - | chunk :: rev_chunks -> - let page = - match linked_pages with - | [] -> chunk - | (succ_hash, _) :: _ -> - P.serialize_page {succ_hash; content = chunk} - in - let page = Bytes.of_string page in - let hash = hash page in - (link_chunks_rev [@tailcall]) - ((hash, page) :: linked_pages) - rev_chunks - in - let rev_chunks = List.rev chunks in - link_chunks_rev [] rev_chunks - - let make_hash_chain data = - let open Result_syntax in - let+ chunks = String.chunk_bytes P.content_limit data in - link_chunks chunks - - (** Main function for computing a hash chain from a byte sequence. Returns the - chain head hash.[for_each_page] may be supplied to run post processing - tasks on each page, for example, to persisit a serialized page to disk. - *) - let serialize_payload ~for_each_page payload = - let open Lwt_result_syntax in - let* () = - fail_unless (Bytes.length payload > 0) Payload_cannot_be_empty - in - let*? hash_chain = make_hash_chain payload in - let+ () = List.iter_es for_each_page hash_chain in - Stdlib.List.hd hash_chain |> fst - end - - module V0 = - Make - (struct - include Sc_rollup_reveal_hash - - let scheme = Sc_rollup_reveal_hash.Blake2B - end) - (struct - type h = Sc_rollup_reveal_hash.t - - type page = {succ_hash : h; content : string} - - let content_limit = - (4 * 1024) - 100 (* We reserve 100 bytes for the continuation hash. *) - - let serialize_hash = Sc_rollup_reveal_hash.to_hex - - let serialize_page page = - Format.asprintf - "%s hash:%s" - page.content - (serialize_hash page.succ_hash) - end) -end diff --git a/src/proto_alpha/lib_dal/dac_preimage_data_manager.ml b/src/proto_alpha/lib_dal/dac_preimage_data_manager.ml deleted file mode 100644 index 70711afc5ab3..000000000000 --- a/src/proto_alpha/lib_dal/dac_preimage_data_manager.ml +++ /dev/null @@ -1,63 +0,0 @@ -(*****************************************************************************) -(* *) -(* Open Source License *) -(* Copyright (c) 2022 Trili Tech *) -(* *) -(* Permission is hereby granted, free of charge, to any person obtaining a *) -(* copy of this software and associated documentation files (the "Software"),*) -(* to deal in the Software without restriction, including without limitation *) -(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) -(* and/or sell copies of the Software, and to permit persons to whom the *) -(* Software is furnished to do so, subject to the following conditions: *) -(* *) -(* The above copyright notice and this permission notice shall be included *) -(* in all copies or substantial portions of the Software. *) -(* *) -(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) -(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) -(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) -(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) -(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) -(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) -(* DEALINGS IN THE SOFTWARE. *) -(* *) -(*****************************************************************************) - -open Protocol -open Environment.Error_monad - -type error += Cannot_write_dac_page_to_disk of string - -let () = - register_error_kind - `Permanent - ~id:"cannot_write_dac_page_to_disk" - ~title:"Cannot write Dac page to disk" - ~description:"Cannot write Dac page to disk" - ~pp:(fun ppf b58_hash -> - Format.fprintf ppf "Could not write dac page for hash %s" b58_hash) - Data_encoding.(obj1 (req "hash" string)) - (function - | Cannot_write_dac_page_to_disk b58_hash -> Some b58_hash | _ -> None) - (fun b58_hash -> Cannot_write_dac_page_to_disk b58_hash) - -module type REVEAL_HASH = module type of Sc_rollup_reveal_hash - -module Make (Hash : REVEAL_HASH) = struct - let path data_dir hash = - let hash = Hash.to_hex hash in - Filename.(concat data_dir hash) - - let save_bytes data_dir hash page_contents = - let open Lwt_result_syntax in - let path = path data_dir hash in - let*! result = - Lwt_utils_unix.with_atomic_open_out path @@ fun chan -> - Lwt_utils_unix.write_bytes chan page_contents - in - match result with - | Ok () -> return () - | Error _ -> tzfail @@ Cannot_write_dac_page_to_disk (Hash.to_hex hash) -end - -module Reveal_hash = Make (Sc_rollup_reveal_hash) diff --git a/src/proto_alpha/lib_dal/dac_signature_manager.ml b/src/proto_alpha/lib_dal/dac_signature_manager.ml deleted file mode 100644 index f0009dc71e87..000000000000 --- a/src/proto_alpha/lib_dal/dac_signature_manager.ml +++ /dev/null @@ -1,180 +0,0 @@ -(*****************************************************************************) -(* *) -(* Open Source License *) -(* Copyright (c) 2022 Trili Tech *) -(* *) -(* Permission is hereby granted, free of charge, to any person obtaining a *) -(* copy of this software and associated documentation files (the "Software"),*) -(* to deal in the Software without restriction, including without limitation *) -(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) -(* and/or sell copies of the Software, and to permit persons to whom the *) -(* Software is furnished to do so, subject to the following conditions: *) -(* *) -(* The above copyright notice and this permission notice shall be included *) -(* in all copies or substantial portions of the Software. *) -(* *) -(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) -(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) -(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) -(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) -(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) -(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) -(* DEALINGS IN THE SOFTWARE. *) -(* *) -(*****************************************************************************) - -open Protocol -open Environment -open Error_monad - -type error += - | Cannot_convert_root_page_hash_to_bytes of string - | Cannot_compute_aggregate_signature of string - | Public_key_for_witness_not_available of int * string - -let () = - register_error_kind - `Permanent - ~id:"cannot_extract_root_page_hash_to_sign" - ~title:"Cannot convert root hash page to byte sequence" - ~description:"Cannot convert root hash page to byte sequence" - ~pp:(fun ppf b58_hash -> - Format.fprintf - ppf - "Cannot convert root hash page to byte sequence: %s" - b58_hash) - Data_encoding.(obj1 (req "hash" (string Plain))) - (function - | Cannot_convert_root_page_hash_to_bytes b58_hash -> Some b58_hash - | _ -> None) - (fun b58_hash -> Cannot_convert_root_page_hash_to_bytes b58_hash) ; - register_error_kind - `Permanent - ~id:"cannot_compute_root_hash_aggregate_signature" - ~title:"Cannot compute aggregate signature of root page hash" - ~description:"Cannot compute aggregate signature of root page hash" - ~pp:(fun ppf b58_hash -> - Format.fprintf - ppf - "Cannot compute aggregate signature of root page hash: %s" - b58_hash) - Data_encoding.(obj1 (req "hash" (string Plain))) - (function - | Cannot_compute_aggregate_signature b58_hash -> Some b58_hash | _ -> None) - (fun b58_hash -> Cannot_compute_aggregate_signature b58_hash) ; - register_error_kind - `Permanent - ~id:"public_key_of_witness_not_available" - ~title: - "Public key of witness dac member not available for verifying signature" - ~description: - "Public key of witness dac member not available for verifying signature" - ~pp:(fun ppf (witness_index, b58_hash) -> - Format.fprintf - ppf - "Public key of dac member %d not available for verifying signature of \ - root page hash %s" - witness_index - b58_hash) - Data_encoding.(obj2 (req "witness_index" int31) (req "hash" (string Plain))) - (function - | Public_key_for_witness_not_available (index, hash) -> Some (index, hash) - | _ -> None) - (fun (index, hash) -> Public_key_for_witness_not_available (index, hash)) - -module type REVEAL_HASH = module type of Sc_rollup_reveal_hash - -module Make (Hashing_scheme : REVEAL_HASH) = struct - let bind_es (f : 'a -> 'b option tzresult Lwt.t) v_opt = - let open Lwt_result_syntax in - match v_opt with None -> return None | Some v -> f v - - let rev_collect_indexed_signatures cctxt dac_sk_uris bytes_to_sign = - let open Lwt_result_syntax in - List.rev_mapi_es - (fun index sk_uri_opt -> - (* TODO: https://gitlab.com/tezos/tezos/-/issues/4306 - Implement Option.bind_es and revisit this. *) - bind_es - (fun sk_uri -> - let*! signature_res = - Tezos_client_base.Client_keys.aggregate_sign - cctxt - sk_uri - bytes_to_sign - in - let signature_opt = Result.to_option signature_res in - return - @@ Option.map (fun signature -> (index, signature)) signature_opt) - sk_uri_opt) - dac_sk_uris - - let compute_signatures_with_witnesses rev_indexed_signatures = - let open Lwt_result_syntax in - List.fold_left_es - (fun (signatures, witnesses) signature_opt -> - match signature_opt with - | None -> return (signatures, witnesses) - | Some (index, signature) -> - let*? bitmap = Bitset.add witnesses index in - return (signature :: signatures, bitmap)) - ([], Bitset.empty) - rev_indexed_signatures - - let sign_root_hash cctxt dac_sk_uris root_hash = - let open Lwt_result_syntax in - let bytes_to_sign = - Data_encoding.Binary.to_bytes_opt Hashing_scheme.encoding root_hash - in - let root_hash = Hashing_scheme.to_hex root_hash in - match bytes_to_sign with - | None -> tzfail @@ Cannot_convert_root_page_hash_to_bytes root_hash - | Some bytes_to_sign -> ( - let* rev_indexed_signatures = - rev_collect_indexed_signatures cctxt dac_sk_uris bytes_to_sign - in - let* signatures, witnesses = - compute_signatures_with_witnesses rev_indexed_signatures - in - let final_signature = - Tezos_crypto.Aggregate_signature.aggregate_signature_opt signatures - in - match final_signature with - | None -> tzfail @@ Cannot_compute_aggregate_signature root_hash - | Some signature -> return @@ (signature, witnesses)) - - let verify ~public_keys_opt root_page_hash signature witnesses = - let open Lwt_result_syntax in - let hash_as_bytes = - Data_encoding.Binary.to_bytes_opt Hashing_scheme.encoding root_page_hash - in - match hash_as_bytes with - | None -> - tzfail - @@ Cannot_convert_root_page_hash_to_bytes - (Hashing_scheme.to_hex root_page_hash) - | Some bytes -> - let* pk_msg_list = - public_keys_opt - |> List.mapi (fun i public_key_opt -> (i, public_key_opt)) - |> List.filter_map_es (fun (i, public_key_opt) -> - let*? is_witness = Bitset.mem witnesses i in - - match public_key_opt with - | None -> - if is_witness then - tzfail - @@ Public_key_for_witness_not_available - (i, Hashing_scheme.to_hex root_page_hash) - else return None - | Some public_key -> - if is_witness then return @@ Some (public_key, None, bytes) - else return None) - in - return - @@ Tezos_crypto.Aggregate_signature.aggregate_check - pk_msg_list - signature -end - -module Reveal_hash = Make (Sc_rollup_reveal_hash) diff --git a/src/proto_alpha/lib_dal/dal_plugin_registration.ml b/src/proto_alpha/lib_dal/dal_plugin_registration.ml index 63c78694dcb6..935a70af0325 100644 --- a/src/proto_alpha/lib_dal/dal_plugin_registration.ml +++ b/src/proto_alpha/lib_dal/dal_plugin_registration.ml @@ -127,8 +127,6 @@ module Plugin = struct in List.filter (Dal.Attestation.is_attested confirmed_slots) all_slots |> Dal.Slot_index.to_int_list |> return - - module RPC = RPC end let () = Dal_plugin.register (module Plugin) -- GitLab