From 3b2c47cbb6db0c2a74e6a362ec1bb162b0b9c2e6 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Thu, 11 May 2023 09:55:12 +0200 Subject: [PATCH 01/13] Tests: run migration tests on Mumbai->Nairobi as well --- tezt/tests/main.ml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tezt/tests/main.ml b/tezt/tests/main.ml index a0dc520214a4..cd85b650330e 100644 --- a/tezt/tests/main.ml +++ b/tezt/tests/main.ml @@ -84,6 +84,16 @@ let register_protocol_migration_tests () = ~loser_protocols:[migrate_from] ; Sc_rollup.register_migration ~migrate_from ~migrate_to +let register_old_protocol_migration_tests () = + List.iter + (fun p -> + match (Protocol.previous_protocol p, p) with + | _, Alpha -> () (* Already in register_protocol_migration_tests *) + | None, _ -> () + | Some migrate_from, migrate_to -> + Sc_rollup.register_migration ~migrate_from ~migrate_to) + Protocol.all + (* Register tests that use [Protocol.register_test] and for which we rely on [?supports] to decide which protocols the tests should run on. As a consequence, all those tests should be registered with [Protocol.all] @@ -218,6 +228,7 @@ let register_protocol_specific_because_regression_tests () = let () = register_protocol_independent_tests () ; register_protocol_migration_tests () ; + register_old_protocol_migration_tests () ; register_protocol_tests_that_use_supports_correctly () ; register_protocol_specific_because_regression_tests () ; Tezos_scoru_wasm_regressions.register () ; -- GitLab From 2fb38f1e416d666bc64d9f7e23c64f4c4aea6dc5 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Thu, 11 May 2023 11:57:50 +0200 Subject: [PATCH 02/13] L2/Store: expose underlying Irmin store Used for migrations --- src/lib_layer2_store/irmin_store.ml | 11 ++++++++++- src/lib_layer2_store/irmin_store.mli | 15 ++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/lib_layer2_store/irmin_store.ml b/src/lib_layer2_store/irmin_store.ml index 78b8c9bc8828..6d327464bf9e 100644 --- a/src/lib_layer2_store/irmin_store.ml +++ b/src/lib_layer2_store/irmin_store.ml @@ -31,7 +31,8 @@ module Make (N : sig end) = struct module Maker = Irmin_pack_unix.Maker (Tezos_context_encoding.Context.Conf) - include Maker.Make (Tezos_context_encoding.Context.Schema) + module Raw = Maker.Make (Tezos_context_encoding.Context.Schema) + include Raw let make_key_path path key = path @ [key] @@ -109,4 +110,12 @@ struct @@ fun () -> let*! bytes_opt = find store path in return bytes_opt + + module Raw_irmin = struct + type rw = [`Read | `Write] t + + include Raw + + let unsafe = Fun.id + end end diff --git a/src/lib_layer2_store/irmin_store.mli b/src/lib_layer2_store/irmin_store.mli index e1a9c5716a2e..a6fe7d9b7e7b 100644 --- a/src/lib_layer2_store/irmin_store.mli +++ b/src/lib_layer2_store/irmin_store.mli @@ -23,6 +23,19 @@ (* DEALINGS IN THE SOFTWARE. *) (* *) (*****************************************************************************) + module Make (N : sig val name : string -end) : Store_sigs.BACKEND +end) : sig + include Store_sigs.BACKEND + + module Raw_irmin : sig + type rw = [`Read | `Write] t + + include + Irmin.Generic_key.S + with module Schema.Path = Tezos_context_encoding.Context.Schema.Path + + val unsafe : rw -> t + end +end -- GitLab From a65fd1c9a33bdc8bfaae6ce9b2e10d48a6fac6db Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Thu, 11 May 2023 12:49:22 +0200 Subject: [PATCH 03/13] L2/Store: expose path of components --- src/lib_layer2_store/store_sigs.ml | 9 +++++++++ src/lib_layer2_store/store_utils.ml | 2 ++ 2 files changed, 11 insertions(+) diff --git a/src/lib_layer2_store/store_sigs.ml b/src/lib_layer2_store/store_sigs.ml index 24bec359d1f5..4cb00049fdb3 100644 --- a/src/lib_layer2_store/store_sigs.ml +++ b/src/lib_layer2_store/store_sigs.ml @@ -97,6 +97,9 @@ module type Mutable_value = sig (** The type of the values that will be persisted. *) type value + (** Path in the irmin tree. *) + val path_key : path + (** [set store value] persists [value] for this [Mutable_value] on [store]. *) val set : [> `Write] store -> value -> unit tzresult Lwt.t @@ -155,6 +158,9 @@ module type Map = sig (** The type of values persisted by the map. *) type value + (** Path in the irmin tree. *) + val path : path + (** [mem store key] checks whether there is a binding of the map for key [key] in [store]. *) val mem : [> `Read] store -> key -> bool tzresult Lwt.t @@ -210,6 +216,9 @@ module type Nested_map = sig indexed by primary and secondary key. *) type value + (** Path in the irmin tree. *) + val path : path + (** [mem store ~primary_key ~secondary_key] returns whether there is a value for the nested map persisted on [store] for the nested map, indexed by [primary_key] and then by [secondary_key]. *) diff --git a/src/lib_layer2_store/store_utils.ml b/src/lib_layer2_store/store_utils.ml index 66e7def4403e..dace42a23d58 100644 --- a/src/lib_layer2_store/store_utils.ml +++ b/src/lib_layer2_store/store_utils.ml @@ -149,6 +149,8 @@ module Make (B : BACKEND) = struct type value = V.value + let path = S.path + module Secondary_key_map = Map.Make (struct type t = K2.key -- GitLab From 01126a486322cd70cc74b0f51f0d78855f6f0159 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Tue, 9 May 2023 12:41:47 +0200 Subject: [PATCH 04/13] SCORU/Node/Store: iter on L2 blocks --- src/proto_alpha/lib_sc_rollup_node/store.ml | 21 ++++++++++++++++++++ src/proto_alpha/lib_sc_rollup_node/store.mli | 6 ++++++ 2 files changed, 27 insertions(+) diff --git a/src/proto_alpha/lib_sc_rollup_node/store.ml b/src/proto_alpha/lib_sc_rollup_node/store.ml index 4f87a0d36983..7fc5b457dcad 100644 --- a/src/proto_alpha/lib_sc_rollup_node/store.ml +++ b/src/proto_alpha/lib_sc_rollup_node/store.ml @@ -2,6 +2,7 @@ (* *) (* Open Source License *) (* Copyright (c) 2021 Nomadic Labs, *) +(* Copyright (c) 2023 Functori, *) (* *) (* Permission is hereby granted, free of charge, to any person obtaining a *) (* copy of this software and associated documentation files (the "Software"),*) @@ -450,3 +451,23 @@ let load (type a) (mode : a mode) ~l2_blocks_cache_size data_dir : levels_to_hashes; irmin_store; } + +let iter_l2_blocks ({l2_blocks; l2_head; _} : _ t) f = + let open Lwt_result_syntax in + let* head = L2_head.read l2_head in + match head with + | None -> + (* No reachable head, nothing to do *) + return_unit + | Some head -> + let rec loop hash = + let* block = L2_blocks.read l2_blocks hash in + match block with + | None -> + (* The block does not exist, the known chain stops here, so do we. *) + return_unit + | Some (block, header) -> + let* () = f {block with header} in + loop header.predecessor + in + loop head.header.block_hash diff --git a/src/proto_alpha/lib_sc_rollup_node/store.mli b/src/proto_alpha/lib_sc_rollup_node/store.mli index b43b800956ce..66d3d3b8941c 100644 --- a/src/proto_alpha/lib_sc_rollup_node/store.mli +++ b/src/proto_alpha/lib_sc_rollup_node/store.mli @@ -2,6 +2,7 @@ (* *) (* Open Source License *) (* Copyright (c) 2022 Nomadic Labs, *) +(* Copyright (c) 2023 Functori, *) (* *) (* Permission is hereby granted, free of charge, to any person obtaining a *) (* copy of this software and associated documentation files (the "Software"),*) @@ -161,3 +162,8 @@ val load : (** [readonly store] returns a read-only version of [store]. *) val readonly : _ t -> ro + +(** [iter_l2_blocks store f] iterates [f] on all L2 blocks reachable from the + head, from newest to oldest. *) +val iter_l2_blocks : + _ t -> (Sc_rollup_block.t -> unit tzresult Lwt.t) -> unit tzresult Lwt.t -- GitLab From 56f11a0207a77b03e0e148e31ae0a742845fbe33 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Tue, 9 May 2023 12:45:25 +0200 Subject: [PATCH 05/13] SCORU/Node/Store: signature for versioned store --- src/proto_alpha/lib_sc_rollup_node/store.mli | 34 +--------- .../lib_sc_rollup_node/store_sig.ml | 62 +++++++++++++++++++ 2 files changed, 63 insertions(+), 33 deletions(-) create mode 100644 src/proto_alpha/lib_sc_rollup_node/store_sig.ml diff --git a/src/proto_alpha/lib_sc_rollup_node/store.mli b/src/proto_alpha/lib_sc_rollup_node/store.mli index 66d3d3b8941c..b0bf1810f7d8 100644 --- a/src/proto_alpha/lib_sc_rollup_node/store.mli +++ b/src/proto_alpha/lib_sc_rollup_node/store.mli @@ -134,36 +134,4 @@ type +'a store = { irmin_store : 'a Irmin_store.t; } -(** Type of store. The parameter indicates if the store can be written or only - read. *) -type 'a t = ([< `Read | `Write > `Read] as 'a) store - -(** Read/write store {!t}. *) -type rw = Store_sigs.rw t - -(** Read only store {!t}. *) -type ro = Store_sigs.ro t - -(** [close store] closes the store. *) -val close : _ t -> unit tzresult Lwt.t - -(** [load mode ~l2_blocks_cache_size directory] loads a store from the data - persisted in [directory]. If [mode] is {!Store_sigs.Read_only}, then the - indexes and irmin store will be opened in readonly mode and only read - operations will be permitted. This allows to open a store for read access - that is already opened in {!Store_sigs.Read_write} mode in another - process. [l2_blocks_cache_size] is the number of L2 blocks the rollup node - will keep in memory. *) -val load : - 'a Store_sigs.mode -> - l2_blocks_cache_size:int -> - string -> - 'a store tzresult Lwt.t - -(** [readonly store] returns a read-only version of [store]. *) -val readonly : _ t -> ro - -(** [iter_l2_blocks store f] iterates [f] on all L2 blocks reachable from the - head, from newest to oldest. *) -val iter_l2_blocks : - _ t -> (Sc_rollup_block.t -> unit tzresult Lwt.t) -> unit tzresult Lwt.t +include Store_sig.S with type 'a store := 'a store diff --git a/src/proto_alpha/lib_sc_rollup_node/store_sig.ml b/src/proto_alpha/lib_sc_rollup_node/store_sig.ml new file mode 100644 index 000000000000..3493ef2df1ae --- /dev/null +++ b/src/proto_alpha/lib_sc_rollup_node/store_sig.ml @@ -0,0 +1,62 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +module type S = sig + type +'a store + + (** Type of store. The parameter indicates if the store can be written or only + read. *) + type 'a t = ([< `Read | `Write > `Read] as 'a) store + + (** Read/write store {!t}. *) + type rw = Store_sigs.rw t + + (** Read only store {!t}. *) + type ro = Store_sigs.ro t + + (** [close store] closes the store. *) + val close : _ t -> unit tzresult Lwt.t + + (** [load mode ~l2_blocks_cache_size directory] loads a store from the data + persisted in [directory]. If [mode] is {!Store_sigs.Read_only}, then the + indexes and irmin store will be opened in readonly mode and only read + operations will be permitted. This allows to open a store for read access + that is already opened in {!Store_sigs.Read_write} mode in another + process. [l2_blocks_cache_size] is the number of L2 blocks the rollup node + will keep in memory. *) + val load : + 'a Store_sigs.mode -> + l2_blocks_cache_size:int -> + string -> + 'a store tzresult Lwt.t + + (** [readonly store] returns a read-only version of [store]. *) + val readonly : _ t -> ro + + (** [iter_l2_blocks store f] iterates [f] on all L2 blocks reachable from the + head, from newest to oldest. *) + val iter_l2_blocks : + _ t -> (Sc_rollup_block.t -> unit tzresult Lwt.t) -> unit tzresult Lwt.t +end -- GitLab From c7d54b6702d442dcf80039de43472e996dd50f42 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Tue, 9 May 2023 13:58:52 +0200 Subject: [PATCH 06/13] SCORU/Node/Store: two versions of the store module This is necessary to support the version of the store used in the Mumbai rollup node. --- src/proto_alpha/lib_sc_rollup_node/store.ml | 448 +--------------- .../lib_sc_rollup_node/store_v0.ml | 504 ++++++++++++++++++ .../{store.mli => store_v0.mli} | 45 +- .../lib_sc_rollup_node/store_v1.ml | 251 +++++++++ .../lib_sc_rollup_node/store_v1.mli | 79 +++ 5 files changed, 871 insertions(+), 456 deletions(-) create mode 100644 src/proto_alpha/lib_sc_rollup_node/store_v0.ml rename src/proto_alpha/lib_sc_rollup_node/{store.mli => store_v0.mli} (78%) create mode 100644 src/proto_alpha/lib_sc_rollup_node/store_v1.ml create mode 100644 src/proto_alpha/lib_sc_rollup_node/store_v1.mli diff --git a/src/proto_alpha/lib_sc_rollup_node/store.ml b/src/proto_alpha/lib_sc_rollup_node/store.ml index 7fc5b457dcad..90d354f79c80 100644 --- a/src/proto_alpha/lib_sc_rollup_node/store.ml +++ b/src/proto_alpha/lib_sc_rollup_node/store.ml @@ -24,450 +24,4 @@ (* *) (*****************************************************************************) -open Protocol -include Store_sigs -include Store_utils - -(** Aggregated collection of messages from the L1 inbox *) -open Alpha_context - -module Irmin_store = struct - module IStore = Irmin_store.Make (struct - let name = "Tezos smart rollup node" - end) - - include IStore - include Store_utils.Make (IStore) -end - -module Empty_header = struct - type t = unit - - let name = "empty" - - let encoding = Data_encoding.unit - - let fixed_size = 0 -end - -module Add_empty_header = struct - module Header = Empty_header - - let header _ = () -end - -module Make_hash_index_key (H : Environment.S.HASH) = -Indexed_store.Make_index_key (struct - include Indexed_store.Make_fixed_encodable (H) - - let equal = H.equal -end) - -(** L2 blocks *) -module L2_blocks = - Indexed_store.Make_indexed_file - (struct - let name = "l2_blocks" - end) - (Tezos_store_shared.Block_key) - (struct - type t = (unit, unit) Sc_rollup_block.block - - let name = "sc_rollup_block_info" - - let encoding = - Sc_rollup_block.block_encoding Data_encoding.unit Data_encoding.unit - - module Header = struct - type t = Sc_rollup_block.header - - let name = "sc_rollup_block_header" - - let encoding = Sc_rollup_block.header_encoding - - let fixed_size = Sc_rollup_block.header_size - end - end) - -(** Unaggregated messages per block *) -module Messages = - Indexed_store.Make_indexed_file - (struct - let name = "messages" - end) - (Make_hash_index_key (Sc_rollup.Inbox_merkelized_payload_hashes.Hash)) - (struct - type t = Sc_rollup.Inbox_message.t list - - let name = "messages_list" - - let encoding = - Data_encoding.(list @@ dynamic_size Sc_rollup.Inbox_message.encoding) - - module Header = struct - type t = bool * Block_hash.t * Timestamp.t * int - - let name = "messages_inbox_info" - - let encoding = - let open Data_encoding in - obj4 - (req "is_first_block" bool) - (req "predecessor" Block_hash.encoding) - (req "predecessor_timestamp" Timestamp.encoding) - (req "num_messages" int31) - - let fixed_size = - WithExceptions.Option.get ~loc:__LOC__ - @@ Data_encoding.Binary.fixed_length encoding - end - end) - -(** Inbox state for each block *) -module Inboxes = - Indexed_store.Make_simple_indexed_file - (struct - let name = "inboxes" - end) - (Make_hash_index_key (Sc_rollup.Inbox.Hash)) - (struct - type t = Sc_rollup.Inbox.t - - let name = "inbox" - - let encoding = Sc_rollup.Inbox.encoding - - include Add_empty_header - end) - -module Commitments = - Indexed_store.Make_indexable - (struct - let name = "commitments" - end) - (Make_hash_index_key (Sc_rollup.Commitment.Hash)) - (Indexed_store.Make_index_value (Indexed_store.Make_fixed_encodable (struct - include Sc_rollup.Commitment - - let name = "commitment" - end))) - -module Commitments_published_at_level = struct - type element = { - first_published_at_level : Raw_level.t; - published_at_level : Raw_level.t option; - } - - let element_encoding = - let open Data_encoding in - let opt_level_encoding = - conv - (function None -> -1l | Some l -> Raw_level.to_int32 l) - (fun l -> if l = -1l then None else Some (Raw_level.of_int32_exn l)) - Data_encoding.int32 - in - conv - (fun {first_published_at_level; published_at_level} -> - (first_published_at_level, published_at_level)) - (fun (first_published_at_level, published_at_level) -> - {first_published_at_level; published_at_level}) - @@ obj2 - (req "first_published_at_level" Raw_level.encoding) - (req "published_at_level" opt_level_encoding) - - include - Indexed_store.Make_indexable - (struct - let name = "commitments" - end) - (Make_hash_index_key (Sc_rollup.Commitment.Hash)) - (Indexed_store.Make_index_value (Indexed_store.Make_fixed_encodable (struct - type t = element - - let name = "published_levels" - - let encoding = element_encoding - end))) -end - -module L2_head = Indexed_store.Make_singleton (struct - type t = Sc_rollup_block.t - - let name = "l2_head" - - let encoding = Sc_rollup_block.encoding -end) - -module Last_finalized_level = Indexed_store.Make_singleton (struct - type t = int32 - - let name = "finalized_level" - - let encoding = Data_encoding.int32 -end) - -(** Table from L1 levels to blocks hashes. *) -module Levels_to_hashes = - Indexed_store.Make_indexable - (struct - let name = "tezos_levels" - end) - (Indexed_store.Make_index_key (struct - type t = int32 - - let encoding = Data_encoding.int32 - - let name = "level" - - let fixed_size = 4 - - let equal = Int32.equal - end)) - (Tezos_store_shared.Block_key) - -(** Store attestation statuses for DAL slots on L1. *) -module Dal_slots_statuses = - Irmin_store.Make_nested_map - (struct - let path = ["dal"; "slots_statuses"] - end) - (struct - type key = Block_hash.t - - let to_path_representation = Block_hash.to_b58check - end) - (struct - type key = Dal.Slot_index.t - - let encoding = Dal.Slot_index.encoding - - let compare = Dal.Slot_index.compare - - let name = "slot_index" - end) - (struct - (* TODO: https://gitlab.com/tezos/tezos/-/issues/4780. - - Rename Confirm-ed/ation to Attest-ed/ation in the rollup node. *) - type value = [`Confirmed | `Unconfirmed] - - let name = "slot_status" - - let encoding = - (* We don't use - Data_encoding.string_enum because a union is more storage-efficient. *) - let open Data_encoding in - union - ~tag_size:`Uint8 - [ - case - ~title:"Confirmed" - (Tag 0) - (obj1 (req "kind" (constant "Confirmed"))) - (function `Confirmed -> Some () | `Unconfirmed -> None) - (fun () -> `Confirmed); - case - ~title:"Unconfirmed" - (Tag 1) - (obj1 (req "kind" (constant "Unconfirmed"))) - (function `Unconfirmed -> Some () | `Confirmed -> None) - (fun () -> `Unconfirmed); - ] - end) - -module Dal_slots_headers = - Irmin_store.Make_nested_map - (struct - let path = ["dal"; "slot_headers"] - end) - (struct - type key = Block_hash.t - - let to_path_representation = Block_hash.to_b58check - end) - (struct - type key = Dal.Slot_index.t - - let encoding = Dal.Slot_index.encoding - - let compare = Dal.Slot_index.compare - - let name = "slot_index" - end) - (struct - type value = Dal.Slot.Header.t - - let name = "slot_header" - - let encoding = Dal.Slot.Header.encoding - end) - -(* Published slot headers per block hash, stored as a list of bindings from - `Dal_slot_index.t` to `Dal.Slot.t`. The encoding function converts this - list into a `Dal.Slot_index.t`-indexed map. Note that the block_hash - refers to the block where slots headers have been confirmed, not - the block where they have been published. -*) - -(** Confirmed DAL slots history. See documentation of - {Dal_slot_repr.Slots_history} for more details. *) -module Dal_confirmed_slots_history = - Irmin_store.Make_append_only_map - (struct - let path = ["dal"; "confirmed_slots_history"] - end) - (struct - type key = Block_hash.t - - let to_path_representation = Block_hash.to_b58check - end) - (struct - type value = Dal.Slots_history.t - - let name = "dal_slot_histories" - - let encoding = Dal.Slots_history.encoding - end) - -(** Confirmed DAL slots histories cache. See documentation of - {Dal_slot_repr.Slots_history} for more details. *) -module Dal_confirmed_slots_histories = - (* TODO: https://gitlab.com/tezos/tezos/-/issues/4390 - Store single history points in map instead of whole history. *) - Irmin_store.Make_append_only_map - (struct - let path = ["dal"; "confirmed_slots_histories_cache"] - end) - (struct - type key = Block_hash.t - - let to_path_representation = Block_hash.to_b58check - end) - (struct - type value = Dal.Slots_history.History_cache.t - - let name = "dal_slot_history_cache" - - let encoding = Dal.Slots_history.History_cache.encoding - end) - -type 'a store = { - l2_blocks : 'a L2_blocks.t; - messages : 'a Messages.t; - inboxes : 'a Inboxes.t; - commitments : 'a Commitments.t; - commitments_published_at_level : 'a Commitments_published_at_level.t; - l2_head : 'a L2_head.t; - last_finalized_level : 'a Last_finalized_level.t; - levels_to_hashes : 'a Levels_to_hashes.t; - irmin_store : 'a Irmin_store.t; -} - -type 'a t = ([< `Read | `Write > `Read] as 'a) store - -type rw = Store_sigs.rw t - -type ro = Store_sigs.ro t - -let readonly - ({ - l2_blocks; - messages; - inboxes; - commitments; - commitments_published_at_level; - l2_head; - last_finalized_level; - levels_to_hashes; - irmin_store; - } : - _ t) : ro = - { - l2_blocks = L2_blocks.readonly l2_blocks; - messages = Messages.readonly messages; - inboxes = Inboxes.readonly inboxes; - commitments = Commitments.readonly commitments; - commitments_published_at_level = - Commitments_published_at_level.readonly commitments_published_at_level; - l2_head = L2_head.readonly l2_head; - last_finalized_level = Last_finalized_level.readonly last_finalized_level; - levels_to_hashes = Levels_to_hashes.readonly levels_to_hashes; - irmin_store = Irmin_store.readonly irmin_store; - } - -let close - ({ - l2_blocks; - messages; - inboxes; - commitments; - commitments_published_at_level; - l2_head = _; - last_finalized_level = _; - levels_to_hashes; - irmin_store; - } : - _ t) = - let open Lwt_result_syntax in - let+ () = L2_blocks.close l2_blocks - and+ () = Messages.close messages - and+ () = Inboxes.close inboxes - and+ () = Commitments.close commitments - and+ () = Commitments_published_at_level.close commitments_published_at_level - and+ () = Levels_to_hashes.close levels_to_hashes - and+ () = Irmin_store.close irmin_store in - () - -let load (type a) (mode : a mode) ~l2_blocks_cache_size data_dir : - a store tzresult Lwt.t = - let open Lwt_result_syntax in - let path name = Filename.concat data_dir name in - let cache_size = l2_blocks_cache_size in - let* l2_blocks = L2_blocks.load mode ~path:(path "l2_blocks") ~cache_size in - let* messages = Messages.load mode ~path:(path "messages") ~cache_size in - let* inboxes = Inboxes.load mode ~path:(path "inboxes") ~cache_size in - let* commitments = Commitments.load mode ~path:(path "commitments") in - let* commitments_published_at_level = - Commitments_published_at_level.load - mode - ~path:(path "commitments_published_at_level") - in - let* l2_head = L2_head.load mode ~path:(path "l2_head") in - let* last_finalized_level = - Last_finalized_level.load mode ~path:(path "last_finalized_level") - in - let* levels_to_hashes = - Levels_to_hashes.load mode ~path:(path "levels_to_hashes") - in - let+ irmin_store = Irmin_store.load mode (path "irmin_store") in - { - l2_blocks; - messages; - inboxes; - commitments; - commitments_published_at_level; - l2_head; - last_finalized_level; - levels_to_hashes; - irmin_store; - } - -let iter_l2_blocks ({l2_blocks; l2_head; _} : _ t) f = - let open Lwt_result_syntax in - let* head = L2_head.read l2_head in - match head with - | None -> - (* No reachable head, nothing to do *) - return_unit - | Some head -> - let rec loop hash = - let* block = L2_blocks.read l2_blocks hash in - match block with - | None -> - (* The block does not exist, the known chain stops here, so do we. *) - return_unit - | Some (block, header) -> - let* () = f {block with header} in - loop header.predecessor - in - loop head.header.block_hash +include Store_v1 diff --git a/src/proto_alpha/lib_sc_rollup_node/store_v0.ml b/src/proto_alpha/lib_sc_rollup_node/store_v0.ml new file mode 100644 index 000000000000..58f8b20c7ca1 --- /dev/null +++ b/src/proto_alpha/lib_sc_rollup_node/store_v0.ml @@ -0,0 +1,504 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2021 Nomadic Labs, *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(** This module is a copy of + [src/proto_016_PtMumbai/lib_sc_rollup_node/store.ml], which contains the + store for the Mumbai rollup node. *) + +open Protocol +include Store_sigs +include Store_utils + +(** Aggregated collection of messages from the L1 inbox *) +open Alpha_context + +module Irmin_store = struct + module IStore = Irmin_store.Make (struct + let name = "Tezos smart rollup node" + end) + + include IStore + include Store_utils.Make (IStore) +end + +module Empty_header = struct + type t = unit + + let name = "empty" + + let encoding = Data_encoding.unit + + let fixed_size = 0 +end + +module Add_empty_header = struct + module Header = Empty_header + + let header _ = () +end + +module Make_hash_index_key (H : Environment.S.HASH) = +Indexed_store.Make_index_key (struct + include Indexed_store.Make_fixed_encodable (H) + + let equal = H.equal +end) + +(** L2 blocks *) +module L2_blocks = + Indexed_store.Make_indexed_file + (struct + let name = "l2_blocks" + end) + (Tezos_store_shared.Block_key) + (struct + type t = (unit, unit) Sc_rollup_block.block + + let name = "sc_rollup_block_info" + + let encoding = + Sc_rollup_block.block_encoding Data_encoding.unit Data_encoding.unit + + module Header = struct + type t = Sc_rollup_block.header + + let name = "sc_rollup_block_header" + + let encoding = Sc_rollup_block.header_encoding + + let fixed_size = Sc_rollup_block.header_size + end + end) + +(** Unaggregated messages per block *) +module Messages = + Indexed_store.Make_indexed_file + (struct + let name = "messages" + end) + (Make_hash_index_key (Sc_rollup.Inbox_merkelized_payload_hashes.Hash)) + (struct + type t = Sc_rollup.Inbox_message.t list + + let name = "messages_list" + + let encoding = + Data_encoding.(list @@ dynamic_size Sc_rollup.Inbox_message.encoding) + + module Header = struct + type t = Block_hash.t * Timestamp.t * int + + let name = "messages_inbox_info" + + let encoding = + let open Data_encoding in + obj3 + (req "predecessor" Block_hash.encoding) + (req "predecessor_timestamp" Timestamp.encoding) + (req "num_messages" int31) + + let fixed_size = + WithExceptions.Option.get ~loc:__LOC__ + @@ Data_encoding.Binary.fixed_length encoding + end + end) + +(** Inbox state for each block *) +module Inboxes = + Indexed_store.Make_simple_indexed_file + (struct + let name = "inboxes" + end) + (Make_hash_index_key (Sc_rollup.Inbox.Hash)) + (struct + type t = Sc_rollup.Inbox.t + + let name = "inbox" + + let encoding = Sc_rollup.Inbox.encoding + + include Add_empty_header + end) + +module Commitments = + Indexed_store.Make_indexable + (struct + let name = "commitments" + end) + (Make_hash_index_key (Sc_rollup.Commitment.Hash)) + (Indexed_store.Make_index_value (Indexed_store.Make_fixed_encodable (struct + include Sc_rollup.Commitment + + let name = "commitment" + end))) + +module Commitments_published_at_level = struct + type element = { + first_published_at_level : Raw_level.t; + published_at_level : Raw_level.t option; + } + + let element_encoding = + let open Data_encoding in + let opt_level_encoding = + conv + (function None -> -1l | Some l -> Raw_level.to_int32 l) + (fun l -> if l = -1l then None else Some (Raw_level.of_int32_exn l)) + Data_encoding.int32 + in + conv + (fun {first_published_at_level; published_at_level} -> + (first_published_at_level, published_at_level)) + (fun (first_published_at_level, published_at_level) -> + {first_published_at_level; published_at_level}) + @@ obj2 + (req "first_published_at_level" Raw_level.encoding) + (req "published_at_level" opt_level_encoding) + + include + Indexed_store.Make_indexable + (struct + let name = "commitments" + end) + (Make_hash_index_key (Sc_rollup.Commitment.Hash)) + (Indexed_store.Make_index_value (Indexed_store.Make_fixed_encodable (struct + type t = element + + let name = "published_levels" + + let encoding = element_encoding + end))) +end + +module L2_head = Indexed_store.Make_singleton (struct + type t = Sc_rollup_block.t + + let name = "l2_head" + + let encoding = Sc_rollup_block.encoding +end) + +module Last_finalized_level = Indexed_store.Make_singleton (struct + type t = int32 + + let name = "finalized_level" + + let encoding = Data_encoding.int32 +end) + +(** Table from L1 levels to blocks hashes. *) +module Levels_to_hashes = + Indexed_store.Make_indexable + (struct + let name = "tezos_levels" + end) + (Indexed_store.Make_index_key (struct + type t = int32 + + let encoding = Data_encoding.int32 + + let name = "level" + + let fixed_size = 4 + + let equal = Int32.equal + end)) + (Tezos_store_shared.Block_key) + +(* Published slot headers per block hash, + stored as a list of bindings from `Dal_slot_index.t` + to `Dal.Slot.t`. The encoding function converts this + list into a `Dal.Slot_index.t`-indexed map. *) +module Dal_slot_pages = + Irmin_store.Make_nested_map + (struct + let path = ["dal"; "slot_pages"] + end) + (struct + type key = Block_hash.t + + let to_path_representation = Block_hash.to_b58check + end) + (struct + type key = Dal.Slot_index.t * Dal.Page.Index.t + + let encoding = + Data_encoding.(tup2 Dal.Slot_index.encoding Dal.Page.Index.encoding) + + let compare (i1, p1) (i2, p2) = + Compare.or_else (Dal.Slot_index.compare i1 i2) (fun () -> + Dal.Page.Index.compare p1 p2) + + let name = "slot_index" + end) + (struct + type value = Dal.Page.content + + let encoding = Dal.Page.content_encoding + + let name = "slot_pages" + end) + +(** stores slots whose data have been considered and pages stored to disk (if + they are confirmed). *) +module Dal_processed_slots = + Irmin_store.Make_nested_map + (struct + let path = ["dal"; "processed_slots"] + end) + (struct + type key = Block_hash.t + + let to_path_representation = Block_hash.to_b58check + end) + (struct + type key = Dal.Slot_index.t + + let encoding = Dal.Slot_index.encoding + + let compare = Dal.Slot_index.compare + + let name = "slot_index" + end) + (struct + type value = [`Confirmed | `Unconfirmed] + + let name = "slot_processing_status" + + let encoding = + let open Data_encoding in + let mk_case constr ~tag ~title = + case + ~title + (Tag tag) + (obj1 (req "kind" (constant title))) + (fun x -> if x = constr then Some () else None) + (fun () -> constr) + in + union + ~tag_size:`Uint8 + [ + mk_case `Confirmed ~tag:0 ~title:"Confirmed"; + mk_case `Unconfirmed ~tag:1 ~title:"Unconfirmed"; + ] + end) + +module Dal_slots_headers = + Irmin_store.Make_nested_map + (struct + let path = ["dal"; "slot_headers"] + end) + (struct + type key = Block_hash.t + + let to_path_representation = Block_hash.to_b58check + end) + (struct + type key = Dal.Slot_index.t + + let encoding = Dal.Slot_index.encoding + + let compare = Dal.Slot_index.compare + + let name = "slot_index" + end) + (struct + type value = Dal.Slot.Header.t + + let name = "slot_header" + + let encoding = Dal.Slot.Header.encoding + end) + +(* Published slot headers per block hash, stored as a list of bindings from + `Dal_slot_index.t` to `Dal.Slot.t`. The encoding function converts this + list into a `Dal.Slot_index.t`-indexed map. Note that the block_hash + refers to the block where slots headers have been confirmed, not + the block where they have been published. +*) + +(** Confirmed DAL slots history. See documentation of + {!Dal_slot_repr.Slots_history} for more details. *) +module Dal_confirmed_slots_history = + Irmin_store.Make_append_only_map + (struct + let path = ["dal"; "confirmed_slots_history"] + end) + (struct + type key = Block_hash.t + + let to_path_representation = Block_hash.to_b58check + end) + (struct + type value = Dal.Slots_history.t + + let name = "dal_slot_histories" + + let encoding = Dal.Slots_history.encoding + end) + +(** Confirmed DAL slots histories cache. See documentation of + {!Dal_slot_repr.Slots_history} for more details. *) +module Dal_confirmed_slots_histories = + (* TODO: https://gitlab.com/tezos/tezos/-/issues/4390 + Store single history points in map instead of whole history. *) + Irmin_store.Make_append_only_map + (struct + let path = ["dal"; "confirmed_slots_histories_cache"] + end) + (struct + type key = Block_hash.t + + let to_path_representation = Block_hash.to_b58check + end) + (struct + type value = Dal.Slots_history.History_cache.t + + let name = "dal_slot_history_cache" + + let encoding = Dal.Slots_history.History_cache.encoding + end) + +type 'a store = { + l2_blocks : 'a L2_blocks.t; + messages : 'a Messages.t; + inboxes : 'a Inboxes.t; + commitments : 'a Commitments.t; + commitments_published_at_level : 'a Commitments_published_at_level.t; + l2_head : 'a L2_head.t; + last_finalized_level : 'a Last_finalized_level.t; + levels_to_hashes : 'a Levels_to_hashes.t; + irmin_store : 'a Irmin_store.t; +} + +type 'a t = ([< `Read | `Write > `Read] as 'a) store + +type rw = Store_sigs.rw t + +type ro = Store_sigs.ro t + +let readonly + ({ + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head; + last_finalized_level; + levels_to_hashes; + irmin_store; + } : + _ t) : ro = + { + l2_blocks = L2_blocks.readonly l2_blocks; + messages = Messages.readonly messages; + inboxes = Inboxes.readonly inboxes; + commitments = Commitments.readonly commitments; + commitments_published_at_level = + Commitments_published_at_level.readonly commitments_published_at_level; + l2_head = L2_head.readonly l2_head; + last_finalized_level = Last_finalized_level.readonly last_finalized_level; + levels_to_hashes = Levels_to_hashes.readonly levels_to_hashes; + irmin_store = Irmin_store.readonly irmin_store; + } + +let close + ({ + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head = _; + last_finalized_level = _; + levels_to_hashes; + irmin_store; + } : + _ t) = + let open Lwt_result_syntax in + let+ () = L2_blocks.close l2_blocks + and+ () = Messages.close messages + and+ () = Inboxes.close inboxes + and+ () = Commitments.close commitments + and+ () = Commitments_published_at_level.close commitments_published_at_level + and+ () = Levels_to_hashes.close levels_to_hashes + and+ () = Irmin_store.close irmin_store in + () + +let load (type a) (mode : a mode) ~l2_blocks_cache_size data_dir : + a store tzresult Lwt.t = + let open Lwt_result_syntax in + let path name = Filename.concat data_dir name in + let cache_size = l2_blocks_cache_size in + let* l2_blocks = L2_blocks.load mode ~path:(path "l2_blocks") ~cache_size in + let* messages = Messages.load mode ~path:(path "messages") ~cache_size in + let* inboxes = Inboxes.load mode ~path:(path "inboxes") ~cache_size in + let* commitments = Commitments.load mode ~path:(path "commitments") in + let* commitments_published_at_level = + Commitments_published_at_level.load + mode + ~path:(path "commitments_published_at_level") + in + let* l2_head = L2_head.load mode ~path:(path "l2_head") in + let* last_finalized_level = + Last_finalized_level.load mode ~path:(path "last_finalized_level") + in + let* levels_to_hashes = + Levels_to_hashes.load mode ~path:(path "levels_to_hashes") + in + let+ irmin_store = Irmin_store.load mode (path "irmin_store") in + { + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head; + last_finalized_level; + levels_to_hashes; + irmin_store; + } + +let iter_l2_blocks ({l2_blocks; l2_head; _} : _ t) f = + let open Lwt_result_syntax in + let* head = L2_head.read l2_head in + match head with + | None -> + (* No reachable head, nothing to do *) + return_unit + | Some head -> + let rec loop hash = + let* block = L2_blocks.read l2_blocks hash in + match block with + | None -> + (* The block does not exist, the known chain stops here, so do we. *) + return_unit + | Some (block, header) -> + let* () = f {block with header} in + loop header.predecessor + in + loop head.header.block_hash diff --git a/src/proto_alpha/lib_sc_rollup_node/store.mli b/src/proto_alpha/lib_sc_rollup_node/store_v0.mli similarity index 78% rename from src/proto_alpha/lib_sc_rollup_node/store.mli rename to src/proto_alpha/lib_sc_rollup_node/store_v0.mli index b0bf1810f7d8..f2d9a002d6c1 100644 --- a/src/proto_alpha/lib_sc_rollup_node/store.mli +++ b/src/proto_alpha/lib_sc_rollup_node/store_v0.mli @@ -24,11 +24,25 @@ (* *) (*****************************************************************************) +(** This version of the store is used for the rollup nodes for protocol Mumbai, + i.e. = 16. + + This interface is a copy of + [src/proto_016_PtMumbai/lib_sc_rollup_node/store.mli], which contains the + layout for the Mumbai rollup node. +*) + open Protocol open Alpha_context open Indexed_store -module Irmin_store : Store_sigs.Store +module Irmin_store : sig + include module type of Irmin_store.Make (struct + let name = "Tezos smart rollup node" + end) + + include Store_sigs.Store with type 'a t := 'a t +end module L2_blocks : INDEXED_FILE @@ -41,7 +55,7 @@ module Messages : INDEXED_FILE with type key := Sc_rollup.Inbox_merkelized_payload_hashes.Hash.t and type value := Sc_rollup.Inbox_message.t list - and type header := bool * Block_hash.t * Timestamp.t * int + and type header := Block_hash.t * Timestamp.t * int (** Aggregated collection of messages from the L1 inbox *) module Inboxes : @@ -108,14 +122,27 @@ module Dal_confirmed_slots_histories : and type value := Dal.Slots_history.History_cache.t and type 'a store := 'a Irmin_store.t -(** [Dal_slots_statuses] is a [Store_utils.Nested_map] used to store the - attestation status of DAL slots. The values of this storage module have type - `[`Confirmed | `Unconfirmed]`, depending on whether the content of the slot - has been attested on L1 or not. If an entry is not present for a - [(block_hash, slot_index)], this means that the corresponding block is not - processed yet. +(** [Dal_slot_pages] is a [Store_utils.Nested_map] used to store the contents + of dal slots fetched by the rollup node, as a list of pages. The values of + this storage module have type `string list`. A value of the form + [page_contents] refers to a page of a slot that has been confirmed, and + whose contents are [page_contents]. +*) +module Dal_slot_pages : + Store_sigs.Nested_map + with type primary_key := Block_hash.t + and type secondary_key := Dal.Slot_index.t * Dal.Page.Index.t + and type value := Dal.Page.content + and type 'a store := 'a Irmin_store.t + +(** [Dal_processed_slots] is a [Store_utils.Nested_map] used to store the processing + status of dal slots content fetched by the rollup node. The values of + this storage module have type `[`Confirmed | `Unconfirmed]`, depending on + whether the content of the slot has been confirmed or not. If an entry is + not present for a [(block_hash, slot_index)], this either means that it's + not processed yet. *) -module Dal_slots_statuses : +module Dal_processed_slots : Store_sigs.Nested_map with type primary_key := Block_hash.t and type secondary_key := Dal.Slot_index.t diff --git a/src/proto_alpha/lib_sc_rollup_node/store_v1.ml b/src/proto_alpha/lib_sc_rollup_node/store_v1.ml new file mode 100644 index 000000000000..e3ae2962e74f --- /dev/null +++ b/src/proto_alpha/lib_sc_rollup_node/store_v1.ml @@ -0,0 +1,251 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2021 Nomadic Labs, *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +open Protocol +open Alpha_context +include Store_sigs +include Store_utils +include Store_v0 + +module Make_hash_index_key (H : Environment.S.HASH) = +Indexed_store.Make_index_key (struct + include Indexed_store.Make_fixed_encodable (H) + + let equal = H.equal +end) + +(** Unaggregated messages per block *) +module Messages = + Indexed_store.Make_indexed_file + (struct + let name = "messages" + end) + (Make_hash_index_key (Sc_rollup.Inbox_merkelized_payload_hashes.Hash)) + (struct + type t = Sc_rollup.Inbox_message.t list + + let name = "messages_list" + + let encoding = + Data_encoding.(list @@ dynamic_size Sc_rollup.Inbox_message.encoding) + + module Header = struct + type t = bool * Block_hash.t * Timestamp.t * int + + let name = "messages_inbox_info" + + let encoding = + let open Data_encoding in + obj4 + (req "is_first_block" bool) + (req "predecessor" Block_hash.encoding) + (req "predecessor_timestamp" Timestamp.encoding) + (req "num_messages" int31) + + let fixed_size = + WithExceptions.Option.get ~loc:__LOC__ + @@ Data_encoding.Binary.fixed_length encoding + end + end) + +module Dal_pages = struct + type removed_in_v1 +end + +module Dal_processed_slots = struct + type removed_in_v1 +end + +(** Store attestation statuses for DAL slots on L1. *) +module Dal_slots_statuses = + Irmin_store.Make_nested_map + (struct + let path = ["dal"; "slots_statuses"] + end) + (struct + type key = Block_hash.t + + let to_path_representation = Block_hash.to_b58check + end) + (struct + type key = Dal.Slot_index.t + + let encoding = Dal.Slot_index.encoding + + let compare = Dal.Slot_index.compare + + let name = "slot_index" + end) + (struct + (* TODO: https://gitlab.com/tezos/tezos/-/issues/4780. + + Rename Confirm-ed/ation to Attest-ed/ation in the rollup node. *) + type value = [`Confirmed | `Unconfirmed] + + let name = "slot_status" + + let encoding = + (* We don't use + Data_encoding.string_enum because a union is more storage-efficient. *) + let open Data_encoding in + union + ~tag_size:`Uint8 + [ + case + ~title:"Confirmed" + (Tag 0) + (obj1 (req "kind" (constant "Confirmed"))) + (function `Confirmed -> Some () | `Unconfirmed -> None) + (fun () -> `Confirmed); + case + ~title:"Unconfirmed" + (Tag 1) + (obj1 (req "kind" (constant "Unconfirmed"))) + (function `Unconfirmed -> Some () | `Confirmed -> None) + (fun () -> `Unconfirmed); + ] + end) + +type nonrec 'a store = { + l2_blocks : 'a L2_blocks.t; + messages : 'a Messages.t; + inboxes : 'a Inboxes.t; + commitments : 'a Commitments.t; + commitments_published_at_level : 'a Commitments_published_at_level.t; + l2_head : 'a L2_head.t; + last_finalized_level : 'a Last_finalized_level.t; + levels_to_hashes : 'a Levels_to_hashes.t; + irmin_store : 'a Irmin_store.t; +} + +type 'a t = ([< `Read | `Write > `Read] as 'a) store + +type rw = Store_sigs.rw t + +type ro = Store_sigs.ro t + +let readonly + ({ + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head; + last_finalized_level; + levels_to_hashes; + irmin_store; + } : + _ t) : ro = + { + l2_blocks = L2_blocks.readonly l2_blocks; + messages = Messages.readonly messages; + inboxes = Inboxes.readonly inboxes; + commitments = Commitments.readonly commitments; + commitments_published_at_level = + Commitments_published_at_level.readonly commitments_published_at_level; + l2_head = L2_head.readonly l2_head; + last_finalized_level = Last_finalized_level.readonly last_finalized_level; + levels_to_hashes = Levels_to_hashes.readonly levels_to_hashes; + irmin_store = Irmin_store.readonly irmin_store; + } + +let close + ({ + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head = _; + last_finalized_level = _; + levels_to_hashes; + irmin_store; + } : + _ t) = + let open Lwt_result_syntax in + let+ () = L2_blocks.close l2_blocks + and+ () = Messages.close messages + and+ () = Inboxes.close inboxes + and+ () = Commitments.close commitments + and+ () = Commitments_published_at_level.close commitments_published_at_level + and+ () = Levels_to_hashes.close levels_to_hashes + and+ () = Irmin_store.close irmin_store in + () + +let load (type a) (mode : a mode) ~l2_blocks_cache_size data_dir : + a store tzresult Lwt.t = + let open Lwt_result_syntax in + let path name = Filename.concat data_dir name in + let cache_size = l2_blocks_cache_size in + let* l2_blocks = L2_blocks.load mode ~path:(path "l2_blocks") ~cache_size in + let* messages = Messages.load mode ~path:(path "messages") ~cache_size in + let* inboxes = Inboxes.load mode ~path:(path "inboxes") ~cache_size in + let* commitments = Commitments.load mode ~path:(path "commitments") in + let* commitments_published_at_level = + Commitments_published_at_level.load + mode + ~path:(path "commitments_published_at_level") + in + let* l2_head = L2_head.load mode ~path:(path "l2_head") in + let* last_finalized_level = + Last_finalized_level.load mode ~path:(path "last_finalized_level") + in + let* levels_to_hashes = + Levels_to_hashes.load mode ~path:(path "levels_to_hashes") + in + let+ irmin_store = Irmin_store.load mode (path "irmin_store") in + { + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head; + last_finalized_level; + levels_to_hashes; + irmin_store; + } + +let iter_l2_blocks ({l2_blocks; l2_head; _} : _ t) f = + let open Lwt_result_syntax in + let* head = L2_head.read l2_head in + match head with + | None -> + (* No reachable head, nothing to do *) + return_unit + | Some head -> + let rec loop hash = + let* block = L2_blocks.read l2_blocks hash in + match block with + | None -> + (* The block does not exist, the known chain stops here, so do we. *) + return_unit + | Some (block, header) -> + let* () = f {block with header} in + loop header.predecessor + in + loop head.header.block_hash diff --git a/src/proto_alpha/lib_sc_rollup_node/store_v1.mli b/src/proto_alpha/lib_sc_rollup_node/store_v1.mli new file mode 100644 index 000000000000..766290712100 --- /dev/null +++ b/src/proto_alpha/lib_sc_rollup_node/store_v1.mli @@ -0,0 +1,79 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2022 Nomadic Labs, *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(** This version of the store is used for the rollup nodes for protocols for and + after Nairobi, i.e. >= 17. *) + +open Protocol +open Alpha_context +open Indexed_store + +include module type of struct + include Store_v0 +end + +(** Storage for persisting messages downloaded from the L1 node. *) +module Messages : + INDEXED_FILE + with type key := Sc_rollup.Inbox_merkelized_payload_hashes.Hash.t + and type value := Sc_rollup.Inbox_message.t list + and type header := bool * Block_hash.t * Timestamp.t * int + +module Dal_pages : sig + type removed_in_v1 +end + +module Dal_processed_slots : sig + type removed_in_v1 +end + +(** [Dal_slots_statuses] is a [Store_utils.Nested_map] used to store the + attestation status of DAL slots. The values of this storage module have type + `[`Confirmed | `Unconfirmed]`, depending on whether the content of the slot + has been attested on L1 or not. If an entry is not present for a + [(block_hash, slot_index)], this means that the corresponding block is not + processed yet. +*) +module Dal_slots_statuses : + Store_sigs.Nested_map + with type primary_key := Block_hash.t + and type secondary_key := Dal.Slot_index.t + and type value := [`Confirmed | `Unconfirmed] + and type 'a store := 'a Irmin_store.t + +type +'a store = { + l2_blocks : 'a L2_blocks.t; + messages : 'a Messages.t; + inboxes : 'a Inboxes.t; + commitments : 'a Commitments.t; + commitments_published_at_level : 'a Commitments_published_at_level.t; + l2_head : 'a L2_head.t; + last_finalized_level : 'a Last_finalized_level.t; + levels_to_hashes : 'a Levels_to_hashes.t; + irmin_store : 'a Irmin_store.t; +} + +include Store_sig.S with type 'a store := 'a store -- GitLab From df1a32ac8b6dea590d8874d09a000f2aaddfb7d1 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Fri, 12 May 2023 12:20:34 +0200 Subject: [PATCH 07/13] SCORU/Node: generic store migration logic --- .../lib_sc_rollup_node/node_context.ml | 4 + .../lib_sc_rollup_node/store_migration.ml | 215 ++++++++++++++++++ .../lib_sc_rollup_node/store_migration.mli | 67 ++++++ .../lib_sc_rollup_node/store_sig.ml | 3 + .../lib_sc_rollup_node/store_v0.ml | 2 + .../lib_sc_rollup_node/store_v1.ml | 2 + .../lib_sc_rollup_node/store_version.ml | 29 +++ .../lib_sc_rollup_node/store_version.mli | 29 +++ 8 files changed, 351 insertions(+) create mode 100644 src/proto_alpha/lib_sc_rollup_node/store_migration.ml create mode 100644 src/proto_alpha/lib_sc_rollup_node/store_migration.mli create mode 100644 src/proto_alpha/lib_sc_rollup_node/store_version.ml create mode 100644 src/proto_alpha/lib_sc_rollup_node/store_version.mli diff --git a/src/proto_alpha/lib_sc_rollup_node/node_context.ml b/src/proto_alpha/lib_sc_rollup_node/node_context.ml index e8d49390a4fa..f3d0be249b2d 100644 --- a/src/proto_alpha/lib_sc_rollup_node/node_context.ml +++ b/src/proto_alpha/lib_sc_rollup_node/node_context.ml @@ -290,6 +290,10 @@ let init (cctxt : Protocol_client_context.full) ~data_dir ?log_kernel_debug_file let open Lwt_result_syntax in let*? () = check_config configuration in let* lockfile = lock ~data_dir in + let* () = + Store_migration.maybe_run_migration + ~storage_dir:(Configuration.default_storage_dir data_dir) + in let dal_cctxt = Option.map Dal_node_client.make_unix_cctxt dal_node_endpoint in diff --git a/src/proto_alpha/lib_sc_rollup_node/store_migration.ml b/src/proto_alpha/lib_sc_rollup_node/store_migration.ml new file mode 100644 index 000000000000..d5fc57c71982 --- /dev/null +++ b/src/proto_alpha/lib_sc_rollup_node/store_migration.ml @@ -0,0 +1,215 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +open Store_version + +let messages_store_location ~storage_dir = + let open Filename.Infix in + storage_dir // "messages" + +let version_of_unversionned_store ~storage_dir = + let open Lwt_syntax in + let path = messages_store_location ~storage_dir in + let* messages_store_v0 = Store_v0.Messages.load ~path ~cache_size:1 Read_only + and* messages_store_v1 = + Store_v1.Messages.load ~path ~cache_size:1 Read_only + in + let cleanup () = + let open Lwt_syntax in + let* (_ : unit tzresult) = + match messages_store_v0 with + | Error _ -> Lwt.return_ok () + | Ok s -> Store_v0.Messages.close s + and* (_ : unit tzresult) = + match messages_store_v1 with + | Error _ -> Lwt.return_ok () + | Ok s -> Store_v1.Messages.close s + in + return_unit + in + let guess_version () = + let open Lwt_result_syntax in + match (messages_store_v0, messages_store_v1) with + | Ok _, Error _ -> return_some V0 + | Error _, Ok _ -> return_some V1 + | Ok _, Ok _ -> + (* Empty store, both loads succeed *) + return_none + | Error _, Error _ -> + failwith + "Cannot determine unversionned store version (no messages decodable)" + in + Lwt.finalize guess_version cleanup + +let version_of_store ~storage_dir = + (* TODO: https://gitlab.com/tezos/tezos/-/issues/5554 + Use store version information when available. *) + version_of_unversionned_store ~storage_dir + +module type MIGRATION_ACTIONS = sig + type from_store + + type dest_store + + val migrate_block_action : + from_store -> dest_store -> Sc_rollup_block.t -> unit tzresult Lwt.t + + val final_actions : + storage_dir:string -> + tmp_dir:string -> + from_store -> + dest_store -> + unit tzresult Lwt.t +end + +module type S = sig + val migrate : storage_dir:string -> unit tzresult Lwt.t +end + +let migrations = Stdlib.Hashtbl.create 7 + +module Make + (S_from : Store_sig.S) + (S_dest : Store_sig.S) + (Actions : MIGRATION_ACTIONS + with type from_store := Store_sigs.ro S_from.t + and type dest_store := Store_sigs.rw S_dest.t) : S = struct + let tmp_dir ~storage_dir = + Filename.concat (Configuration.default_storage_dir storage_dir) + @@ Format.asprintf + "migration_%a_%a" + Store_version.pp + S_from.version + Store_version.pp + S_dest.version + + let migrate ~storage_dir = + let open Lwt_result_syntax in + let* source_store = + S_from.load Read_only ~l2_blocks_cache_size:1 storage_dir + in + let tmp_dir = tmp_dir ~storage_dir in + let*! tmp_dir_exists = Lwt_utils_unix.dir_exists tmp_dir in + let*? () = + if tmp_dir_exists then + error_with + "Store migration (from %a to %a) is already ongoing. Wait for it to \ + finish or remove %S and restart." + Store_version.pp + S_from.version + Store_version.pp + S_dest.version + tmp_dir + else Ok () + in + let*! () = Lwt_utils_unix.create_dir tmp_dir in + let* dest_store = S_dest.load Read_write ~l2_blocks_cache_size:1 tmp_dir in + let cleanup () = + let open Lwt_syntax in + let* (_ : unit tzresult) = S_from.close source_store + and* (_ : unit tzresult) = S_dest.close dest_store in + (* Don't remove migration dir to allow for later resume. *) + return_unit + in + let run_migration () = + let* () = + S_from.iter_l2_blocks + source_store + (Actions.migrate_block_action source_store dest_store) + in + let* () = + Actions.final_actions ~storage_dir ~tmp_dir source_store dest_store + in + let*! () = Lwt_utils_unix.remove_dir tmp_dir in + return_unit + in + Lwt.finalize run_migration cleanup + + let () = Stdlib.Hashtbl.add migrations S_from.version (S_dest.version, migrate) +end + +let migration_path ~from ~dest = + let rec path acc from dest = + if from = dest then Some (List.rev acc) + else + let first_steps = + Stdlib.Hashtbl.find_all migrations from + |> List.stable_sort (fun (va, _) (vb, _) -> + (* Try biggest jumps first that don't go beyond the + destination *) + if va > dest && vb > dest then Stdlib.compare va vb + else if va > dest then 1 + else if vb > dest then -1 + else Stdlib.compare vb va) + in + (* Recursively look for migration sub-paths *) + let paths = + List.filter_map + (fun (step, migration) -> + path ((from, step, migration) :: acc) step dest) + first_steps + in + (* Choose shortest migration path *) + List.stable_sort List.compare_lengths paths |> List.hd + in + path [] from dest + +let maybe_run_migration ~storage_dir = + let open Lwt_result_syntax in + let* current_version = version_of_store ~storage_dir in + let last_version = Store.version in + match (current_version, last_version) with + | None, _ -> + (* Store not initialized, nothing to do *) + return_unit + | Some current, last when last = current -> + (* Up to date, nothing to do *) + return_unit + | Some current, last -> ( + let migrations = migration_path ~from:current ~dest:last in + match migrations with + | None -> + failwith + "Store version %a is not supported by this rollup node because \ + there is no migration path from it to %a." + Store_version.pp + current + Store_version.pp + last + | Some migrations -> + Format.printf "Starting store migration@." ; + let+ () = + List.iter_es + (fun (vx, vy, migrate) -> + Format.printf + "- Migrating store from %a to %a@." + Store_version.pp + vx + Store_version.pp + vy ; + migrate ~storage_dir) + migrations + in + Format.printf "Store migration completed@.") diff --git a/src/proto_alpha/lib_sc_rollup_node/store_migration.mli b/src/proto_alpha/lib_sc_rollup_node/store_migration.mli new file mode 100644 index 000000000000..0b99202ca768 --- /dev/null +++ b/src/proto_alpha/lib_sc_rollup_node/store_migration.mli @@ -0,0 +1,67 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(** Type of parameter for migration functor {!Make}. *) +module type MIGRATION_ACTIONS = sig + (** Type of store from which data is migrated. *) + type from_store + + (** Type of store to which the data is migrated. *) + type dest_store + + (** Action or actions to migrate data associated to a block. NOTE: + [dest_store] is an empty R/W store initialized in a temporary location. *) + val migrate_block_action : + from_store -> dest_store -> Sc_rollup_block.t -> unit tzresult Lwt.t + + (** The final actions to be performed in the migration. In particular, this is + where data from the temporary store in [dest_store] in [tmp_dir] should be + reported in the actual [storage_dir]. *) + val final_actions : + storage_dir:string -> + tmp_dir:string -> + from_store -> + dest_store -> + unit tzresult Lwt.t +end + +module type S = sig + (** Migration function for the store located in [storage_dir]. *) + val migrate : storage_dir:string -> unit tzresult Lwt.t +end + +(** Functor to create and {e register} a migration. *) +module Make + (S_from : Store_sig.S) + (S_dest : Store_sig.S) + (Actions : MIGRATION_ACTIONS + with type from_store := Store_sigs.ro S_from.t + and type dest_store := Store_sigs.rw S_dest.t) : S + +(** Migrate store located in rollup node {e store} directory [storage_dir] if + needed. If there is no possible migration path registered to go from the + current version to the last {!Store.version}, this function resolves with an + error. *) +val maybe_run_migration : storage_dir:string -> unit tzresult Lwt.t diff --git a/src/proto_alpha/lib_sc_rollup_node/store_sig.ml b/src/proto_alpha/lib_sc_rollup_node/store_sig.ml index 3493ef2df1ae..ffac28ff0409 100644 --- a/src/proto_alpha/lib_sc_rollup_node/store_sig.ml +++ b/src/proto_alpha/lib_sc_rollup_node/store_sig.ml @@ -36,6 +36,9 @@ module type S = sig (** Read only store {!t}. *) type ro = Store_sigs.ro t + (** Version supported by this code. *) + val version : Store_version.t + (** [close store] closes the store. *) val close : _ t -> unit tzresult Lwt.t diff --git a/src/proto_alpha/lib_sc_rollup_node/store_v0.ml b/src/proto_alpha/lib_sc_rollup_node/store_v0.ml index 58f8b20c7ca1..066a37f739fb 100644 --- a/src/proto_alpha/lib_sc_rollup_node/store_v0.ml +++ b/src/proto_alpha/lib_sc_rollup_node/store_v0.ml @@ -35,6 +35,8 @@ include Store_utils (** Aggregated collection of messages from the L1 inbox *) open Alpha_context +let version = Store_version.V0 + module Irmin_store = struct module IStore = Irmin_store.Make (struct let name = "Tezos smart rollup node" diff --git a/src/proto_alpha/lib_sc_rollup_node/store_v1.ml b/src/proto_alpha/lib_sc_rollup_node/store_v1.ml index e3ae2962e74f..adecbb8a2b5b 100644 --- a/src/proto_alpha/lib_sc_rollup_node/store_v1.ml +++ b/src/proto_alpha/lib_sc_rollup_node/store_v1.ml @@ -30,6 +30,8 @@ include Store_sigs include Store_utils include Store_v0 +let version = Store_version.V1 + module Make_hash_index_key (H : Environment.S.HASH) = Indexed_store.Make_index_key (struct include Indexed_store.Make_fixed_encodable (H) diff --git a/src/proto_alpha/lib_sc_rollup_node/store_version.ml b/src/proto_alpha/lib_sc_rollup_node/store_version.ml new file mode 100644 index 000000000000..ec294eb1a23f --- /dev/null +++ b/src/proto_alpha/lib_sc_rollup_node/store_version.ml @@ -0,0 +1,29 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +type t = V0 | V1 + +let pp ppf v = + Format.pp_print_string ppf @@ match v with V0 -> "v0" | V1 -> "v1" diff --git a/src/proto_alpha/lib_sc_rollup_node/store_version.mli b/src/proto_alpha/lib_sc_rollup_node/store_version.mli new file mode 100644 index 000000000000..86b8485e19c1 --- /dev/null +++ b/src/proto_alpha/lib_sc_rollup_node/store_version.mli @@ -0,0 +1,29 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +type t = V0 | V1 + +(** Pretty-printer for store versions *) +val pp : Format.formatter -> t -> unit -- GitLab From 67828c196194f479a1892859a2c56aa1e965e5e6 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Thu, 11 May 2023 14:38:24 +0200 Subject: [PATCH 08/13] SCORU/Node/Store: migrate messages from v0 to v1 --- .../lib_sc_rollup_node/store_migration.ml | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/proto_alpha/lib_sc_rollup_node/store_migration.ml b/src/proto_alpha/lib_sc_rollup_node/store_migration.ml index d5fc57c71982..dd842c6772ad 100644 --- a/src/proto_alpha/lib_sc_rollup_node/store_migration.ml +++ b/src/proto_alpha/lib_sc_rollup_node/store_migration.ml @@ -213,3 +213,48 @@ let maybe_run_migration ~storage_dir = migrations in Format.printf "Store migration completed@.") + +module V0_to_V1 = struct + let convert_store_messages + (messages, (block_hash, timestamp, number_of_messages)) = + ( messages, + (false (* is migration block *), block_hash, timestamp, number_of_messages) + ) + + let migrate_messages (v0_store : _ Store_v0.t) (v1_store : _ Store_v1.t) + (l2_block : Sc_rollup_block.t) = + let open Lwt_result_syntax in + let* v0_messages = + Store_v0.Messages.read v0_store.messages l2_block.header.inbox_witness + in + match v0_messages with + | None -> return_unit + | Some v0_messages -> + let value, header = convert_store_messages v0_messages in + Store_v1.Messages.append + v1_store.messages + ~key:l2_block.header.inbox_witness + ~header + ~value + + let final_actions ~storage_dir ~tmp_dir (_v0_store : _ Store_v0.t) + (_v1_store : _ Store_v1.t) = + let open Lwt_result_syntax in + let*! () = + Lwt_utils_unix.remove_dir (messages_store_location ~storage_dir) + in + let*! () = + Lwt_unix.rename + (messages_store_location ~storage_dir:tmp_dir) + (messages_store_location ~storage_dir) + in + return_unit + + include + Make (Store_v0) (Store_v1) + (struct + let migrate_block_action = migrate_messages + + let final_actions = final_actions + end) +end -- GitLab From d57e389e10d0cd1ce8720f3dcdb59025a73c0bdc Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Thu, 11 May 2023 12:49:50 +0200 Subject: [PATCH 09/13] SCORU/Node/Store: migrate DAL data from v0 to v1 --- .../lib_sc_rollup_node/store_migration.ml | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/src/proto_alpha/lib_sc_rollup_node/store_migration.ml b/src/proto_alpha/lib_sc_rollup_node/store_migration.ml index dd842c6772ad..5e3c45dc772b 100644 --- a/src/proto_alpha/lib_sc_rollup_node/store_migration.ml +++ b/src/proto_alpha/lib_sc_rollup_node/store_migration.ml @@ -237,8 +237,40 @@ module V0_to_V1 = struct ~header ~value + (* In place migration of processed slots under new key name by hand *) + let migrate_dal_processed_slots_irmin (v1_store : _ Store_v1.t) = + let open Lwt_syntax in + let open Store_v1 in + let info () = + let date = + Tezos_base.Time.( + System.now () |> System.to_protocol |> Protocol.to_seconds) + in + let author = + Format.asprintf + "Rollup node %a" + Tezos_version_parser.pp + Tezos_version.Current_git_info.version + in + let message = "Migration store from v0 to v1" in + Irmin_store.Raw_irmin.Info.v ~author ~message date + in + let store = Irmin_store.Raw_irmin.unsafe v1_store.irmin_store in + let old_root = Store_v0.Dal_processed_slots.path in + let new_root = Dal_slots_statuses.path in + let* old_tree = Irmin_store.Raw_irmin.find_tree store old_root in + match old_tree with + | None -> return_unit + | Some _ -> + (* Move the tree in the new key *) + Irmin_store.Raw_irmin.with_tree_exn + ~info + store + new_root + (fun _new_tree -> return old_tree) + let final_actions ~storage_dir ~tmp_dir (_v0_store : _ Store_v0.t) - (_v1_store : _ Store_v1.t) = + (v1_store : _ Store_v1.t) = let open Lwt_result_syntax in let*! () = Lwt_utils_unix.remove_dir (messages_store_location ~storage_dir) @@ -248,6 +280,7 @@ module V0_to_V1 = struct (messages_store_location ~storage_dir:tmp_dir) (messages_store_location ~storage_dir) in + let*! () = migrate_dal_processed_slots_irmin v1_store in return_unit include -- GitLab From 12c90f0fb7366ae89b91d35702b9ea361c611af2 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Tue, 9 May 2023 12:41:47 +0200 Subject: [PATCH 10/13] SCORU/Node: Backport !8676 to Nairobi rollup node, use store v1 - SCORU/Node/Store: iter on L2 blocks - SCORU/Node/Store: signature for versioned store - SCORU/Node/Store: two versions of the store module - SCORU/Node: generic store migration logic - SCORU/Node/Store: migrate messages from v0 to v1 - SCORU/Node/Store: migration of DAL data from v0 to v1 --- .../lib_sc_rollup_node/node_context.ml | 4 + .../lib_sc_rollup_node/store.ml | 429 +-------------- .../lib_sc_rollup_node/store_migration.ml | 293 ++++++++++ .../lib_sc_rollup_node/store_migration.mli | 67 +++ .../lib_sc_rollup_node/store_sig.ml | 65 +++ .../lib_sc_rollup_node/store_v0.ml | 506 ++++++++++++++++++ .../{store.mli => store_v0.mli} | 77 +-- .../lib_sc_rollup_node/store_v1.ml | 253 +++++++++ .../lib_sc_rollup_node/store_v1.mli | 79 +++ .../lib_sc_rollup_node/store_version.ml | 29 + .../lib_sc_rollup_node/store_version.mli | 29 + 11 files changed, 1366 insertions(+), 465 deletions(-) create mode 100644 src/proto_017_PtNairob/lib_sc_rollup_node/store_migration.ml create mode 100644 src/proto_017_PtNairob/lib_sc_rollup_node/store_migration.mli create mode 100644 src/proto_017_PtNairob/lib_sc_rollup_node/store_sig.ml create mode 100644 src/proto_017_PtNairob/lib_sc_rollup_node/store_v0.ml rename src/proto_017_PtNairob/lib_sc_rollup_node/{store.mli => store_v0.mli} (75%) create mode 100644 src/proto_017_PtNairob/lib_sc_rollup_node/store_v1.ml create mode 100644 src/proto_017_PtNairob/lib_sc_rollup_node/store_v1.mli create mode 100644 src/proto_017_PtNairob/lib_sc_rollup_node/store_version.ml create mode 100644 src/proto_017_PtNairob/lib_sc_rollup_node/store_version.mli diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/node_context.ml b/src/proto_017_PtNairob/lib_sc_rollup_node/node_context.ml index ffbab6706ed1..017d28c07b1f 100644 --- a/src/proto_017_PtNairob/lib_sc_rollup_node/node_context.ml +++ b/src/proto_017_PtNairob/lib_sc_rollup_node/node_context.ml @@ -303,6 +303,10 @@ let init (cctxt : Protocol_client_context.full) ~data_dir ?log_kernel_debug_file |> Protocol.Alpha_context.Sc_rollup.Address.of_bytes_exn in let* lockfile = lock ~data_dir in + let* () = + Store_migration.maybe_run_migration + ~storage_dir:(Configuration.default_storage_dir data_dir) + in let dal_cctxt = Option.map Dal_node_client.make_unix_cctxt dal_node_endpoint in diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/store.ml b/src/proto_017_PtNairob/lib_sc_rollup_node/store.ml index 4f87a0d36983..90d354f79c80 100644 --- a/src/proto_017_PtNairob/lib_sc_rollup_node/store.ml +++ b/src/proto_017_PtNairob/lib_sc_rollup_node/store.ml @@ -2,6 +2,7 @@ (* *) (* Open Source License *) (* Copyright (c) 2021 Nomadic Labs, *) +(* Copyright (c) 2023 Functori, *) (* *) (* Permission is hereby granted, free of charge, to any person obtaining a *) (* copy of this software and associated documentation files (the "Software"),*) @@ -23,430 +24,4 @@ (* *) (*****************************************************************************) -open Protocol -include Store_sigs -include Store_utils - -(** Aggregated collection of messages from the L1 inbox *) -open Alpha_context - -module Irmin_store = struct - module IStore = Irmin_store.Make (struct - let name = "Tezos smart rollup node" - end) - - include IStore - include Store_utils.Make (IStore) -end - -module Empty_header = struct - type t = unit - - let name = "empty" - - let encoding = Data_encoding.unit - - let fixed_size = 0 -end - -module Add_empty_header = struct - module Header = Empty_header - - let header _ = () -end - -module Make_hash_index_key (H : Environment.S.HASH) = -Indexed_store.Make_index_key (struct - include Indexed_store.Make_fixed_encodable (H) - - let equal = H.equal -end) - -(** L2 blocks *) -module L2_blocks = - Indexed_store.Make_indexed_file - (struct - let name = "l2_blocks" - end) - (Tezos_store_shared.Block_key) - (struct - type t = (unit, unit) Sc_rollup_block.block - - let name = "sc_rollup_block_info" - - let encoding = - Sc_rollup_block.block_encoding Data_encoding.unit Data_encoding.unit - - module Header = struct - type t = Sc_rollup_block.header - - let name = "sc_rollup_block_header" - - let encoding = Sc_rollup_block.header_encoding - - let fixed_size = Sc_rollup_block.header_size - end - end) - -(** Unaggregated messages per block *) -module Messages = - Indexed_store.Make_indexed_file - (struct - let name = "messages" - end) - (Make_hash_index_key (Sc_rollup.Inbox_merkelized_payload_hashes.Hash)) - (struct - type t = Sc_rollup.Inbox_message.t list - - let name = "messages_list" - - let encoding = - Data_encoding.(list @@ dynamic_size Sc_rollup.Inbox_message.encoding) - - module Header = struct - type t = bool * Block_hash.t * Timestamp.t * int - - let name = "messages_inbox_info" - - let encoding = - let open Data_encoding in - obj4 - (req "is_first_block" bool) - (req "predecessor" Block_hash.encoding) - (req "predecessor_timestamp" Timestamp.encoding) - (req "num_messages" int31) - - let fixed_size = - WithExceptions.Option.get ~loc:__LOC__ - @@ Data_encoding.Binary.fixed_length encoding - end - end) - -(** Inbox state for each block *) -module Inboxes = - Indexed_store.Make_simple_indexed_file - (struct - let name = "inboxes" - end) - (Make_hash_index_key (Sc_rollup.Inbox.Hash)) - (struct - type t = Sc_rollup.Inbox.t - - let name = "inbox" - - let encoding = Sc_rollup.Inbox.encoding - - include Add_empty_header - end) - -module Commitments = - Indexed_store.Make_indexable - (struct - let name = "commitments" - end) - (Make_hash_index_key (Sc_rollup.Commitment.Hash)) - (Indexed_store.Make_index_value (Indexed_store.Make_fixed_encodable (struct - include Sc_rollup.Commitment - - let name = "commitment" - end))) - -module Commitments_published_at_level = struct - type element = { - first_published_at_level : Raw_level.t; - published_at_level : Raw_level.t option; - } - - let element_encoding = - let open Data_encoding in - let opt_level_encoding = - conv - (function None -> -1l | Some l -> Raw_level.to_int32 l) - (fun l -> if l = -1l then None else Some (Raw_level.of_int32_exn l)) - Data_encoding.int32 - in - conv - (fun {first_published_at_level; published_at_level} -> - (first_published_at_level, published_at_level)) - (fun (first_published_at_level, published_at_level) -> - {first_published_at_level; published_at_level}) - @@ obj2 - (req "first_published_at_level" Raw_level.encoding) - (req "published_at_level" opt_level_encoding) - - include - Indexed_store.Make_indexable - (struct - let name = "commitments" - end) - (Make_hash_index_key (Sc_rollup.Commitment.Hash)) - (Indexed_store.Make_index_value (Indexed_store.Make_fixed_encodable (struct - type t = element - - let name = "published_levels" - - let encoding = element_encoding - end))) -end - -module L2_head = Indexed_store.Make_singleton (struct - type t = Sc_rollup_block.t - - let name = "l2_head" - - let encoding = Sc_rollup_block.encoding -end) - -module Last_finalized_level = Indexed_store.Make_singleton (struct - type t = int32 - - let name = "finalized_level" - - let encoding = Data_encoding.int32 -end) - -(** Table from L1 levels to blocks hashes. *) -module Levels_to_hashes = - Indexed_store.Make_indexable - (struct - let name = "tezos_levels" - end) - (Indexed_store.Make_index_key (struct - type t = int32 - - let encoding = Data_encoding.int32 - - let name = "level" - - let fixed_size = 4 - - let equal = Int32.equal - end)) - (Tezos_store_shared.Block_key) - -(** Store attestation statuses for DAL slots on L1. *) -module Dal_slots_statuses = - Irmin_store.Make_nested_map - (struct - let path = ["dal"; "slots_statuses"] - end) - (struct - type key = Block_hash.t - - let to_path_representation = Block_hash.to_b58check - end) - (struct - type key = Dal.Slot_index.t - - let encoding = Dal.Slot_index.encoding - - let compare = Dal.Slot_index.compare - - let name = "slot_index" - end) - (struct - (* TODO: https://gitlab.com/tezos/tezos/-/issues/4780. - - Rename Confirm-ed/ation to Attest-ed/ation in the rollup node. *) - type value = [`Confirmed | `Unconfirmed] - - let name = "slot_status" - - let encoding = - (* We don't use - Data_encoding.string_enum because a union is more storage-efficient. *) - let open Data_encoding in - union - ~tag_size:`Uint8 - [ - case - ~title:"Confirmed" - (Tag 0) - (obj1 (req "kind" (constant "Confirmed"))) - (function `Confirmed -> Some () | `Unconfirmed -> None) - (fun () -> `Confirmed); - case - ~title:"Unconfirmed" - (Tag 1) - (obj1 (req "kind" (constant "Unconfirmed"))) - (function `Unconfirmed -> Some () | `Confirmed -> None) - (fun () -> `Unconfirmed); - ] - end) - -module Dal_slots_headers = - Irmin_store.Make_nested_map - (struct - let path = ["dal"; "slot_headers"] - end) - (struct - type key = Block_hash.t - - let to_path_representation = Block_hash.to_b58check - end) - (struct - type key = Dal.Slot_index.t - - let encoding = Dal.Slot_index.encoding - - let compare = Dal.Slot_index.compare - - let name = "slot_index" - end) - (struct - type value = Dal.Slot.Header.t - - let name = "slot_header" - - let encoding = Dal.Slot.Header.encoding - end) - -(* Published slot headers per block hash, stored as a list of bindings from - `Dal_slot_index.t` to `Dal.Slot.t`. The encoding function converts this - list into a `Dal.Slot_index.t`-indexed map. Note that the block_hash - refers to the block where slots headers have been confirmed, not - the block where they have been published. -*) - -(** Confirmed DAL slots history. See documentation of - {Dal_slot_repr.Slots_history} for more details. *) -module Dal_confirmed_slots_history = - Irmin_store.Make_append_only_map - (struct - let path = ["dal"; "confirmed_slots_history"] - end) - (struct - type key = Block_hash.t - - let to_path_representation = Block_hash.to_b58check - end) - (struct - type value = Dal.Slots_history.t - - let name = "dal_slot_histories" - - let encoding = Dal.Slots_history.encoding - end) - -(** Confirmed DAL slots histories cache. See documentation of - {Dal_slot_repr.Slots_history} for more details. *) -module Dal_confirmed_slots_histories = - (* TODO: https://gitlab.com/tezos/tezos/-/issues/4390 - Store single history points in map instead of whole history. *) - Irmin_store.Make_append_only_map - (struct - let path = ["dal"; "confirmed_slots_histories_cache"] - end) - (struct - type key = Block_hash.t - - let to_path_representation = Block_hash.to_b58check - end) - (struct - type value = Dal.Slots_history.History_cache.t - - let name = "dal_slot_history_cache" - - let encoding = Dal.Slots_history.History_cache.encoding - end) - -type 'a store = { - l2_blocks : 'a L2_blocks.t; - messages : 'a Messages.t; - inboxes : 'a Inboxes.t; - commitments : 'a Commitments.t; - commitments_published_at_level : 'a Commitments_published_at_level.t; - l2_head : 'a L2_head.t; - last_finalized_level : 'a Last_finalized_level.t; - levels_to_hashes : 'a Levels_to_hashes.t; - irmin_store : 'a Irmin_store.t; -} - -type 'a t = ([< `Read | `Write > `Read] as 'a) store - -type rw = Store_sigs.rw t - -type ro = Store_sigs.ro t - -let readonly - ({ - l2_blocks; - messages; - inboxes; - commitments; - commitments_published_at_level; - l2_head; - last_finalized_level; - levels_to_hashes; - irmin_store; - } : - _ t) : ro = - { - l2_blocks = L2_blocks.readonly l2_blocks; - messages = Messages.readonly messages; - inboxes = Inboxes.readonly inboxes; - commitments = Commitments.readonly commitments; - commitments_published_at_level = - Commitments_published_at_level.readonly commitments_published_at_level; - l2_head = L2_head.readonly l2_head; - last_finalized_level = Last_finalized_level.readonly last_finalized_level; - levels_to_hashes = Levels_to_hashes.readonly levels_to_hashes; - irmin_store = Irmin_store.readonly irmin_store; - } - -let close - ({ - l2_blocks; - messages; - inboxes; - commitments; - commitments_published_at_level; - l2_head = _; - last_finalized_level = _; - levels_to_hashes; - irmin_store; - } : - _ t) = - let open Lwt_result_syntax in - let+ () = L2_blocks.close l2_blocks - and+ () = Messages.close messages - and+ () = Inboxes.close inboxes - and+ () = Commitments.close commitments - and+ () = Commitments_published_at_level.close commitments_published_at_level - and+ () = Levels_to_hashes.close levels_to_hashes - and+ () = Irmin_store.close irmin_store in - () - -let load (type a) (mode : a mode) ~l2_blocks_cache_size data_dir : - a store tzresult Lwt.t = - let open Lwt_result_syntax in - let path name = Filename.concat data_dir name in - let cache_size = l2_blocks_cache_size in - let* l2_blocks = L2_blocks.load mode ~path:(path "l2_blocks") ~cache_size in - let* messages = Messages.load mode ~path:(path "messages") ~cache_size in - let* inboxes = Inboxes.load mode ~path:(path "inboxes") ~cache_size in - let* commitments = Commitments.load mode ~path:(path "commitments") in - let* commitments_published_at_level = - Commitments_published_at_level.load - mode - ~path:(path "commitments_published_at_level") - in - let* l2_head = L2_head.load mode ~path:(path "l2_head") in - let* last_finalized_level = - Last_finalized_level.load mode ~path:(path "last_finalized_level") - in - let* levels_to_hashes = - Levels_to_hashes.load mode ~path:(path "levels_to_hashes") - in - let+ irmin_store = Irmin_store.load mode (path "irmin_store") in - { - l2_blocks; - messages; - inboxes; - commitments; - commitments_published_at_level; - l2_head; - last_finalized_level; - levels_to_hashes; - irmin_store; - } +include Store_v1 diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/store_migration.ml b/src/proto_017_PtNairob/lib_sc_rollup_node/store_migration.ml new file mode 100644 index 000000000000..5e3c45dc772b --- /dev/null +++ b/src/proto_017_PtNairob/lib_sc_rollup_node/store_migration.ml @@ -0,0 +1,293 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +open Store_version + +let messages_store_location ~storage_dir = + let open Filename.Infix in + storage_dir // "messages" + +let version_of_unversionned_store ~storage_dir = + let open Lwt_syntax in + let path = messages_store_location ~storage_dir in + let* messages_store_v0 = Store_v0.Messages.load ~path ~cache_size:1 Read_only + and* messages_store_v1 = + Store_v1.Messages.load ~path ~cache_size:1 Read_only + in + let cleanup () = + let open Lwt_syntax in + let* (_ : unit tzresult) = + match messages_store_v0 with + | Error _ -> Lwt.return_ok () + | Ok s -> Store_v0.Messages.close s + and* (_ : unit tzresult) = + match messages_store_v1 with + | Error _ -> Lwt.return_ok () + | Ok s -> Store_v1.Messages.close s + in + return_unit + in + let guess_version () = + let open Lwt_result_syntax in + match (messages_store_v0, messages_store_v1) with + | Ok _, Error _ -> return_some V0 + | Error _, Ok _ -> return_some V1 + | Ok _, Ok _ -> + (* Empty store, both loads succeed *) + return_none + | Error _, Error _ -> + failwith + "Cannot determine unversionned store version (no messages decodable)" + in + Lwt.finalize guess_version cleanup + +let version_of_store ~storage_dir = + (* TODO: https://gitlab.com/tezos/tezos/-/issues/5554 + Use store version information when available. *) + version_of_unversionned_store ~storage_dir + +module type MIGRATION_ACTIONS = sig + type from_store + + type dest_store + + val migrate_block_action : + from_store -> dest_store -> Sc_rollup_block.t -> unit tzresult Lwt.t + + val final_actions : + storage_dir:string -> + tmp_dir:string -> + from_store -> + dest_store -> + unit tzresult Lwt.t +end + +module type S = sig + val migrate : storage_dir:string -> unit tzresult Lwt.t +end + +let migrations = Stdlib.Hashtbl.create 7 + +module Make + (S_from : Store_sig.S) + (S_dest : Store_sig.S) + (Actions : MIGRATION_ACTIONS + with type from_store := Store_sigs.ro S_from.t + and type dest_store := Store_sigs.rw S_dest.t) : S = struct + let tmp_dir ~storage_dir = + Filename.concat (Configuration.default_storage_dir storage_dir) + @@ Format.asprintf + "migration_%a_%a" + Store_version.pp + S_from.version + Store_version.pp + S_dest.version + + let migrate ~storage_dir = + let open Lwt_result_syntax in + let* source_store = + S_from.load Read_only ~l2_blocks_cache_size:1 storage_dir + in + let tmp_dir = tmp_dir ~storage_dir in + let*! tmp_dir_exists = Lwt_utils_unix.dir_exists tmp_dir in + let*? () = + if tmp_dir_exists then + error_with + "Store migration (from %a to %a) is already ongoing. Wait for it to \ + finish or remove %S and restart." + Store_version.pp + S_from.version + Store_version.pp + S_dest.version + tmp_dir + else Ok () + in + let*! () = Lwt_utils_unix.create_dir tmp_dir in + let* dest_store = S_dest.load Read_write ~l2_blocks_cache_size:1 tmp_dir in + let cleanup () = + let open Lwt_syntax in + let* (_ : unit tzresult) = S_from.close source_store + and* (_ : unit tzresult) = S_dest.close dest_store in + (* Don't remove migration dir to allow for later resume. *) + return_unit + in + let run_migration () = + let* () = + S_from.iter_l2_blocks + source_store + (Actions.migrate_block_action source_store dest_store) + in + let* () = + Actions.final_actions ~storage_dir ~tmp_dir source_store dest_store + in + let*! () = Lwt_utils_unix.remove_dir tmp_dir in + return_unit + in + Lwt.finalize run_migration cleanup + + let () = Stdlib.Hashtbl.add migrations S_from.version (S_dest.version, migrate) +end + +let migration_path ~from ~dest = + let rec path acc from dest = + if from = dest then Some (List.rev acc) + else + let first_steps = + Stdlib.Hashtbl.find_all migrations from + |> List.stable_sort (fun (va, _) (vb, _) -> + (* Try biggest jumps first that don't go beyond the + destination *) + if va > dest && vb > dest then Stdlib.compare va vb + else if va > dest then 1 + else if vb > dest then -1 + else Stdlib.compare vb va) + in + (* Recursively look for migration sub-paths *) + let paths = + List.filter_map + (fun (step, migration) -> + path ((from, step, migration) :: acc) step dest) + first_steps + in + (* Choose shortest migration path *) + List.stable_sort List.compare_lengths paths |> List.hd + in + path [] from dest + +let maybe_run_migration ~storage_dir = + let open Lwt_result_syntax in + let* current_version = version_of_store ~storage_dir in + let last_version = Store.version in + match (current_version, last_version) with + | None, _ -> + (* Store not initialized, nothing to do *) + return_unit + | Some current, last when last = current -> + (* Up to date, nothing to do *) + return_unit + | Some current, last -> ( + let migrations = migration_path ~from:current ~dest:last in + match migrations with + | None -> + failwith + "Store version %a is not supported by this rollup node because \ + there is no migration path from it to %a." + Store_version.pp + current + Store_version.pp + last + | Some migrations -> + Format.printf "Starting store migration@." ; + let+ () = + List.iter_es + (fun (vx, vy, migrate) -> + Format.printf + "- Migrating store from %a to %a@." + Store_version.pp + vx + Store_version.pp + vy ; + migrate ~storage_dir) + migrations + in + Format.printf "Store migration completed@.") + +module V0_to_V1 = struct + let convert_store_messages + (messages, (block_hash, timestamp, number_of_messages)) = + ( messages, + (false (* is migration block *), block_hash, timestamp, number_of_messages) + ) + + let migrate_messages (v0_store : _ Store_v0.t) (v1_store : _ Store_v1.t) + (l2_block : Sc_rollup_block.t) = + let open Lwt_result_syntax in + let* v0_messages = + Store_v0.Messages.read v0_store.messages l2_block.header.inbox_witness + in + match v0_messages with + | None -> return_unit + | Some v0_messages -> + let value, header = convert_store_messages v0_messages in + Store_v1.Messages.append + v1_store.messages + ~key:l2_block.header.inbox_witness + ~header + ~value + + (* In place migration of processed slots under new key name by hand *) + let migrate_dal_processed_slots_irmin (v1_store : _ Store_v1.t) = + let open Lwt_syntax in + let open Store_v1 in + let info () = + let date = + Tezos_base.Time.( + System.now () |> System.to_protocol |> Protocol.to_seconds) + in + let author = + Format.asprintf + "Rollup node %a" + Tezos_version_parser.pp + Tezos_version.Current_git_info.version + in + let message = "Migration store from v0 to v1" in + Irmin_store.Raw_irmin.Info.v ~author ~message date + in + let store = Irmin_store.Raw_irmin.unsafe v1_store.irmin_store in + let old_root = Store_v0.Dal_processed_slots.path in + let new_root = Dal_slots_statuses.path in + let* old_tree = Irmin_store.Raw_irmin.find_tree store old_root in + match old_tree with + | None -> return_unit + | Some _ -> + (* Move the tree in the new key *) + Irmin_store.Raw_irmin.with_tree_exn + ~info + store + new_root + (fun _new_tree -> return old_tree) + + let final_actions ~storage_dir ~tmp_dir (_v0_store : _ Store_v0.t) + (v1_store : _ Store_v1.t) = + let open Lwt_result_syntax in + let*! () = + Lwt_utils_unix.remove_dir (messages_store_location ~storage_dir) + in + let*! () = + Lwt_unix.rename + (messages_store_location ~storage_dir:tmp_dir) + (messages_store_location ~storage_dir) + in + let*! () = migrate_dal_processed_slots_irmin v1_store in + return_unit + + include + Make (Store_v0) (Store_v1) + (struct + let migrate_block_action = migrate_messages + + let final_actions = final_actions + end) +end diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/store_migration.mli b/src/proto_017_PtNairob/lib_sc_rollup_node/store_migration.mli new file mode 100644 index 000000000000..0b99202ca768 --- /dev/null +++ b/src/proto_017_PtNairob/lib_sc_rollup_node/store_migration.mli @@ -0,0 +1,67 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(** Type of parameter for migration functor {!Make}. *) +module type MIGRATION_ACTIONS = sig + (** Type of store from which data is migrated. *) + type from_store + + (** Type of store to which the data is migrated. *) + type dest_store + + (** Action or actions to migrate data associated to a block. NOTE: + [dest_store] is an empty R/W store initialized in a temporary location. *) + val migrate_block_action : + from_store -> dest_store -> Sc_rollup_block.t -> unit tzresult Lwt.t + + (** The final actions to be performed in the migration. In particular, this is + where data from the temporary store in [dest_store] in [tmp_dir] should be + reported in the actual [storage_dir]. *) + val final_actions : + storage_dir:string -> + tmp_dir:string -> + from_store -> + dest_store -> + unit tzresult Lwt.t +end + +module type S = sig + (** Migration function for the store located in [storage_dir]. *) + val migrate : storage_dir:string -> unit tzresult Lwt.t +end + +(** Functor to create and {e register} a migration. *) +module Make + (S_from : Store_sig.S) + (S_dest : Store_sig.S) + (Actions : MIGRATION_ACTIONS + with type from_store := Store_sigs.ro S_from.t + and type dest_store := Store_sigs.rw S_dest.t) : S + +(** Migrate store located in rollup node {e store} directory [storage_dir] if + needed. If there is no possible migration path registered to go from the + current version to the last {!Store.version}, this function resolves with an + error. *) +val maybe_run_migration : storage_dir:string -> unit tzresult Lwt.t diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/store_sig.ml b/src/proto_017_PtNairob/lib_sc_rollup_node/store_sig.ml new file mode 100644 index 000000000000..ffac28ff0409 --- /dev/null +++ b/src/proto_017_PtNairob/lib_sc_rollup_node/store_sig.ml @@ -0,0 +1,65 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +module type S = sig + type +'a store + + (** Type of store. The parameter indicates if the store can be written or only + read. *) + type 'a t = ([< `Read | `Write > `Read] as 'a) store + + (** Read/write store {!t}. *) + type rw = Store_sigs.rw t + + (** Read only store {!t}. *) + type ro = Store_sigs.ro t + + (** Version supported by this code. *) + val version : Store_version.t + + (** [close store] closes the store. *) + val close : _ t -> unit tzresult Lwt.t + + (** [load mode ~l2_blocks_cache_size directory] loads a store from the data + persisted in [directory]. If [mode] is {!Store_sigs.Read_only}, then the + indexes and irmin store will be opened in readonly mode and only read + operations will be permitted. This allows to open a store for read access + that is already opened in {!Store_sigs.Read_write} mode in another + process. [l2_blocks_cache_size] is the number of L2 blocks the rollup node + will keep in memory. *) + val load : + 'a Store_sigs.mode -> + l2_blocks_cache_size:int -> + string -> + 'a store tzresult Lwt.t + + (** [readonly store] returns a read-only version of [store]. *) + val readonly : _ t -> ro + + (** [iter_l2_blocks store f] iterates [f] on all L2 blocks reachable from the + head, from newest to oldest. *) + val iter_l2_blocks : + _ t -> (Sc_rollup_block.t -> unit tzresult Lwt.t) -> unit tzresult Lwt.t +end diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/store_v0.ml b/src/proto_017_PtNairob/lib_sc_rollup_node/store_v0.ml new file mode 100644 index 000000000000..066a37f739fb --- /dev/null +++ b/src/proto_017_PtNairob/lib_sc_rollup_node/store_v0.ml @@ -0,0 +1,506 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2021 Nomadic Labs, *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(** This module is a copy of + [src/proto_016_PtMumbai/lib_sc_rollup_node/store.ml], which contains the + store for the Mumbai rollup node. *) + +open Protocol +include Store_sigs +include Store_utils + +(** Aggregated collection of messages from the L1 inbox *) +open Alpha_context + +let version = Store_version.V0 + +module Irmin_store = struct + module IStore = Irmin_store.Make (struct + let name = "Tezos smart rollup node" + end) + + include IStore + include Store_utils.Make (IStore) +end + +module Empty_header = struct + type t = unit + + let name = "empty" + + let encoding = Data_encoding.unit + + let fixed_size = 0 +end + +module Add_empty_header = struct + module Header = Empty_header + + let header _ = () +end + +module Make_hash_index_key (H : Environment.S.HASH) = +Indexed_store.Make_index_key (struct + include Indexed_store.Make_fixed_encodable (H) + + let equal = H.equal +end) + +(** L2 blocks *) +module L2_blocks = + Indexed_store.Make_indexed_file + (struct + let name = "l2_blocks" + end) + (Tezos_store_shared.Block_key) + (struct + type t = (unit, unit) Sc_rollup_block.block + + let name = "sc_rollup_block_info" + + let encoding = + Sc_rollup_block.block_encoding Data_encoding.unit Data_encoding.unit + + module Header = struct + type t = Sc_rollup_block.header + + let name = "sc_rollup_block_header" + + let encoding = Sc_rollup_block.header_encoding + + let fixed_size = Sc_rollup_block.header_size + end + end) + +(** Unaggregated messages per block *) +module Messages = + Indexed_store.Make_indexed_file + (struct + let name = "messages" + end) + (Make_hash_index_key (Sc_rollup.Inbox_merkelized_payload_hashes.Hash)) + (struct + type t = Sc_rollup.Inbox_message.t list + + let name = "messages_list" + + let encoding = + Data_encoding.(list @@ dynamic_size Sc_rollup.Inbox_message.encoding) + + module Header = struct + type t = Block_hash.t * Timestamp.t * int + + let name = "messages_inbox_info" + + let encoding = + let open Data_encoding in + obj3 + (req "predecessor" Block_hash.encoding) + (req "predecessor_timestamp" Timestamp.encoding) + (req "num_messages" int31) + + let fixed_size = + WithExceptions.Option.get ~loc:__LOC__ + @@ Data_encoding.Binary.fixed_length encoding + end + end) + +(** Inbox state for each block *) +module Inboxes = + Indexed_store.Make_simple_indexed_file + (struct + let name = "inboxes" + end) + (Make_hash_index_key (Sc_rollup.Inbox.Hash)) + (struct + type t = Sc_rollup.Inbox.t + + let name = "inbox" + + let encoding = Sc_rollup.Inbox.encoding + + include Add_empty_header + end) + +module Commitments = + Indexed_store.Make_indexable + (struct + let name = "commitments" + end) + (Make_hash_index_key (Sc_rollup.Commitment.Hash)) + (Indexed_store.Make_index_value (Indexed_store.Make_fixed_encodable (struct + include Sc_rollup.Commitment + + let name = "commitment" + end))) + +module Commitments_published_at_level = struct + type element = { + first_published_at_level : Raw_level.t; + published_at_level : Raw_level.t option; + } + + let element_encoding = + let open Data_encoding in + let opt_level_encoding = + conv + (function None -> -1l | Some l -> Raw_level.to_int32 l) + (fun l -> if l = -1l then None else Some (Raw_level.of_int32_exn l)) + Data_encoding.int32 + in + conv + (fun {first_published_at_level; published_at_level} -> + (first_published_at_level, published_at_level)) + (fun (first_published_at_level, published_at_level) -> + {first_published_at_level; published_at_level}) + @@ obj2 + (req "first_published_at_level" Raw_level.encoding) + (req "published_at_level" opt_level_encoding) + + include + Indexed_store.Make_indexable + (struct + let name = "commitments" + end) + (Make_hash_index_key (Sc_rollup.Commitment.Hash)) + (Indexed_store.Make_index_value (Indexed_store.Make_fixed_encodable (struct + type t = element + + let name = "published_levels" + + let encoding = element_encoding + end))) +end + +module L2_head = Indexed_store.Make_singleton (struct + type t = Sc_rollup_block.t + + let name = "l2_head" + + let encoding = Sc_rollup_block.encoding +end) + +module Last_finalized_level = Indexed_store.Make_singleton (struct + type t = int32 + + let name = "finalized_level" + + let encoding = Data_encoding.int32 +end) + +(** Table from L1 levels to blocks hashes. *) +module Levels_to_hashes = + Indexed_store.Make_indexable + (struct + let name = "tezos_levels" + end) + (Indexed_store.Make_index_key (struct + type t = int32 + + let encoding = Data_encoding.int32 + + let name = "level" + + let fixed_size = 4 + + let equal = Int32.equal + end)) + (Tezos_store_shared.Block_key) + +(* Published slot headers per block hash, + stored as a list of bindings from `Dal_slot_index.t` + to `Dal.Slot.t`. The encoding function converts this + list into a `Dal.Slot_index.t`-indexed map. *) +module Dal_slot_pages = + Irmin_store.Make_nested_map + (struct + let path = ["dal"; "slot_pages"] + end) + (struct + type key = Block_hash.t + + let to_path_representation = Block_hash.to_b58check + end) + (struct + type key = Dal.Slot_index.t * Dal.Page.Index.t + + let encoding = + Data_encoding.(tup2 Dal.Slot_index.encoding Dal.Page.Index.encoding) + + let compare (i1, p1) (i2, p2) = + Compare.or_else (Dal.Slot_index.compare i1 i2) (fun () -> + Dal.Page.Index.compare p1 p2) + + let name = "slot_index" + end) + (struct + type value = Dal.Page.content + + let encoding = Dal.Page.content_encoding + + let name = "slot_pages" + end) + +(** stores slots whose data have been considered and pages stored to disk (if + they are confirmed). *) +module Dal_processed_slots = + Irmin_store.Make_nested_map + (struct + let path = ["dal"; "processed_slots"] + end) + (struct + type key = Block_hash.t + + let to_path_representation = Block_hash.to_b58check + end) + (struct + type key = Dal.Slot_index.t + + let encoding = Dal.Slot_index.encoding + + let compare = Dal.Slot_index.compare + + let name = "slot_index" + end) + (struct + type value = [`Confirmed | `Unconfirmed] + + let name = "slot_processing_status" + + let encoding = + let open Data_encoding in + let mk_case constr ~tag ~title = + case + ~title + (Tag tag) + (obj1 (req "kind" (constant title))) + (fun x -> if x = constr then Some () else None) + (fun () -> constr) + in + union + ~tag_size:`Uint8 + [ + mk_case `Confirmed ~tag:0 ~title:"Confirmed"; + mk_case `Unconfirmed ~tag:1 ~title:"Unconfirmed"; + ] + end) + +module Dal_slots_headers = + Irmin_store.Make_nested_map + (struct + let path = ["dal"; "slot_headers"] + end) + (struct + type key = Block_hash.t + + let to_path_representation = Block_hash.to_b58check + end) + (struct + type key = Dal.Slot_index.t + + let encoding = Dal.Slot_index.encoding + + let compare = Dal.Slot_index.compare + + let name = "slot_index" + end) + (struct + type value = Dal.Slot.Header.t + + let name = "slot_header" + + let encoding = Dal.Slot.Header.encoding + end) + +(* Published slot headers per block hash, stored as a list of bindings from + `Dal_slot_index.t` to `Dal.Slot.t`. The encoding function converts this + list into a `Dal.Slot_index.t`-indexed map. Note that the block_hash + refers to the block where slots headers have been confirmed, not + the block where they have been published. +*) + +(** Confirmed DAL slots history. See documentation of + {!Dal_slot_repr.Slots_history} for more details. *) +module Dal_confirmed_slots_history = + Irmin_store.Make_append_only_map + (struct + let path = ["dal"; "confirmed_slots_history"] + end) + (struct + type key = Block_hash.t + + let to_path_representation = Block_hash.to_b58check + end) + (struct + type value = Dal.Slots_history.t + + let name = "dal_slot_histories" + + let encoding = Dal.Slots_history.encoding + end) + +(** Confirmed DAL slots histories cache. See documentation of + {!Dal_slot_repr.Slots_history} for more details. *) +module Dal_confirmed_slots_histories = + (* TODO: https://gitlab.com/tezos/tezos/-/issues/4390 + Store single history points in map instead of whole history. *) + Irmin_store.Make_append_only_map + (struct + let path = ["dal"; "confirmed_slots_histories_cache"] + end) + (struct + type key = Block_hash.t + + let to_path_representation = Block_hash.to_b58check + end) + (struct + type value = Dal.Slots_history.History_cache.t + + let name = "dal_slot_history_cache" + + let encoding = Dal.Slots_history.History_cache.encoding + end) + +type 'a store = { + l2_blocks : 'a L2_blocks.t; + messages : 'a Messages.t; + inboxes : 'a Inboxes.t; + commitments : 'a Commitments.t; + commitments_published_at_level : 'a Commitments_published_at_level.t; + l2_head : 'a L2_head.t; + last_finalized_level : 'a Last_finalized_level.t; + levels_to_hashes : 'a Levels_to_hashes.t; + irmin_store : 'a Irmin_store.t; +} + +type 'a t = ([< `Read | `Write > `Read] as 'a) store + +type rw = Store_sigs.rw t + +type ro = Store_sigs.ro t + +let readonly + ({ + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head; + last_finalized_level; + levels_to_hashes; + irmin_store; + } : + _ t) : ro = + { + l2_blocks = L2_blocks.readonly l2_blocks; + messages = Messages.readonly messages; + inboxes = Inboxes.readonly inboxes; + commitments = Commitments.readonly commitments; + commitments_published_at_level = + Commitments_published_at_level.readonly commitments_published_at_level; + l2_head = L2_head.readonly l2_head; + last_finalized_level = Last_finalized_level.readonly last_finalized_level; + levels_to_hashes = Levels_to_hashes.readonly levels_to_hashes; + irmin_store = Irmin_store.readonly irmin_store; + } + +let close + ({ + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head = _; + last_finalized_level = _; + levels_to_hashes; + irmin_store; + } : + _ t) = + let open Lwt_result_syntax in + let+ () = L2_blocks.close l2_blocks + and+ () = Messages.close messages + and+ () = Inboxes.close inboxes + and+ () = Commitments.close commitments + and+ () = Commitments_published_at_level.close commitments_published_at_level + and+ () = Levels_to_hashes.close levels_to_hashes + and+ () = Irmin_store.close irmin_store in + () + +let load (type a) (mode : a mode) ~l2_blocks_cache_size data_dir : + a store tzresult Lwt.t = + let open Lwt_result_syntax in + let path name = Filename.concat data_dir name in + let cache_size = l2_blocks_cache_size in + let* l2_blocks = L2_blocks.load mode ~path:(path "l2_blocks") ~cache_size in + let* messages = Messages.load mode ~path:(path "messages") ~cache_size in + let* inboxes = Inboxes.load mode ~path:(path "inboxes") ~cache_size in + let* commitments = Commitments.load mode ~path:(path "commitments") in + let* commitments_published_at_level = + Commitments_published_at_level.load + mode + ~path:(path "commitments_published_at_level") + in + let* l2_head = L2_head.load mode ~path:(path "l2_head") in + let* last_finalized_level = + Last_finalized_level.load mode ~path:(path "last_finalized_level") + in + let* levels_to_hashes = + Levels_to_hashes.load mode ~path:(path "levels_to_hashes") + in + let+ irmin_store = Irmin_store.load mode (path "irmin_store") in + { + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head; + last_finalized_level; + levels_to_hashes; + irmin_store; + } + +let iter_l2_blocks ({l2_blocks; l2_head; _} : _ t) f = + let open Lwt_result_syntax in + let* head = L2_head.read l2_head in + match head with + | None -> + (* No reachable head, nothing to do *) + return_unit + | Some head -> + let rec loop hash = + let* block = L2_blocks.read l2_blocks hash in + match block with + | None -> + (* The block does not exist, the known chain stops here, so do we. *) + return_unit + | Some (block, header) -> + let* () = f {block with header} in + loop header.predecessor + in + loop head.header.block_hash diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/store.mli b/src/proto_017_PtNairob/lib_sc_rollup_node/store_v0.mli similarity index 75% rename from src/proto_017_PtNairob/lib_sc_rollup_node/store.mli rename to src/proto_017_PtNairob/lib_sc_rollup_node/store_v0.mli index 2bf1bd361a0f..f2d9a002d6c1 100644 --- a/src/proto_017_PtNairob/lib_sc_rollup_node/store.mli +++ b/src/proto_017_PtNairob/lib_sc_rollup_node/store_v0.mli @@ -2,6 +2,7 @@ (* *) (* Open Source License *) (* Copyright (c) 2022 Nomadic Labs, *) +(* Copyright (c) 2023 Functori, *) (* *) (* Permission is hereby granted, free of charge, to any person obtaining a *) (* copy of this software and associated documentation files (the "Software"),*) @@ -23,11 +24,25 @@ (* *) (*****************************************************************************) +(** This version of the store is used for the rollup nodes for protocol Mumbai, + i.e. = 16. + + This interface is a copy of + [src/proto_016_PtMumbai/lib_sc_rollup_node/store.mli], which contains the + layout for the Mumbai rollup node. +*) + open Protocol open Alpha_context open Indexed_store -module Irmin_store : Store_sigs.Store +module Irmin_store : sig + include module type of Irmin_store.Make (struct + let name = "Tezos smart rollup node" + end) + + include Store_sigs.Store with type 'a t := 'a t +end module L2_blocks : INDEXED_FILE @@ -40,7 +55,7 @@ module Messages : INDEXED_FILE with type key := Sc_rollup.Inbox_merkelized_payload_hashes.Hash.t and type value := Sc_rollup.Inbox_message.t list - and type header := bool * Block_hash.t * Timestamp.t * int + and type header := Block_hash.t * Timestamp.t * int (** Aggregated collection of messages from the L1 inbox *) module Inboxes : @@ -100,21 +115,34 @@ module Dal_confirmed_slots_history : and type 'a store := 'a Irmin_store.t (** Confirmed DAL slots histories cache. See documentation of - {Dal_slot_repr.Slots_history} for more details. *) + {!Dal_slot_repr.Slots_history} for more details. *) module Dal_confirmed_slots_histories : Store_sigs.Append_only_map with type key := Block_hash.t and type value := Dal.Slots_history.History_cache.t and type 'a store := 'a Irmin_store.t -(** [Dal_slots_statuses] is a [Store_utils.Nested_map] used to store the - attestation status of DAL slots. The values of this storage module have type - `[`Confirmed | `Unconfirmed]`, depending on whether the content of the slot - has been attested on L1 or not. If an entry is not present for a - [(block_hash, slot_index)], this means that the corresponding block is not - processed yet. +(** [Dal_slot_pages] is a [Store_utils.Nested_map] used to store the contents + of dal slots fetched by the rollup node, as a list of pages. The values of + this storage module have type `string list`. A value of the form + [page_contents] refers to a page of a slot that has been confirmed, and + whose contents are [page_contents]. +*) +module Dal_slot_pages : + Store_sigs.Nested_map + with type primary_key := Block_hash.t + and type secondary_key := Dal.Slot_index.t * Dal.Page.Index.t + and type value := Dal.Page.content + and type 'a store := 'a Irmin_store.t + +(** [Dal_processed_slots] is a [Store_utils.Nested_map] used to store the processing + status of dal slots content fetched by the rollup node. The values of + this storage module have type `[`Confirmed | `Unconfirmed]`, depending on + whether the content of the slot has been confirmed or not. If an entry is + not present for a [(block_hash, slot_index)], this either means that it's + not processed yet. *) -module Dal_slots_statuses : +module Dal_processed_slots : Store_sigs.Nested_map with type primary_key := Block_hash.t and type secondary_key := Dal.Slot_index.t @@ -133,31 +161,4 @@ type +'a store = { irmin_store : 'a Irmin_store.t; } -(** Type of store. The parameter indicates if the store can be written or only - read. *) -type 'a t = ([< `Read | `Write > `Read] as 'a) store - -(** Read/write store {!t}. *) -type rw = Store_sigs.rw t - -(** Read only store {!t}. *) -type ro = Store_sigs.ro t - -(** [close store] closes the store. *) -val close : _ t -> unit tzresult Lwt.t - -(** [load mode ~l2_blocks_cache_size directory] loads a store from the data - persisted in [directory]. If [mode] is {!Store_sigs.Read_only}, then the - indexes and irmin store will be opened in readonly mode and only read - operations will be permitted. This allows to open a store for read access - that is already opened in {!Store_sigs.Read_write} mode in another - process. [l2_blocks_cache_size] is the number of L2 blocks the rollup node - will keep in memory. *) -val load : - 'a Store_sigs.mode -> - l2_blocks_cache_size:int -> - string -> - 'a store tzresult Lwt.t - -(** [readonly store] returns a read-only version of [store]. *) -val readonly : _ t -> ro +include Store_sig.S with type 'a store := 'a store diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/store_v1.ml b/src/proto_017_PtNairob/lib_sc_rollup_node/store_v1.ml new file mode 100644 index 000000000000..adecbb8a2b5b --- /dev/null +++ b/src/proto_017_PtNairob/lib_sc_rollup_node/store_v1.ml @@ -0,0 +1,253 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2021 Nomadic Labs, *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +open Protocol +open Alpha_context +include Store_sigs +include Store_utils +include Store_v0 + +let version = Store_version.V1 + +module Make_hash_index_key (H : Environment.S.HASH) = +Indexed_store.Make_index_key (struct + include Indexed_store.Make_fixed_encodable (H) + + let equal = H.equal +end) + +(** Unaggregated messages per block *) +module Messages = + Indexed_store.Make_indexed_file + (struct + let name = "messages" + end) + (Make_hash_index_key (Sc_rollup.Inbox_merkelized_payload_hashes.Hash)) + (struct + type t = Sc_rollup.Inbox_message.t list + + let name = "messages_list" + + let encoding = + Data_encoding.(list @@ dynamic_size Sc_rollup.Inbox_message.encoding) + + module Header = struct + type t = bool * Block_hash.t * Timestamp.t * int + + let name = "messages_inbox_info" + + let encoding = + let open Data_encoding in + obj4 + (req "is_first_block" bool) + (req "predecessor" Block_hash.encoding) + (req "predecessor_timestamp" Timestamp.encoding) + (req "num_messages" int31) + + let fixed_size = + WithExceptions.Option.get ~loc:__LOC__ + @@ Data_encoding.Binary.fixed_length encoding + end + end) + +module Dal_pages = struct + type removed_in_v1 +end + +module Dal_processed_slots = struct + type removed_in_v1 +end + +(** Store attestation statuses for DAL slots on L1. *) +module Dal_slots_statuses = + Irmin_store.Make_nested_map + (struct + let path = ["dal"; "slots_statuses"] + end) + (struct + type key = Block_hash.t + + let to_path_representation = Block_hash.to_b58check + end) + (struct + type key = Dal.Slot_index.t + + let encoding = Dal.Slot_index.encoding + + let compare = Dal.Slot_index.compare + + let name = "slot_index" + end) + (struct + (* TODO: https://gitlab.com/tezos/tezos/-/issues/4780. + + Rename Confirm-ed/ation to Attest-ed/ation in the rollup node. *) + type value = [`Confirmed | `Unconfirmed] + + let name = "slot_status" + + let encoding = + (* We don't use + Data_encoding.string_enum because a union is more storage-efficient. *) + let open Data_encoding in + union + ~tag_size:`Uint8 + [ + case + ~title:"Confirmed" + (Tag 0) + (obj1 (req "kind" (constant "Confirmed"))) + (function `Confirmed -> Some () | `Unconfirmed -> None) + (fun () -> `Confirmed); + case + ~title:"Unconfirmed" + (Tag 1) + (obj1 (req "kind" (constant "Unconfirmed"))) + (function `Unconfirmed -> Some () | `Confirmed -> None) + (fun () -> `Unconfirmed); + ] + end) + +type nonrec 'a store = { + l2_blocks : 'a L2_blocks.t; + messages : 'a Messages.t; + inboxes : 'a Inboxes.t; + commitments : 'a Commitments.t; + commitments_published_at_level : 'a Commitments_published_at_level.t; + l2_head : 'a L2_head.t; + last_finalized_level : 'a Last_finalized_level.t; + levels_to_hashes : 'a Levels_to_hashes.t; + irmin_store : 'a Irmin_store.t; +} + +type 'a t = ([< `Read | `Write > `Read] as 'a) store + +type rw = Store_sigs.rw t + +type ro = Store_sigs.ro t + +let readonly + ({ + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head; + last_finalized_level; + levels_to_hashes; + irmin_store; + } : + _ t) : ro = + { + l2_blocks = L2_blocks.readonly l2_blocks; + messages = Messages.readonly messages; + inboxes = Inboxes.readonly inboxes; + commitments = Commitments.readonly commitments; + commitments_published_at_level = + Commitments_published_at_level.readonly commitments_published_at_level; + l2_head = L2_head.readonly l2_head; + last_finalized_level = Last_finalized_level.readonly last_finalized_level; + levels_to_hashes = Levels_to_hashes.readonly levels_to_hashes; + irmin_store = Irmin_store.readonly irmin_store; + } + +let close + ({ + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head = _; + last_finalized_level = _; + levels_to_hashes; + irmin_store; + } : + _ t) = + let open Lwt_result_syntax in + let+ () = L2_blocks.close l2_blocks + and+ () = Messages.close messages + and+ () = Inboxes.close inboxes + and+ () = Commitments.close commitments + and+ () = Commitments_published_at_level.close commitments_published_at_level + and+ () = Levels_to_hashes.close levels_to_hashes + and+ () = Irmin_store.close irmin_store in + () + +let load (type a) (mode : a mode) ~l2_blocks_cache_size data_dir : + a store tzresult Lwt.t = + let open Lwt_result_syntax in + let path name = Filename.concat data_dir name in + let cache_size = l2_blocks_cache_size in + let* l2_blocks = L2_blocks.load mode ~path:(path "l2_blocks") ~cache_size in + let* messages = Messages.load mode ~path:(path "messages") ~cache_size in + let* inboxes = Inboxes.load mode ~path:(path "inboxes") ~cache_size in + let* commitments = Commitments.load mode ~path:(path "commitments") in + let* commitments_published_at_level = + Commitments_published_at_level.load + mode + ~path:(path "commitments_published_at_level") + in + let* l2_head = L2_head.load mode ~path:(path "l2_head") in + let* last_finalized_level = + Last_finalized_level.load mode ~path:(path "last_finalized_level") + in + let* levels_to_hashes = + Levels_to_hashes.load mode ~path:(path "levels_to_hashes") + in + let+ irmin_store = Irmin_store.load mode (path "irmin_store") in + { + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head; + last_finalized_level; + levels_to_hashes; + irmin_store; + } + +let iter_l2_blocks ({l2_blocks; l2_head; _} : _ t) f = + let open Lwt_result_syntax in + let* head = L2_head.read l2_head in + match head with + | None -> + (* No reachable head, nothing to do *) + return_unit + | Some head -> + let rec loop hash = + let* block = L2_blocks.read l2_blocks hash in + match block with + | None -> + (* The block does not exist, the known chain stops here, so do we. *) + return_unit + | Some (block, header) -> + let* () = f {block with header} in + loop header.predecessor + in + loop head.header.block_hash diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/store_v1.mli b/src/proto_017_PtNairob/lib_sc_rollup_node/store_v1.mli new file mode 100644 index 000000000000..766290712100 --- /dev/null +++ b/src/proto_017_PtNairob/lib_sc_rollup_node/store_v1.mli @@ -0,0 +1,79 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2022 Nomadic Labs, *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(** This version of the store is used for the rollup nodes for protocols for and + after Nairobi, i.e. >= 17. *) + +open Protocol +open Alpha_context +open Indexed_store + +include module type of struct + include Store_v0 +end + +(** Storage for persisting messages downloaded from the L1 node. *) +module Messages : + INDEXED_FILE + with type key := Sc_rollup.Inbox_merkelized_payload_hashes.Hash.t + and type value := Sc_rollup.Inbox_message.t list + and type header := bool * Block_hash.t * Timestamp.t * int + +module Dal_pages : sig + type removed_in_v1 +end + +module Dal_processed_slots : sig + type removed_in_v1 +end + +(** [Dal_slots_statuses] is a [Store_utils.Nested_map] used to store the + attestation status of DAL slots. The values of this storage module have type + `[`Confirmed | `Unconfirmed]`, depending on whether the content of the slot + has been attested on L1 or not. If an entry is not present for a + [(block_hash, slot_index)], this means that the corresponding block is not + processed yet. +*) +module Dal_slots_statuses : + Store_sigs.Nested_map + with type primary_key := Block_hash.t + and type secondary_key := Dal.Slot_index.t + and type value := [`Confirmed | `Unconfirmed] + and type 'a store := 'a Irmin_store.t + +type +'a store = { + l2_blocks : 'a L2_blocks.t; + messages : 'a Messages.t; + inboxes : 'a Inboxes.t; + commitments : 'a Commitments.t; + commitments_published_at_level : 'a Commitments_published_at_level.t; + l2_head : 'a L2_head.t; + last_finalized_level : 'a Last_finalized_level.t; + levels_to_hashes : 'a Levels_to_hashes.t; + irmin_store : 'a Irmin_store.t; +} + +include Store_sig.S with type 'a store := 'a store diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/store_version.ml b/src/proto_017_PtNairob/lib_sc_rollup_node/store_version.ml new file mode 100644 index 000000000000..ec294eb1a23f --- /dev/null +++ b/src/proto_017_PtNairob/lib_sc_rollup_node/store_version.ml @@ -0,0 +1,29 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +type t = V0 | V1 + +let pp ppf v = + Format.pp_print_string ppf @@ match v with V0 -> "v0" | V1 -> "v1" diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/store_version.mli b/src/proto_017_PtNairob/lib_sc_rollup_node/store_version.mli new file mode 100644 index 000000000000..86b8485e19c1 --- /dev/null +++ b/src/proto_017_PtNairob/lib_sc_rollup_node/store_version.mli @@ -0,0 +1,29 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +type t = V0 | V1 + +(** Pretty-printer for store versions *) +val pp : Format.formatter -> t -> unit -- GitLab From 0d931e1d53df5a3116e831f6c962e5e42b4b8289 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Fri, 12 May 2023 09:33:12 +0200 Subject: [PATCH 11/13] SCORU/Node: Backport !8676 to Mumbai rollup node, use store v0 - SCORU/Node/Store: iter on L2 blocks - SCORU/Node/Store: signature for versioned store - SCORU/Node/Store: two versions of the store module - SCORU/Node: generic store migration logic - SCORU/Node/Store: migrate messages from v0 to v1 - SCORU/Node/Store: migration of DAL data from v0 to v1 --- .../lib_sc_rollup_node/node_context.ml | 4 + .../lib_sc_rollup_node/store.ml | 456 +--------------- .../lib_sc_rollup_node/store_migration.ml | 293 ++++++++++ .../lib_sc_rollup_node/store_migration.mli | 67 +++ .../lib_sc_rollup_node/store_sig.ml | 65 +++ .../lib_sc_rollup_node/store_v0.ml | 506 ++++++++++++++++++ .../{store.mli => store_v0.mli} | 48 +- .../lib_sc_rollup_node/store_v1.ml | 253 +++++++++ .../lib_sc_rollup_node/store_v1.mli | 79 +++ .../lib_sc_rollup_node/store_version.ml | 29 + .../lib_sc_rollup_node/store_version.mli | 29 + 11 files changed, 1345 insertions(+), 484 deletions(-) create mode 100644 src/proto_016_PtMumbai/lib_sc_rollup_node/store_migration.ml create mode 100644 src/proto_016_PtMumbai/lib_sc_rollup_node/store_migration.mli create mode 100644 src/proto_016_PtMumbai/lib_sc_rollup_node/store_sig.ml create mode 100644 src/proto_016_PtMumbai/lib_sc_rollup_node/store_v0.ml rename src/proto_016_PtMumbai/lib_sc_rollup_node/{store.mli => store_v0.mli} (85%) create mode 100644 src/proto_016_PtMumbai/lib_sc_rollup_node/store_v1.ml create mode 100644 src/proto_016_PtMumbai/lib_sc_rollup_node/store_v1.mli create mode 100644 src/proto_016_PtMumbai/lib_sc_rollup_node/store_version.ml create mode 100644 src/proto_016_PtMumbai/lib_sc_rollup_node/store_version.mli diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/node_context.ml b/src/proto_016_PtMumbai/lib_sc_rollup_node/node_context.ml index 64f584b83961..88ea8779abdd 100644 --- a/src/proto_016_PtMumbai/lib_sc_rollup_node/node_context.ml +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/node_context.ml @@ -303,6 +303,10 @@ let init (cctxt : Protocol_client_context.full) ~data_dir ?log_kernel_debug_file |> Protocol.Alpha_context.Sc_rollup.Address.of_bytes_exn in let* lockfile = lock ~data_dir in + let* () = + Store_migration.maybe_run_migration + ~storage_dir:(Configuration.default_storage_dir data_dir) + in let dal_cctxt = Option.map Dal_node_client.make_unix_cctxt dal_node_endpoint in diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/store.ml b/src/proto_016_PtMumbai/lib_sc_rollup_node/store.ml index 38e4c30c0772..b45c19513a98 100644 --- a/src/proto_016_PtMumbai/lib_sc_rollup_node/store.ml +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/store.ml @@ -2,6 +2,7 @@ (* *) (* Open Source License *) (* Copyright (c) 2021 Nomadic Labs, *) +(* Copyright (c) 2023 Functori, *) (* *) (* Permission is hereby granted, free of charge, to any person obtaining a *) (* copy of this software and associated documentation files (the "Software"),*) @@ -23,457 +24,4 @@ (* *) (*****************************************************************************) -open Protocol -include Store_sigs -include Store_utils - -(** Aggregated collection of messages from the L1 inbox *) -open Alpha_context - -module Irmin_store = struct - module IStore = Irmin_store.Make (struct - let name = "Tezos smart rollup node" - end) - - include IStore - include Store_utils.Make (IStore) -end - -module Empty_header = struct - type t = unit - - let name = "empty" - - let encoding = Data_encoding.unit - - let fixed_size = 0 -end - -module Add_empty_header = struct - module Header = Empty_header - - let header _ = () -end - -module Make_hash_index_key (H : Environment.S.HASH) = -Indexed_store.Make_index_key (struct - include Indexed_store.Make_fixed_encodable (H) - - let equal = H.equal -end) - -(** L2 blocks *) -module L2_blocks = - Indexed_store.Make_indexed_file - (struct - let name = "l2_blocks" - end) - (Tezos_store_shared.Block_key) - (struct - type t = (unit, unit) Sc_rollup_block.block - - let name = "sc_rollup_block_info" - - let encoding = - Sc_rollup_block.block_encoding Data_encoding.unit Data_encoding.unit - - module Header = struct - type t = Sc_rollup_block.header - - let name = "sc_rollup_block_header" - - let encoding = Sc_rollup_block.header_encoding - - let fixed_size = Sc_rollup_block.header_size - end - end) - -(** Unaggregated messages per block *) -module Messages = - Indexed_store.Make_indexed_file - (struct - let name = "messages" - end) - (Make_hash_index_key (Sc_rollup.Inbox_merkelized_payload_hashes.Hash)) - (struct - type t = Sc_rollup.Inbox_message.t list - - let name = "messages_list" - - let encoding = - Data_encoding.(list @@ dynamic_size Sc_rollup.Inbox_message.encoding) - - module Header = struct - type t = Block_hash.t * Timestamp.t * int - - let name = "messages_inbox_info" - - let encoding = - let open Data_encoding in - obj3 - (req "predecessor" Block_hash.encoding) - (req "predecessor_timestamp" Timestamp.encoding) - (req "num_messages" int31) - - let fixed_size = - WithExceptions.Option.get ~loc:__LOC__ - @@ Data_encoding.Binary.fixed_length encoding - end - end) - -(** Inbox state for each block *) -module Inboxes = - Indexed_store.Make_simple_indexed_file - (struct - let name = "inboxes" - end) - (Make_hash_index_key (Sc_rollup.Inbox.Hash)) - (struct - type t = Sc_rollup.Inbox.t - - let name = "inbox" - - let encoding = Sc_rollup.Inbox.encoding - - include Add_empty_header - end) - -module Commitments = - Indexed_store.Make_indexable - (struct - let name = "commitments" - end) - (Make_hash_index_key (Sc_rollup.Commitment.Hash)) - (Indexed_store.Make_index_value (Indexed_store.Make_fixed_encodable (struct - include Sc_rollup.Commitment - - let name = "commitment" - end))) - -module Commitments_published_at_level = struct - type element = { - first_published_at_level : Raw_level.t; - published_at_level : Raw_level.t option; - } - - let element_encoding = - let open Data_encoding in - let opt_level_encoding = - conv - (function None -> -1l | Some l -> Raw_level.to_int32 l) - (fun l -> if l = -1l then None else Some (Raw_level.of_int32_exn l)) - Data_encoding.int32 - in - conv - (fun {first_published_at_level; published_at_level} -> - (first_published_at_level, published_at_level)) - (fun (first_published_at_level, published_at_level) -> - {first_published_at_level; published_at_level}) - @@ obj2 - (req "first_published_at_level" Raw_level.encoding) - (req "published_at_level" opt_level_encoding) - - include - Indexed_store.Make_indexable - (struct - let name = "commitments" - end) - (Make_hash_index_key (Sc_rollup.Commitment.Hash)) - (Indexed_store.Make_index_value (Indexed_store.Make_fixed_encodable (struct - type t = element - - let name = "published_levels" - - let encoding = element_encoding - end))) -end - -module L2_head = Indexed_store.Make_singleton (struct - type t = Sc_rollup_block.t - - let name = "l2_head" - - let encoding = Sc_rollup_block.encoding -end) - -module Last_finalized_level = Indexed_store.Make_singleton (struct - type t = int32 - - let name = "finalized_level" - - let encoding = Data_encoding.int32 -end) - -(** Table from L1 levels to blocks hashes. *) -module Levels_to_hashes = - Indexed_store.Make_indexable - (struct - let name = "tezos_levels" - end) - (Indexed_store.Make_index_key (struct - type t = int32 - - let encoding = Data_encoding.int32 - - let name = "level" - - let fixed_size = 4 - - let equal = Int32.equal - end)) - (Tezos_store_shared.Block_key) - -(* Published slot headers per block hash, - stored as a list of bindings from `Dal_slot_index.t` - to `Dal.Slot.t`. The encoding function converts this - list into a `Dal.Slot_index.t`-indexed map. *) -module Dal_slot_pages = - Irmin_store.Make_nested_map - (struct - let path = ["dal"; "slot_pages"] - end) - (struct - type key = Block_hash.t - - let to_path_representation = Block_hash.to_b58check - end) - (struct - type key = Dal.Slot_index.t * Dal.Page.Index.t - - let encoding = - Data_encoding.(tup2 Dal.Slot_index.encoding Dal.Page.Index.encoding) - - let compare (i1, p1) (i2, p2) = - Compare.or_else (Dal.Slot_index.compare i1 i2) (fun () -> - Dal.Page.Index.compare p1 p2) - - let name = "slot_index" - end) - (struct - type value = Dal.Page.content - - let encoding = Dal.Page.content_encoding - - let name = "slot_pages" - end) - -(** stores slots whose data have been considered and pages stored to disk (if - they are confirmed). *) -module Dal_processed_slots = - Irmin_store.Make_nested_map - (struct - let path = ["dal"; "processed_slots"] - end) - (struct - type key = Block_hash.t - - let to_path_representation = Block_hash.to_b58check - end) - (struct - type key = Dal.Slot_index.t - - let encoding = Dal.Slot_index.encoding - - let compare = Dal.Slot_index.compare - - let name = "slot_index" - end) - (struct - type value = [`Confirmed | `Unconfirmed] - - let name = "slot_processing_status" - - let encoding = - let open Data_encoding in - let mk_case constr ~tag ~title = - case - ~title - (Tag tag) - (obj1 (req "kind" (constant title))) - (fun x -> if x = constr then Some () else None) - (fun () -> constr) - in - union - ~tag_size:`Uint8 - [ - mk_case `Confirmed ~tag:0 ~title:"Confirmed"; - mk_case `Unconfirmed ~tag:1 ~title:"Unconfirmed"; - ] - end) - -module Dal_slots_headers = - Irmin_store.Make_nested_map - (struct - let path = ["dal"; "slot_headers"] - end) - (struct - type key = Block_hash.t - - let to_path_representation = Block_hash.to_b58check - end) - (struct - type key = Dal.Slot_index.t - - let encoding = Dal.Slot_index.encoding - - let compare = Dal.Slot_index.compare - - let name = "slot_index" - end) - (struct - type value = Dal.Slot.Header.t - - let name = "slot_header" - - let encoding = Dal.Slot.Header.encoding - end) - -(* Published slot headers per block hash, stored as a list of bindings from - `Dal_slot_index.t` to `Dal.Slot.t`. The encoding function converts this - list into a `Dal.Slot_index.t`-indexed map. Note that the block_hash - refers to the block where slots headers have been confirmed, not - the block where they have been published. -*) - -(** Confirmed DAL slots history. See documentation of - {Dal_slot_repr.Slots_history} for more details. *) -module Dal_confirmed_slots_history = - Irmin_store.Make_append_only_map - (struct - let path = ["dal"; "confirmed_slots_history"] - end) - (struct - type key = Block_hash.t - - let to_path_representation = Block_hash.to_b58check - end) - (struct - type value = Dal.Slots_history.t - - let name = "dal_slot_histories" - - let encoding = Dal.Slots_history.encoding - end) - -(** Confirmed DAL slots histories cache. See documentation of - {Dal_slot_repr.Slots_history} for more details. *) -module Dal_confirmed_slots_histories = - (* TODO: https://gitlab.com/tezos/tezos/-/issues/4390 - Store single history points in map instead of whole history. *) - Irmin_store.Make_append_only_map - (struct - let path = ["dal"; "confirmed_slots_histories_cache"] - end) - (struct - type key = Block_hash.t - - let to_path_representation = Block_hash.to_b58check - end) - (struct - type value = Dal.Slots_history.History_cache.t - - let name = "dal_slot_history_cache" - - let encoding = Dal.Slots_history.History_cache.encoding - end) - -type 'a store = { - l2_blocks : 'a L2_blocks.t; - messages : 'a Messages.t; - inboxes : 'a Inboxes.t; - commitments : 'a Commitments.t; - commitments_published_at_level : 'a Commitments_published_at_level.t; - l2_head : 'a L2_head.t; - last_finalized_level : 'a Last_finalized_level.t; - levels_to_hashes : 'a Levels_to_hashes.t; - irmin_store : 'a Irmin_store.t; -} - -type 'a t = ([< `Read | `Write > `Read] as 'a) store - -type rw = Store_sigs.rw t - -type ro = Store_sigs.ro t - -let readonly - ({ - l2_blocks; - messages; - inboxes; - commitments; - commitments_published_at_level; - l2_head; - last_finalized_level; - levels_to_hashes; - irmin_store; - } : - _ t) : ro = - { - l2_blocks = L2_blocks.readonly l2_blocks; - messages = Messages.readonly messages; - inboxes = Inboxes.readonly inboxes; - commitments = Commitments.readonly commitments; - commitments_published_at_level = - Commitments_published_at_level.readonly commitments_published_at_level; - l2_head = L2_head.readonly l2_head; - last_finalized_level = Last_finalized_level.readonly last_finalized_level; - levels_to_hashes = Levels_to_hashes.readonly levels_to_hashes; - irmin_store = Irmin_store.readonly irmin_store; - } - -let close - ({ - l2_blocks; - messages; - inboxes; - commitments; - commitments_published_at_level; - l2_head = _; - last_finalized_level = _; - levels_to_hashes; - irmin_store; - } : - _ t) = - let open Lwt_result_syntax in - let+ () = L2_blocks.close l2_blocks - and+ () = Messages.close messages - and+ () = Inboxes.close inboxes - and+ () = Commitments.close commitments - and+ () = Commitments_published_at_level.close commitments_published_at_level - and+ () = Levels_to_hashes.close levels_to_hashes - and+ () = Irmin_store.close irmin_store in - () - -let load (type a) (mode : a mode) ~l2_blocks_cache_size data_dir : - a store tzresult Lwt.t = - let open Lwt_result_syntax in - let path name = Filename.concat data_dir name in - let cache_size = l2_blocks_cache_size in - let* l2_blocks = L2_blocks.load mode ~path:(path "l2_blocks") ~cache_size in - let* messages = Messages.load mode ~path:(path "messages") ~cache_size in - let* inboxes = Inboxes.load mode ~path:(path "inboxes") ~cache_size in - let* commitments = Commitments.load mode ~path:(path "commitments") in - let* commitments_published_at_level = - Commitments_published_at_level.load - mode - ~path:(path "commitments_published_at_level") - in - let* l2_head = L2_head.load mode ~path:(path "l2_head") in - let* last_finalized_level = - Last_finalized_level.load mode ~path:(path "last_finalized_level") - in - let* levels_to_hashes = - Levels_to_hashes.load mode ~path:(path "levels_to_hashes") - in - let+ irmin_store = Irmin_store.load mode (path "irmin_store") in - { - l2_blocks; - messages; - inboxes; - commitments; - commitments_published_at_level; - l2_head; - last_finalized_level; - levels_to_hashes; - irmin_store; - } +include Store_v0 diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/store_migration.ml b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_migration.ml new file mode 100644 index 000000000000..5e3c45dc772b --- /dev/null +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_migration.ml @@ -0,0 +1,293 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +open Store_version + +let messages_store_location ~storage_dir = + let open Filename.Infix in + storage_dir // "messages" + +let version_of_unversionned_store ~storage_dir = + let open Lwt_syntax in + let path = messages_store_location ~storage_dir in + let* messages_store_v0 = Store_v0.Messages.load ~path ~cache_size:1 Read_only + and* messages_store_v1 = + Store_v1.Messages.load ~path ~cache_size:1 Read_only + in + let cleanup () = + let open Lwt_syntax in + let* (_ : unit tzresult) = + match messages_store_v0 with + | Error _ -> Lwt.return_ok () + | Ok s -> Store_v0.Messages.close s + and* (_ : unit tzresult) = + match messages_store_v1 with + | Error _ -> Lwt.return_ok () + | Ok s -> Store_v1.Messages.close s + in + return_unit + in + let guess_version () = + let open Lwt_result_syntax in + match (messages_store_v0, messages_store_v1) with + | Ok _, Error _ -> return_some V0 + | Error _, Ok _ -> return_some V1 + | Ok _, Ok _ -> + (* Empty store, both loads succeed *) + return_none + | Error _, Error _ -> + failwith + "Cannot determine unversionned store version (no messages decodable)" + in + Lwt.finalize guess_version cleanup + +let version_of_store ~storage_dir = + (* TODO: https://gitlab.com/tezos/tezos/-/issues/5554 + Use store version information when available. *) + version_of_unversionned_store ~storage_dir + +module type MIGRATION_ACTIONS = sig + type from_store + + type dest_store + + val migrate_block_action : + from_store -> dest_store -> Sc_rollup_block.t -> unit tzresult Lwt.t + + val final_actions : + storage_dir:string -> + tmp_dir:string -> + from_store -> + dest_store -> + unit tzresult Lwt.t +end + +module type S = sig + val migrate : storage_dir:string -> unit tzresult Lwt.t +end + +let migrations = Stdlib.Hashtbl.create 7 + +module Make + (S_from : Store_sig.S) + (S_dest : Store_sig.S) + (Actions : MIGRATION_ACTIONS + with type from_store := Store_sigs.ro S_from.t + and type dest_store := Store_sigs.rw S_dest.t) : S = struct + let tmp_dir ~storage_dir = + Filename.concat (Configuration.default_storage_dir storage_dir) + @@ Format.asprintf + "migration_%a_%a" + Store_version.pp + S_from.version + Store_version.pp + S_dest.version + + let migrate ~storage_dir = + let open Lwt_result_syntax in + let* source_store = + S_from.load Read_only ~l2_blocks_cache_size:1 storage_dir + in + let tmp_dir = tmp_dir ~storage_dir in + let*! tmp_dir_exists = Lwt_utils_unix.dir_exists tmp_dir in + let*? () = + if tmp_dir_exists then + error_with + "Store migration (from %a to %a) is already ongoing. Wait for it to \ + finish or remove %S and restart." + Store_version.pp + S_from.version + Store_version.pp + S_dest.version + tmp_dir + else Ok () + in + let*! () = Lwt_utils_unix.create_dir tmp_dir in + let* dest_store = S_dest.load Read_write ~l2_blocks_cache_size:1 tmp_dir in + let cleanup () = + let open Lwt_syntax in + let* (_ : unit tzresult) = S_from.close source_store + and* (_ : unit tzresult) = S_dest.close dest_store in + (* Don't remove migration dir to allow for later resume. *) + return_unit + in + let run_migration () = + let* () = + S_from.iter_l2_blocks + source_store + (Actions.migrate_block_action source_store dest_store) + in + let* () = + Actions.final_actions ~storage_dir ~tmp_dir source_store dest_store + in + let*! () = Lwt_utils_unix.remove_dir tmp_dir in + return_unit + in + Lwt.finalize run_migration cleanup + + let () = Stdlib.Hashtbl.add migrations S_from.version (S_dest.version, migrate) +end + +let migration_path ~from ~dest = + let rec path acc from dest = + if from = dest then Some (List.rev acc) + else + let first_steps = + Stdlib.Hashtbl.find_all migrations from + |> List.stable_sort (fun (va, _) (vb, _) -> + (* Try biggest jumps first that don't go beyond the + destination *) + if va > dest && vb > dest then Stdlib.compare va vb + else if va > dest then 1 + else if vb > dest then -1 + else Stdlib.compare vb va) + in + (* Recursively look for migration sub-paths *) + let paths = + List.filter_map + (fun (step, migration) -> + path ((from, step, migration) :: acc) step dest) + first_steps + in + (* Choose shortest migration path *) + List.stable_sort List.compare_lengths paths |> List.hd + in + path [] from dest + +let maybe_run_migration ~storage_dir = + let open Lwt_result_syntax in + let* current_version = version_of_store ~storage_dir in + let last_version = Store.version in + match (current_version, last_version) with + | None, _ -> + (* Store not initialized, nothing to do *) + return_unit + | Some current, last when last = current -> + (* Up to date, nothing to do *) + return_unit + | Some current, last -> ( + let migrations = migration_path ~from:current ~dest:last in + match migrations with + | None -> + failwith + "Store version %a is not supported by this rollup node because \ + there is no migration path from it to %a." + Store_version.pp + current + Store_version.pp + last + | Some migrations -> + Format.printf "Starting store migration@." ; + let+ () = + List.iter_es + (fun (vx, vy, migrate) -> + Format.printf + "- Migrating store from %a to %a@." + Store_version.pp + vx + Store_version.pp + vy ; + migrate ~storage_dir) + migrations + in + Format.printf "Store migration completed@.") + +module V0_to_V1 = struct + let convert_store_messages + (messages, (block_hash, timestamp, number_of_messages)) = + ( messages, + (false (* is migration block *), block_hash, timestamp, number_of_messages) + ) + + let migrate_messages (v0_store : _ Store_v0.t) (v1_store : _ Store_v1.t) + (l2_block : Sc_rollup_block.t) = + let open Lwt_result_syntax in + let* v0_messages = + Store_v0.Messages.read v0_store.messages l2_block.header.inbox_witness + in + match v0_messages with + | None -> return_unit + | Some v0_messages -> + let value, header = convert_store_messages v0_messages in + Store_v1.Messages.append + v1_store.messages + ~key:l2_block.header.inbox_witness + ~header + ~value + + (* In place migration of processed slots under new key name by hand *) + let migrate_dal_processed_slots_irmin (v1_store : _ Store_v1.t) = + let open Lwt_syntax in + let open Store_v1 in + let info () = + let date = + Tezos_base.Time.( + System.now () |> System.to_protocol |> Protocol.to_seconds) + in + let author = + Format.asprintf + "Rollup node %a" + Tezos_version_parser.pp + Tezos_version.Current_git_info.version + in + let message = "Migration store from v0 to v1" in + Irmin_store.Raw_irmin.Info.v ~author ~message date + in + let store = Irmin_store.Raw_irmin.unsafe v1_store.irmin_store in + let old_root = Store_v0.Dal_processed_slots.path in + let new_root = Dal_slots_statuses.path in + let* old_tree = Irmin_store.Raw_irmin.find_tree store old_root in + match old_tree with + | None -> return_unit + | Some _ -> + (* Move the tree in the new key *) + Irmin_store.Raw_irmin.with_tree_exn + ~info + store + new_root + (fun _new_tree -> return old_tree) + + let final_actions ~storage_dir ~tmp_dir (_v0_store : _ Store_v0.t) + (v1_store : _ Store_v1.t) = + let open Lwt_result_syntax in + let*! () = + Lwt_utils_unix.remove_dir (messages_store_location ~storage_dir) + in + let*! () = + Lwt_unix.rename + (messages_store_location ~storage_dir:tmp_dir) + (messages_store_location ~storage_dir) + in + let*! () = migrate_dal_processed_slots_irmin v1_store in + return_unit + + include + Make (Store_v0) (Store_v1) + (struct + let migrate_block_action = migrate_messages + + let final_actions = final_actions + end) +end diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/store_migration.mli b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_migration.mli new file mode 100644 index 000000000000..0b99202ca768 --- /dev/null +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_migration.mli @@ -0,0 +1,67 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(** Type of parameter for migration functor {!Make}. *) +module type MIGRATION_ACTIONS = sig + (** Type of store from which data is migrated. *) + type from_store + + (** Type of store to which the data is migrated. *) + type dest_store + + (** Action or actions to migrate data associated to a block. NOTE: + [dest_store] is an empty R/W store initialized in a temporary location. *) + val migrate_block_action : + from_store -> dest_store -> Sc_rollup_block.t -> unit tzresult Lwt.t + + (** The final actions to be performed in the migration. In particular, this is + where data from the temporary store in [dest_store] in [tmp_dir] should be + reported in the actual [storage_dir]. *) + val final_actions : + storage_dir:string -> + tmp_dir:string -> + from_store -> + dest_store -> + unit tzresult Lwt.t +end + +module type S = sig + (** Migration function for the store located in [storage_dir]. *) + val migrate : storage_dir:string -> unit tzresult Lwt.t +end + +(** Functor to create and {e register} a migration. *) +module Make + (S_from : Store_sig.S) + (S_dest : Store_sig.S) + (Actions : MIGRATION_ACTIONS + with type from_store := Store_sigs.ro S_from.t + and type dest_store := Store_sigs.rw S_dest.t) : S + +(** Migrate store located in rollup node {e store} directory [storage_dir] if + needed. If there is no possible migration path registered to go from the + current version to the last {!Store.version}, this function resolves with an + error. *) +val maybe_run_migration : storage_dir:string -> unit tzresult Lwt.t diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/store_sig.ml b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_sig.ml new file mode 100644 index 000000000000..ffac28ff0409 --- /dev/null +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_sig.ml @@ -0,0 +1,65 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +module type S = sig + type +'a store + + (** Type of store. The parameter indicates if the store can be written or only + read. *) + type 'a t = ([< `Read | `Write > `Read] as 'a) store + + (** Read/write store {!t}. *) + type rw = Store_sigs.rw t + + (** Read only store {!t}. *) + type ro = Store_sigs.ro t + + (** Version supported by this code. *) + val version : Store_version.t + + (** [close store] closes the store. *) + val close : _ t -> unit tzresult Lwt.t + + (** [load mode ~l2_blocks_cache_size directory] loads a store from the data + persisted in [directory]. If [mode] is {!Store_sigs.Read_only}, then the + indexes and irmin store will be opened in readonly mode and only read + operations will be permitted. This allows to open a store for read access + that is already opened in {!Store_sigs.Read_write} mode in another + process. [l2_blocks_cache_size] is the number of L2 blocks the rollup node + will keep in memory. *) + val load : + 'a Store_sigs.mode -> + l2_blocks_cache_size:int -> + string -> + 'a store tzresult Lwt.t + + (** [readonly store] returns a read-only version of [store]. *) + val readonly : _ t -> ro + + (** [iter_l2_blocks store f] iterates [f] on all L2 blocks reachable from the + head, from newest to oldest. *) + val iter_l2_blocks : + _ t -> (Sc_rollup_block.t -> unit tzresult Lwt.t) -> unit tzresult Lwt.t +end diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/store_v0.ml b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_v0.ml new file mode 100644 index 000000000000..066a37f739fb --- /dev/null +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_v0.ml @@ -0,0 +1,506 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2021 Nomadic Labs, *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(** This module is a copy of + [src/proto_016_PtMumbai/lib_sc_rollup_node/store.ml], which contains the + store for the Mumbai rollup node. *) + +open Protocol +include Store_sigs +include Store_utils + +(** Aggregated collection of messages from the L1 inbox *) +open Alpha_context + +let version = Store_version.V0 + +module Irmin_store = struct + module IStore = Irmin_store.Make (struct + let name = "Tezos smart rollup node" + end) + + include IStore + include Store_utils.Make (IStore) +end + +module Empty_header = struct + type t = unit + + let name = "empty" + + let encoding = Data_encoding.unit + + let fixed_size = 0 +end + +module Add_empty_header = struct + module Header = Empty_header + + let header _ = () +end + +module Make_hash_index_key (H : Environment.S.HASH) = +Indexed_store.Make_index_key (struct + include Indexed_store.Make_fixed_encodable (H) + + let equal = H.equal +end) + +(** L2 blocks *) +module L2_blocks = + Indexed_store.Make_indexed_file + (struct + let name = "l2_blocks" + end) + (Tezos_store_shared.Block_key) + (struct + type t = (unit, unit) Sc_rollup_block.block + + let name = "sc_rollup_block_info" + + let encoding = + Sc_rollup_block.block_encoding Data_encoding.unit Data_encoding.unit + + module Header = struct + type t = Sc_rollup_block.header + + let name = "sc_rollup_block_header" + + let encoding = Sc_rollup_block.header_encoding + + let fixed_size = Sc_rollup_block.header_size + end + end) + +(** Unaggregated messages per block *) +module Messages = + Indexed_store.Make_indexed_file + (struct + let name = "messages" + end) + (Make_hash_index_key (Sc_rollup.Inbox_merkelized_payload_hashes.Hash)) + (struct + type t = Sc_rollup.Inbox_message.t list + + let name = "messages_list" + + let encoding = + Data_encoding.(list @@ dynamic_size Sc_rollup.Inbox_message.encoding) + + module Header = struct + type t = Block_hash.t * Timestamp.t * int + + let name = "messages_inbox_info" + + let encoding = + let open Data_encoding in + obj3 + (req "predecessor" Block_hash.encoding) + (req "predecessor_timestamp" Timestamp.encoding) + (req "num_messages" int31) + + let fixed_size = + WithExceptions.Option.get ~loc:__LOC__ + @@ Data_encoding.Binary.fixed_length encoding + end + end) + +(** Inbox state for each block *) +module Inboxes = + Indexed_store.Make_simple_indexed_file + (struct + let name = "inboxes" + end) + (Make_hash_index_key (Sc_rollup.Inbox.Hash)) + (struct + type t = Sc_rollup.Inbox.t + + let name = "inbox" + + let encoding = Sc_rollup.Inbox.encoding + + include Add_empty_header + end) + +module Commitments = + Indexed_store.Make_indexable + (struct + let name = "commitments" + end) + (Make_hash_index_key (Sc_rollup.Commitment.Hash)) + (Indexed_store.Make_index_value (Indexed_store.Make_fixed_encodable (struct + include Sc_rollup.Commitment + + let name = "commitment" + end))) + +module Commitments_published_at_level = struct + type element = { + first_published_at_level : Raw_level.t; + published_at_level : Raw_level.t option; + } + + let element_encoding = + let open Data_encoding in + let opt_level_encoding = + conv + (function None -> -1l | Some l -> Raw_level.to_int32 l) + (fun l -> if l = -1l then None else Some (Raw_level.of_int32_exn l)) + Data_encoding.int32 + in + conv + (fun {first_published_at_level; published_at_level} -> + (first_published_at_level, published_at_level)) + (fun (first_published_at_level, published_at_level) -> + {first_published_at_level; published_at_level}) + @@ obj2 + (req "first_published_at_level" Raw_level.encoding) + (req "published_at_level" opt_level_encoding) + + include + Indexed_store.Make_indexable + (struct + let name = "commitments" + end) + (Make_hash_index_key (Sc_rollup.Commitment.Hash)) + (Indexed_store.Make_index_value (Indexed_store.Make_fixed_encodable (struct + type t = element + + let name = "published_levels" + + let encoding = element_encoding + end))) +end + +module L2_head = Indexed_store.Make_singleton (struct + type t = Sc_rollup_block.t + + let name = "l2_head" + + let encoding = Sc_rollup_block.encoding +end) + +module Last_finalized_level = Indexed_store.Make_singleton (struct + type t = int32 + + let name = "finalized_level" + + let encoding = Data_encoding.int32 +end) + +(** Table from L1 levels to blocks hashes. *) +module Levels_to_hashes = + Indexed_store.Make_indexable + (struct + let name = "tezos_levels" + end) + (Indexed_store.Make_index_key (struct + type t = int32 + + let encoding = Data_encoding.int32 + + let name = "level" + + let fixed_size = 4 + + let equal = Int32.equal + end)) + (Tezos_store_shared.Block_key) + +(* Published slot headers per block hash, + stored as a list of bindings from `Dal_slot_index.t` + to `Dal.Slot.t`. The encoding function converts this + list into a `Dal.Slot_index.t`-indexed map. *) +module Dal_slot_pages = + Irmin_store.Make_nested_map + (struct + let path = ["dal"; "slot_pages"] + end) + (struct + type key = Block_hash.t + + let to_path_representation = Block_hash.to_b58check + end) + (struct + type key = Dal.Slot_index.t * Dal.Page.Index.t + + let encoding = + Data_encoding.(tup2 Dal.Slot_index.encoding Dal.Page.Index.encoding) + + let compare (i1, p1) (i2, p2) = + Compare.or_else (Dal.Slot_index.compare i1 i2) (fun () -> + Dal.Page.Index.compare p1 p2) + + let name = "slot_index" + end) + (struct + type value = Dal.Page.content + + let encoding = Dal.Page.content_encoding + + let name = "slot_pages" + end) + +(** stores slots whose data have been considered and pages stored to disk (if + they are confirmed). *) +module Dal_processed_slots = + Irmin_store.Make_nested_map + (struct + let path = ["dal"; "processed_slots"] + end) + (struct + type key = Block_hash.t + + let to_path_representation = Block_hash.to_b58check + end) + (struct + type key = Dal.Slot_index.t + + let encoding = Dal.Slot_index.encoding + + let compare = Dal.Slot_index.compare + + let name = "slot_index" + end) + (struct + type value = [`Confirmed | `Unconfirmed] + + let name = "slot_processing_status" + + let encoding = + let open Data_encoding in + let mk_case constr ~tag ~title = + case + ~title + (Tag tag) + (obj1 (req "kind" (constant title))) + (fun x -> if x = constr then Some () else None) + (fun () -> constr) + in + union + ~tag_size:`Uint8 + [ + mk_case `Confirmed ~tag:0 ~title:"Confirmed"; + mk_case `Unconfirmed ~tag:1 ~title:"Unconfirmed"; + ] + end) + +module Dal_slots_headers = + Irmin_store.Make_nested_map + (struct + let path = ["dal"; "slot_headers"] + end) + (struct + type key = Block_hash.t + + let to_path_representation = Block_hash.to_b58check + end) + (struct + type key = Dal.Slot_index.t + + let encoding = Dal.Slot_index.encoding + + let compare = Dal.Slot_index.compare + + let name = "slot_index" + end) + (struct + type value = Dal.Slot.Header.t + + let name = "slot_header" + + let encoding = Dal.Slot.Header.encoding + end) + +(* Published slot headers per block hash, stored as a list of bindings from + `Dal_slot_index.t` to `Dal.Slot.t`. The encoding function converts this + list into a `Dal.Slot_index.t`-indexed map. Note that the block_hash + refers to the block where slots headers have been confirmed, not + the block where they have been published. +*) + +(** Confirmed DAL slots history. See documentation of + {!Dal_slot_repr.Slots_history} for more details. *) +module Dal_confirmed_slots_history = + Irmin_store.Make_append_only_map + (struct + let path = ["dal"; "confirmed_slots_history"] + end) + (struct + type key = Block_hash.t + + let to_path_representation = Block_hash.to_b58check + end) + (struct + type value = Dal.Slots_history.t + + let name = "dal_slot_histories" + + let encoding = Dal.Slots_history.encoding + end) + +(** Confirmed DAL slots histories cache. See documentation of + {!Dal_slot_repr.Slots_history} for more details. *) +module Dal_confirmed_slots_histories = + (* TODO: https://gitlab.com/tezos/tezos/-/issues/4390 + Store single history points in map instead of whole history. *) + Irmin_store.Make_append_only_map + (struct + let path = ["dal"; "confirmed_slots_histories_cache"] + end) + (struct + type key = Block_hash.t + + let to_path_representation = Block_hash.to_b58check + end) + (struct + type value = Dal.Slots_history.History_cache.t + + let name = "dal_slot_history_cache" + + let encoding = Dal.Slots_history.History_cache.encoding + end) + +type 'a store = { + l2_blocks : 'a L2_blocks.t; + messages : 'a Messages.t; + inboxes : 'a Inboxes.t; + commitments : 'a Commitments.t; + commitments_published_at_level : 'a Commitments_published_at_level.t; + l2_head : 'a L2_head.t; + last_finalized_level : 'a Last_finalized_level.t; + levels_to_hashes : 'a Levels_to_hashes.t; + irmin_store : 'a Irmin_store.t; +} + +type 'a t = ([< `Read | `Write > `Read] as 'a) store + +type rw = Store_sigs.rw t + +type ro = Store_sigs.ro t + +let readonly + ({ + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head; + last_finalized_level; + levels_to_hashes; + irmin_store; + } : + _ t) : ro = + { + l2_blocks = L2_blocks.readonly l2_blocks; + messages = Messages.readonly messages; + inboxes = Inboxes.readonly inboxes; + commitments = Commitments.readonly commitments; + commitments_published_at_level = + Commitments_published_at_level.readonly commitments_published_at_level; + l2_head = L2_head.readonly l2_head; + last_finalized_level = Last_finalized_level.readonly last_finalized_level; + levels_to_hashes = Levels_to_hashes.readonly levels_to_hashes; + irmin_store = Irmin_store.readonly irmin_store; + } + +let close + ({ + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head = _; + last_finalized_level = _; + levels_to_hashes; + irmin_store; + } : + _ t) = + let open Lwt_result_syntax in + let+ () = L2_blocks.close l2_blocks + and+ () = Messages.close messages + and+ () = Inboxes.close inboxes + and+ () = Commitments.close commitments + and+ () = Commitments_published_at_level.close commitments_published_at_level + and+ () = Levels_to_hashes.close levels_to_hashes + and+ () = Irmin_store.close irmin_store in + () + +let load (type a) (mode : a mode) ~l2_blocks_cache_size data_dir : + a store tzresult Lwt.t = + let open Lwt_result_syntax in + let path name = Filename.concat data_dir name in + let cache_size = l2_blocks_cache_size in + let* l2_blocks = L2_blocks.load mode ~path:(path "l2_blocks") ~cache_size in + let* messages = Messages.load mode ~path:(path "messages") ~cache_size in + let* inboxes = Inboxes.load mode ~path:(path "inboxes") ~cache_size in + let* commitments = Commitments.load mode ~path:(path "commitments") in + let* commitments_published_at_level = + Commitments_published_at_level.load + mode + ~path:(path "commitments_published_at_level") + in + let* l2_head = L2_head.load mode ~path:(path "l2_head") in + let* last_finalized_level = + Last_finalized_level.load mode ~path:(path "last_finalized_level") + in + let* levels_to_hashes = + Levels_to_hashes.load mode ~path:(path "levels_to_hashes") + in + let+ irmin_store = Irmin_store.load mode (path "irmin_store") in + { + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head; + last_finalized_level; + levels_to_hashes; + irmin_store; + } + +let iter_l2_blocks ({l2_blocks; l2_head; _} : _ t) f = + let open Lwt_result_syntax in + let* head = L2_head.read l2_head in + match head with + | None -> + (* No reachable head, nothing to do *) + return_unit + | Some head -> + let rec loop hash = + let* block = L2_blocks.read l2_blocks hash in + match block with + | None -> + (* The block does not exist, the known chain stops here, so do we. *) + return_unit + | Some (block, header) -> + let* () = f {block with header} in + loop header.predecessor + in + loop head.header.block_hash diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/store.mli b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_v0.mli similarity index 85% rename from src/proto_016_PtMumbai/lib_sc_rollup_node/store.mli rename to src/proto_016_PtMumbai/lib_sc_rollup_node/store_v0.mli index 598836d8f1b0..f2d9a002d6c1 100644 --- a/src/proto_016_PtMumbai/lib_sc_rollup_node/store.mli +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_v0.mli @@ -2,6 +2,7 @@ (* *) (* Open Source License *) (* Copyright (c) 2022 Nomadic Labs, *) +(* Copyright (c) 2023 Functori, *) (* *) (* Permission is hereby granted, free of charge, to any person obtaining a *) (* copy of this software and associated documentation files (the "Software"),*) @@ -23,11 +24,25 @@ (* *) (*****************************************************************************) +(** This version of the store is used for the rollup nodes for protocol Mumbai, + i.e. = 16. + + This interface is a copy of + [src/proto_016_PtMumbai/lib_sc_rollup_node/store.mli], which contains the + layout for the Mumbai rollup node. +*) + open Protocol open Alpha_context open Indexed_store -module Irmin_store : Store_sigs.Store +module Irmin_store : sig + include module type of Irmin_store.Make (struct + let name = "Tezos smart rollup node" + end) + + include Store_sigs.Store with type 'a t := 'a t +end module L2_blocks : INDEXED_FILE @@ -100,7 +115,7 @@ module Dal_confirmed_slots_history : and type 'a store := 'a Irmin_store.t (** Confirmed DAL slots histories cache. See documentation of - {Dal_slot_repr.Slots_history} for more details. *) + {!Dal_slot_repr.Slots_history} for more details. *) module Dal_confirmed_slots_histories : Store_sigs.Append_only_map with type key := Block_hash.t @@ -146,31 +161,4 @@ type +'a store = { irmin_store : 'a Irmin_store.t; } -(** Type of store. The parameter indicates if the store can be written or only - read. *) -type 'a t = ([< `Read | `Write > `Read] as 'a) store - -(** Read/write store {!t}. *) -type rw = Store_sigs.rw t - -(** Read only store {!t}. *) -type ro = Store_sigs.ro t - -(** [close store] closes the store. *) -val close : _ t -> unit tzresult Lwt.t - -(** [load mode ~l2_blocks_cache_size directory] loads a store from the data - persisted in [directory]. If [mode] is {!Store_sigs.Read_only}, then the - indexes and irmin store will be opened in readonly mode and only read - operations will be permitted. This allows to open a store for read access - that is already opened in {!Store_sigs.Read_write} mode in another - process. [l2_blocks_cache_size] is the number of L2 blocks the rollup node - will keep in memory. *) -val load : - 'a Store_sigs.mode -> - l2_blocks_cache_size:int -> - string -> - 'a store tzresult Lwt.t - -(** [readonly store] returns a read-only version of [store]. *) -val readonly : _ t -> ro +include Store_sig.S with type 'a store := 'a store diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/store_v1.ml b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_v1.ml new file mode 100644 index 000000000000..adecbb8a2b5b --- /dev/null +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_v1.ml @@ -0,0 +1,253 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2021 Nomadic Labs, *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +open Protocol +open Alpha_context +include Store_sigs +include Store_utils +include Store_v0 + +let version = Store_version.V1 + +module Make_hash_index_key (H : Environment.S.HASH) = +Indexed_store.Make_index_key (struct + include Indexed_store.Make_fixed_encodable (H) + + let equal = H.equal +end) + +(** Unaggregated messages per block *) +module Messages = + Indexed_store.Make_indexed_file + (struct + let name = "messages" + end) + (Make_hash_index_key (Sc_rollup.Inbox_merkelized_payload_hashes.Hash)) + (struct + type t = Sc_rollup.Inbox_message.t list + + let name = "messages_list" + + let encoding = + Data_encoding.(list @@ dynamic_size Sc_rollup.Inbox_message.encoding) + + module Header = struct + type t = bool * Block_hash.t * Timestamp.t * int + + let name = "messages_inbox_info" + + let encoding = + let open Data_encoding in + obj4 + (req "is_first_block" bool) + (req "predecessor" Block_hash.encoding) + (req "predecessor_timestamp" Timestamp.encoding) + (req "num_messages" int31) + + let fixed_size = + WithExceptions.Option.get ~loc:__LOC__ + @@ Data_encoding.Binary.fixed_length encoding + end + end) + +module Dal_pages = struct + type removed_in_v1 +end + +module Dal_processed_slots = struct + type removed_in_v1 +end + +(** Store attestation statuses for DAL slots on L1. *) +module Dal_slots_statuses = + Irmin_store.Make_nested_map + (struct + let path = ["dal"; "slots_statuses"] + end) + (struct + type key = Block_hash.t + + let to_path_representation = Block_hash.to_b58check + end) + (struct + type key = Dal.Slot_index.t + + let encoding = Dal.Slot_index.encoding + + let compare = Dal.Slot_index.compare + + let name = "slot_index" + end) + (struct + (* TODO: https://gitlab.com/tezos/tezos/-/issues/4780. + + Rename Confirm-ed/ation to Attest-ed/ation in the rollup node. *) + type value = [`Confirmed | `Unconfirmed] + + let name = "slot_status" + + let encoding = + (* We don't use + Data_encoding.string_enum because a union is more storage-efficient. *) + let open Data_encoding in + union + ~tag_size:`Uint8 + [ + case + ~title:"Confirmed" + (Tag 0) + (obj1 (req "kind" (constant "Confirmed"))) + (function `Confirmed -> Some () | `Unconfirmed -> None) + (fun () -> `Confirmed); + case + ~title:"Unconfirmed" + (Tag 1) + (obj1 (req "kind" (constant "Unconfirmed"))) + (function `Unconfirmed -> Some () | `Confirmed -> None) + (fun () -> `Unconfirmed); + ] + end) + +type nonrec 'a store = { + l2_blocks : 'a L2_blocks.t; + messages : 'a Messages.t; + inboxes : 'a Inboxes.t; + commitments : 'a Commitments.t; + commitments_published_at_level : 'a Commitments_published_at_level.t; + l2_head : 'a L2_head.t; + last_finalized_level : 'a Last_finalized_level.t; + levels_to_hashes : 'a Levels_to_hashes.t; + irmin_store : 'a Irmin_store.t; +} + +type 'a t = ([< `Read | `Write > `Read] as 'a) store + +type rw = Store_sigs.rw t + +type ro = Store_sigs.ro t + +let readonly + ({ + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head; + last_finalized_level; + levels_to_hashes; + irmin_store; + } : + _ t) : ro = + { + l2_blocks = L2_blocks.readonly l2_blocks; + messages = Messages.readonly messages; + inboxes = Inboxes.readonly inboxes; + commitments = Commitments.readonly commitments; + commitments_published_at_level = + Commitments_published_at_level.readonly commitments_published_at_level; + l2_head = L2_head.readonly l2_head; + last_finalized_level = Last_finalized_level.readonly last_finalized_level; + levels_to_hashes = Levels_to_hashes.readonly levels_to_hashes; + irmin_store = Irmin_store.readonly irmin_store; + } + +let close + ({ + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head = _; + last_finalized_level = _; + levels_to_hashes; + irmin_store; + } : + _ t) = + let open Lwt_result_syntax in + let+ () = L2_blocks.close l2_blocks + and+ () = Messages.close messages + and+ () = Inboxes.close inboxes + and+ () = Commitments.close commitments + and+ () = Commitments_published_at_level.close commitments_published_at_level + and+ () = Levels_to_hashes.close levels_to_hashes + and+ () = Irmin_store.close irmin_store in + () + +let load (type a) (mode : a mode) ~l2_blocks_cache_size data_dir : + a store tzresult Lwt.t = + let open Lwt_result_syntax in + let path name = Filename.concat data_dir name in + let cache_size = l2_blocks_cache_size in + let* l2_blocks = L2_blocks.load mode ~path:(path "l2_blocks") ~cache_size in + let* messages = Messages.load mode ~path:(path "messages") ~cache_size in + let* inboxes = Inboxes.load mode ~path:(path "inboxes") ~cache_size in + let* commitments = Commitments.load mode ~path:(path "commitments") in + let* commitments_published_at_level = + Commitments_published_at_level.load + mode + ~path:(path "commitments_published_at_level") + in + let* l2_head = L2_head.load mode ~path:(path "l2_head") in + let* last_finalized_level = + Last_finalized_level.load mode ~path:(path "last_finalized_level") + in + let* levels_to_hashes = + Levels_to_hashes.load mode ~path:(path "levels_to_hashes") + in + let+ irmin_store = Irmin_store.load mode (path "irmin_store") in + { + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head; + last_finalized_level; + levels_to_hashes; + irmin_store; + } + +let iter_l2_blocks ({l2_blocks; l2_head; _} : _ t) f = + let open Lwt_result_syntax in + let* head = L2_head.read l2_head in + match head with + | None -> + (* No reachable head, nothing to do *) + return_unit + | Some head -> + let rec loop hash = + let* block = L2_blocks.read l2_blocks hash in + match block with + | None -> + (* The block does not exist, the known chain stops here, so do we. *) + return_unit + | Some (block, header) -> + let* () = f {block with header} in + loop header.predecessor + in + loop head.header.block_hash diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/store_v1.mli b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_v1.mli new file mode 100644 index 000000000000..766290712100 --- /dev/null +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_v1.mli @@ -0,0 +1,79 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2022 Nomadic Labs, *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(** This version of the store is used for the rollup nodes for protocols for and + after Nairobi, i.e. >= 17. *) + +open Protocol +open Alpha_context +open Indexed_store + +include module type of struct + include Store_v0 +end + +(** Storage for persisting messages downloaded from the L1 node. *) +module Messages : + INDEXED_FILE + with type key := Sc_rollup.Inbox_merkelized_payload_hashes.Hash.t + and type value := Sc_rollup.Inbox_message.t list + and type header := bool * Block_hash.t * Timestamp.t * int + +module Dal_pages : sig + type removed_in_v1 +end + +module Dal_processed_slots : sig + type removed_in_v1 +end + +(** [Dal_slots_statuses] is a [Store_utils.Nested_map] used to store the + attestation status of DAL slots. The values of this storage module have type + `[`Confirmed | `Unconfirmed]`, depending on whether the content of the slot + has been attested on L1 or not. If an entry is not present for a + [(block_hash, slot_index)], this means that the corresponding block is not + processed yet. +*) +module Dal_slots_statuses : + Store_sigs.Nested_map + with type primary_key := Block_hash.t + and type secondary_key := Dal.Slot_index.t + and type value := [`Confirmed | `Unconfirmed] + and type 'a store := 'a Irmin_store.t + +type +'a store = { + l2_blocks : 'a L2_blocks.t; + messages : 'a Messages.t; + inboxes : 'a Inboxes.t; + commitments : 'a Commitments.t; + commitments_published_at_level : 'a Commitments_published_at_level.t; + l2_head : 'a L2_head.t; + last_finalized_level : 'a Last_finalized_level.t; + levels_to_hashes : 'a Levels_to_hashes.t; + irmin_store : 'a Irmin_store.t; +} + +include Store_sig.S with type 'a store := 'a store diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/store_version.ml b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_version.ml new file mode 100644 index 000000000000..ec294eb1a23f --- /dev/null +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_version.ml @@ -0,0 +1,29 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +type t = V0 | V1 + +let pp ppf v = + Format.pp_print_string ppf @@ match v with V0 -> "v0" | V1 -> "v1" diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/store_version.mli b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_version.mli new file mode 100644 index 000000000000..86b8485e19c1 --- /dev/null +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_version.mli @@ -0,0 +1,29 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +type t = V0 | V1 + +(** Pretty-printer for store versions *) +val pp : Format.formatter -> t -> unit -- GitLab From fb71a7f4acded79c06da197da006ad25d98f2486 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Mon, 22 May 2023 20:55:24 +0200 Subject: [PATCH 12/13] Doc: remove ocamldoc patch for removed file --- docs/odoc_protocols.patch | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/docs/odoc_protocols.patch b/docs/odoc_protocols.patch index 1e85ebd56f6d..0eaa8c6aa162 100644 --- a/docs/odoc_protocols.patch +++ b/docs/odoc_protocols.patch @@ -357,19 +357,6 @@ index ca8e5769b9..90a3c81f2f 100644 val set : 'a Context.t -> state -> 'a Context.t Lwt.t end end -diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/store.mli b/src/proto_016_PtMumbai/lib_sc_rollup_node/store.mli -index 598836d8f1..1fd2fb535f 100644 ---- a/src/proto_016_PtMumbai/lib_sc_rollup_node/store.mli -+++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/store.mli -@@ -100,7 +100,7 @@ module Dal_confirmed_slots_history : - and type 'a store := 'a Irmin_store.t - - (** Confirmed DAL slots histories cache. See documentation of -- {Dal_slot_repr.Slots_history} for more details. *) -+ {!Dal_slot_repr.Slots_history} for more details. *) - module Dal_confirmed_slots_histories : - Store_sigs.Append_only_map - with type key := Block_hash.t diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/context.mli b/src/proto_017_PtNairob/lib_sc_rollup_node/context.mli index 636bb700d1..1a54302fd8 100644 --- a/src/proto_017_PtNairob/lib_sc_rollup_node/context.mli @@ -466,16 +453,3 @@ index ca8e5769b9..90a3c81f2f 100644 val set : 'a Context.t -> state -> 'a Context.t Lwt.t end end -diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/store.mli b/src/proto_017_PtNairob/lib_sc_rollup_node/store.mli -index 2bf1bd361a..b43b800956 100644 ---- a/src/proto_017_PtNairob/lib_sc_rollup_node/store.mli -+++ b/src/proto_017_PtNairob/lib_sc_rollup_node/store.mli -@@ -100,7 +100,7 @@ module Dal_confirmed_slots_history : - and type 'a store := 'a Irmin_store.t - - (** Confirmed DAL slots histories cache. See documentation of -- {Dal_slot_repr.Slots_history} for more details. *) -+ {!Dal_slot_repr.Slots_history} for more details. *) - module Dal_confirmed_slots_histories : - Store_sigs.Append_only_map - with type key := Block_hash.t -- GitLab From eda353dc0cbb214e46cd395933b774812e15b13e Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Tue, 23 May 2023 15:06:43 +0200 Subject: [PATCH 13/13] SCORU/Node/Mumbai: Mumbai rollup node uses store v1 --- .../lib_sc_rollup_node/node_context.ml | 19 ++++++++++++------- .../lib_sc_rollup_node/store.ml | 2 +- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/node_context.ml b/src/proto_016_PtMumbai/lib_sc_rollup_node/node_context.ml index 88ea8779abdd..576c7b9c182a 100644 --- a/src/proto_016_PtMumbai/lib_sc_rollup_node/node_context.ml +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/node_context.ml @@ -813,14 +813,15 @@ let get_messages {store; _} messages_hash = "Could not retrieve messages with payloads merkelized hash %a" Sc_rollup.Inbox_merkelized_payload_hashes.Hash.pp messages_hash - | Some (messages, (predecessor, predecessor_timestamp, _num_messages)) -> + | Some (messages, (_first, predecessor, predecessor_timestamp, _num_messages)) + -> return {predecessor; predecessor_timestamp; messages} let find_messages {store; _} hash = let open Lwt_result_syntax in let+ msgs = Store.Messages.read store.messages hash in Option.map - (fun (messages, (predecessor, predecessor_timestamp, _num_messages)) -> + (fun (messages, (_first, predecessor, predecessor_timestamp, _num_messages)) -> {predecessor; predecessor_timestamp; messages}) msgs @@ -833,7 +834,7 @@ let get_num_messages {store; _} hash = "Could not retrieve number of messages for inbox witness %a" Sc_rollup.Inbox_merkelized_payload_hashes.Hash.pp hash - | Some (_predecessor, _predecessor_timestamp, num_messages) -> + | Some (_first, _predecessor, _predecessor_timestamp, num_messages) -> return num_messages let save_messages {store; _} key {predecessor; predecessor_timestamp; messages} @@ -841,7 +842,11 @@ let save_messages {store; _} key {predecessor; predecessor_timestamp; messages} Store.Messages.append store.messages ~key - ~header:(predecessor, predecessor_timestamp, List.length messages) + ~header: + ( false (* Never first block of protocol for Mumbai *), + predecessor, + predecessor_timestamp, + List.length messages ) ~value:messages let get_full_l2_block node_ctxt block_hash = @@ -885,7 +890,7 @@ let save_slot_header {store; _} ~published_in_block_hash slot_header let processed_slot {store; _} ~confirmed_in_block_hash slot_index = - Store.Dal_processed_slots.find + Store.Dal_slots_statuses.find store.irmin_store ~primary_key:confirmed_in_block_hash ~secondary_key:slot_index @@ -903,7 +908,7 @@ let find_slot_page {store; _} ~confirmed_in_block_hash ~slot_index ~page_index = let save_unconfirmed_slot {store; _} current_block_hash slot_index = (* No page is actually saved *) - Store.Dal_processed_slots.add + Store.Dal_slots_statuses.add store.irmin_store ~primary_key:current_block_hash ~secondary_key:slot_index @@ -923,7 +928,7 @@ let save_confirmed_slot {store; _} current_block_hash slot_index pages = page) pages in - Store.Dal_processed_slots.add + Store.Dal_slots_statuses.add store.irmin_store ~primary_key:current_block_hash ~secondary_key:slot_index diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/store.ml b/src/proto_016_PtMumbai/lib_sc_rollup_node/store.ml index b45c19513a98..90d354f79c80 100644 --- a/src/proto_016_PtMumbai/lib_sc_rollup_node/store.ml +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/store.ml @@ -24,4 +24,4 @@ (* *) (*****************************************************************************) -include Store_v0 +include Store_v1 -- GitLab