diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/daemon.ml b/src/proto_016_PtMumbai/lib_sc_rollup_node/daemon.ml index d16e6a2a0166e8c6894327ec559683b695778c46..c4c572178cc238b8ff42f53d4957e70e90eb3763 100644 --- a/src/proto_016_PtMumbai/lib_sc_rollup_node/daemon.ml +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/daemon.ml @@ -604,7 +604,7 @@ module Make (PVM : Pvm.S) = struct Inbox.Internal_for_tests.process_messages node_ctxt ~predecessor - ~level:head.level + head messages in let* ctxt, _num_messages, num_ticks, initial_tick = diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/inbox.ml b/src/proto_016_PtMumbai/lib_sc_rollup_node/inbox.ml index 4b728a6a8a96ff3f4630bc8a6901f9a8158d631b..280be857426522d2c78394a0b0df348413d8a4b2 100644 --- a/src/proto_016_PtMumbai/lib_sc_rollup_node/inbox.ml +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/inbox.ml @@ -117,8 +117,9 @@ let add_messages ~predecessor_timestamp ~predecessor inbox messages = messages_with_protocol_internal_messages ) let process_messages (node_ctxt : _ Node_context.t) - ~(predecessor : Layer1.header) ~level messages = + ~(predecessor : Layer1.header) (head : Layer1.header) messages = let open Lwt_result_syntax in + let level = head.level in let* inbox = Node_context.inbox_of_head node_ctxt (Layer1.head_of_header predecessor) in @@ -144,7 +145,8 @@ let process_messages (node_ctxt : _ Node_context.t) Node_context.save_messages node_ctxt witness_hash - {predecessor = predecessor.hash; predecessor_timestamp; messages} + ~block_hash:head.hash + messages in let* inbox_hash = Node_context.save_inbox node_ctxt inbox in return @@ -171,11 +173,7 @@ let process_head (node_ctxt : _ Node_context.t) ~(predecessor : Layer1.header) inbox, _witness_hash, _messages_with_protocol_internal_messages ) as res) = - process_messages - node_ctxt - ~predecessor - ~level:head.level - collected_messages + process_messages node_ctxt ~predecessor head collected_messages in let* () = same_inbox_as_layer_1 node_ctxt head.hash inbox in return res diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/inbox.mli b/src/proto_016_PtMumbai/lib_sc_rollup_node/inbox.mli index 9f5fd133ee79e64d61df8ba5fe660bbc7448435d..20af65da0c7c16459c9365934bccdf693add49c6 100644 --- a/src/proto_016_PtMumbai/lib_sc_rollup_node/inbox.mli +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/inbox.mli @@ -84,7 +84,7 @@ module Internal_for_tests : sig val process_messages : Node_context.rw -> predecessor:Layer1.header -> - level:int32 -> + Layer1.header -> Sc_rollup.Inbox_message.t list -> (Sc_rollup.Inbox.Hash.t * Sc_rollup.Inbox.t diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/node_context.ml b/src/proto_016_PtMumbai/lib_sc_rollup_node/node_context.ml index 9d563d818b69613946a29ae82040c534b35fcf34..aea5468a2927902a84ca965d52d03b1e96e4861c 100644 --- a/src/proto_016_PtMumbai/lib_sc_rollup_node/node_context.ml +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/node_context.ml @@ -555,10 +555,13 @@ let get_predecessor node_ctxt (hash, level) = (* [head] is not already known in the L2 chain *) Layer1.get_predecessor node_ctxt.l1_ctxt (hash, level) -let header_of_head node_ctxt Layer1.{hash; level} = +let header_of_hash node_ctxt hash = let open Lwt_result_syntax in let+ header = Layer1.fetch_tezos_shell_header node_ctxt.cctxt hash in - {Layer1.hash; level; header} + {Layer1.hash; level = header.level; header} + +let header_of_head node_ctxt Layer1.{hash; level = _} = + header_of_hash node_ctxt hash let get_tezos_reorg_for_new_head node_ctxt old_head new_head = let open Lwt_result_syntax in @@ -674,17 +677,19 @@ let block_with_tick ({store; _} as node_ctxt) ~max_level tick = let get_commitment {store; _} commitment_hash = let open Lwt_result_syntax in - let* commitment = Store.Commitments.find store.commitments commitment_hash in + let* commitment = Store.Commitments.read store.commitments commitment_hash in match commitment with | None -> failwith "Could not retrieve commitment %a" Sc_rollup.Commitment.Hash.pp commitment_hash - | Some c -> return c + | Some (c, ()) -> return c let find_commitment {store; _} hash = - Store.Commitments.find store.commitments hash + let open Lwt_result_syntax in + let+ commitment = Store.Commitments.read store.commitments hash in + Option.map fst commitment let commitment_exists {store; _} hash = Store.Commitments.mem store.commitments hash @@ -692,7 +697,9 @@ let commitment_exists {store; _} hash = let save_commitment {store; _} commitment = let open Lwt_result_syntax in let hash = Sc_rollup.Commitment.hash_uncarbonated commitment in - let+ () = Store.Commitments.add store.commitments hash commitment in + let+ () = + Store.Commitments.append store.commitments ~key:hash ~value:commitment + in hash let commitment_published_at_level {store; _} commitment = @@ -793,56 +800,64 @@ type messages_info = { messages : Sc_rollup.Inbox_message.t list; } -let get_messages {store; _} messages_hash = +let find_messages node_ctxt messages_hash = let open Lwt_result_syntax in - let* msg = Store.Messages.read store.messages messages_hash in + let* msg = Store.Messages.read node_ctxt.store.messages messages_hash in match msg with + | None -> return_none + | Some (messages, block_hash) -> + let* header = header_of_hash node_ctxt block_hash in + let* pred_header = header_of_hash node_ctxt header.header.predecessor in + return_some + { + predecessor = pred_header.hash; + predecessor_timestamp = pred_header.header.timestamp; + messages; + } + +let get_messages_aux find node_ctxt hash = + let open Lwt_result_syntax in + let* res = find node_ctxt hash in + match res with | None -> failwith "Could not retrieve messages with payloads merkelized hash %a" Sc_rollup.Inbox_merkelized_payload_hashes.Hash.pp - messages_hash - | Some (messages, (_first, predecessor, predecessor_timestamp, _num_messages)) - -> - return {predecessor; predecessor_timestamp; messages} + hash + | Some res -> return res -let find_messages {store; _} hash = - let open Lwt_result_syntax in - let+ msgs = Store.Messages.read store.messages hash in - Option.map - (fun (messages, (_first, predecessor, predecessor_timestamp, _num_messages)) -> - {predecessor; predecessor_timestamp; messages}) - msgs +let get_messages node_ctxt = get_messages_aux find_messages node_ctxt + +let get_messages_without_proto_messages node_ctxt = + get_messages_aux + (fun node_ctxt messages_hash -> + let open Lwt_result_syntax in + let* msg = Store.Messages.read node_ctxt.store.messages messages_hash in + match msg with + | None -> return_none + | Some (messages, _block_hash) -> return_some messages) + node_ctxt let get_num_messages {store; _} hash = let open Lwt_result_syntax in - let* header = Store.Messages.header store.messages hash in - match header with + let* msg = Store.Messages.read store.messages hash in + match msg with | None -> failwith "Could not retrieve number of messages for inbox witness %a" Sc_rollup.Inbox_merkelized_payload_hashes.Hash.pp hash - | Some (_first, _predecessor, _predecessor_timestamp, num_messages) -> - return num_messages - -let save_messages {store; _} key {predecessor; predecessor_timestamp; messages} - = - Store.Messages.append - store.messages - ~key - ~header: - ( false (* Never first block of protocol for Mumbai *), - predecessor, - predecessor_timestamp, - List.length messages ) - ~value:messages + | Some (messages, _block_hash) -> return (List.length messages) + +let save_messages {store; _} key ~block_hash messages = + Store.Messages.append store.messages ~key ~header:block_hash ~value:messages let get_full_l2_block node_ctxt block_hash = let open Lwt_result_syntax in let* block = get_l2_block node_ctxt block_hash in let* inbox = get_inbox node_ctxt block.header.inbox_hash - and* {messages; _} = get_messages node_ctxt block.header.inbox_witness + and* messages = + get_messages_without_proto_messages node_ctxt block.header.inbox_witness and* commitment = Option.map_es (get_commitment node_ctxt) block.header.commitment_hash in diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/node_context.mli b/src/proto_016_PtMumbai/lib_sc_rollup_node/node_context.mli index 411710c5b7c68c0face04e97b2636c62a2a4347a..335d162f2c864031a106d1e27477d8e84b49871d 100644 --- a/src/proto_016_PtMumbai/lib_sc_rollup_node/node_context.mli +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/node_context.mli @@ -354,19 +354,19 @@ val find_messages : Sc_rollup.Inbox_merkelized_payload_hashes.Hash.t -> messages_info option tzresult Lwt.t -(** [get_num_messages t witness_hash] retrieves (without reading all the messages - from disk) the number of messages for the inbox witness [witness_hash] - stored by the rollup node. *) +(** [get_num_messages t witness_hash] retrieves the number of messages for the + inbox witness [witness_hash] stored by the rollup node. *) val get_num_messages : _ t -> Sc_rollup.Inbox_merkelized_payload_hashes.Hash.t -> int tzresult Lwt.t -(** [save_messages t payloads_hash messages] associates the list of [messages] - to the [payloads_hash]. The payload hash must be computed by calling, - e.g. {!Sc_rollup.Inbox.add_all_messages}. *) +(** [save_messages t payloads_hash ~block_hash messages] associates the list of + [messages] to the [payloads_hash]. The payload hash must be computed by + calling, e.g. {!Sc_rollup.Inbox.add_all_messages}. *) val save_messages : rw -> Sc_rollup.Inbox_merkelized_payload_hashes.Hash.t -> - messages_info -> + block_hash:Block_hash.t -> + Sc_rollup.Inbox_message.t list -> unit tzresult Lwt.t (** {3 DAL} *) diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/store.ml b/src/proto_016_PtMumbai/lib_sc_rollup_node/store.ml index 90d354f79c800bfd9c773751bf1b6cd5f9b4392a..f89bd6e8cfb551522c566991a8cede8bb4e94e69 100644 --- a/src/proto_016_PtMumbai/lib_sc_rollup_node/store.ml +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/store.ml @@ -24,4 +24,4 @@ (* *) (*****************************************************************************) -include Store_v1 +include Store_v2 diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/store_migration.ml b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_migration.ml index 2c3fd7901eca6c2b8475346f51737f314f14ecf7..3805963a8cac526f8018a2e8066a367fb1e22169 100644 --- a/src/proto_016_PtMumbai/lib_sc_rollup_node/store_migration.ml +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_migration.ml @@ -224,7 +224,7 @@ let maybe_run_migration ~storage_dir = in Format.printf "Store migration completed@.") -module V0_to_V1 = struct +module V1_migrations = struct let convert_store_messages (messages, (block_hash, timestamp, number_of_messages)) = ( messages, @@ -293,7 +293,7 @@ module V0_to_V1 = struct let*! () = migrate_dal_processed_slots_irmin v1_store in return_unit - include + module From_v0 = Make (Store_v0) (Store_v1) (struct let migrate_block_action = migrate_messages @@ -301,3 +301,138 @@ module V0_to_V1 = struct let final_actions = final_actions end) end + +module V2_migrations = struct + let messages_store_location ~storage_dir = + let open Filename.Infix in + storage_dir // "messages" + + let commitments_store_location ~storage_dir = + let open Filename.Infix in + storage_dir // "commitments" + + let inboxes_store_location ~storage_dir = + let open Filename.Infix in + storage_dir // "inboxes" + + let migrate_messages read_messages (v2_store : _ Store_v2.t) + (l2_block : Sc_rollup_block.t) = + let open Lwt_result_syntax in + let* v1_messages = read_messages l2_block.header.inbox_witness in + match v1_messages with + | None -> return_unit + | Some (messages, _v1_header) -> + let header = l2_block.header.block_hash in + Store_v2.Messages.append + v2_store.messages + ~key:l2_block.header.inbox_witness + ~header + ~value:messages + + let migrate_commitment read_commitment (v2_store : _ Store_v2.t) + (l2_block : Sc_rollup_block.t) = + let open Lwt_result_syntax in + match l2_block.header.commitment_hash with + | None -> return_unit + | Some commitment_hash -> ( + let* v1_commitment = read_commitment commitment_hash in + match v1_commitment with + | None -> return_unit + | Some commitment -> + Store_v2.Commitments.append + v2_store.commitments + ~key:commitment_hash + ~value:commitment) + + let migrate_inbox read_inbox (v2_store : _ Store_v2.t) + (l2_block : Sc_rollup_block.t) = + let open Lwt_result_syntax in + let* v1_inbox = read_inbox l2_block.header.inbox_hash in + match v1_inbox with + | None -> return_unit + | Some (inbox, ()) -> + Store_v2.Inboxes.append + v2_store.inboxes + ~key:l2_block.header.inbox_hash + ~value:inbox + + let final_actions ~storage_dir ~tmp_dir _ _ = + let open Lwt_result_syntax in + let*! () = + Lwt_utils_unix.remove_dir (messages_store_location ~storage_dir) + in + let*! () = + Lwt_utils_unix.remove_dir (commitments_store_location ~storage_dir) + in + let*! () = + Lwt_utils_unix.remove_dir (inboxes_store_location ~storage_dir) + in + let*! () = + Lwt_unix.rename + (messages_store_location ~storage_dir:tmp_dir) + (messages_store_location ~storage_dir) + in + let*! () = + Lwt_unix.rename + (commitments_store_location ~storage_dir:tmp_dir) + (commitments_store_location ~storage_dir) + in + let*! () = + Lwt_unix.rename + (inboxes_store_location ~storage_dir:tmp_dir) + (inboxes_store_location ~storage_dir) + in + return_unit + + module From_v1 = + Make (Store_v1) (Store_v2) + (struct + let migrate_block_action v1_store v2_store l2_block = + let open Lwt_result_syntax in + let* () = + migrate_messages + Store_v1.(Messages.read v1_store.messages) + v2_store + l2_block + and* () = + migrate_commitment + Store_v1.(Commitments.find v1_store.commitments) + v2_store + l2_block + and* () = + migrate_inbox + Store_v1.(Inboxes.read v1_store.inboxes) + v2_store + l2_block + in + return_unit + + let final_actions = final_actions + end) + + module From_v0 = + Make (Store_v0) (Store_v2) + (struct + let migrate_block_action v0_store v2_store l2_block = + let open Lwt_result_syntax in + let* () = + migrate_messages + Store_v0.(Messages.read v0_store.messages) + v2_store + l2_block + and* () = + migrate_commitment + Store_v0.(Commitments.find v0_store.commitments) + v2_store + l2_block + and* () = + migrate_inbox + Store_v0.(Inboxes.read v0_store.inboxes) + v2_store + l2_block + in + return_unit + + let final_actions = final_actions + end) +end diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/store_v2.ml b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_v2.ml new file mode 100644 index 0000000000000000000000000000000000000000..c8363e787effc9c091ed4925dd7a36edc0c282b4 --- /dev/null +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_v2.ml @@ -0,0 +1,269 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2021 Nomadic Labs, *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +open Protocol +open Alpha_context +include Store_sigs +include Store_utils +include Store_v1 + +let version = Store_version.V2 + +module Make_hash_index_key (H : Environment.S.HASH) = +Indexed_store.Make_index_key (struct + include Indexed_store.Make_fixed_encodable (H) + + let equal = H.equal +end) + +(** Unaggregated messages per block *) +module Messages = + Indexed_store.Make_indexed_file + (struct + let name = "messages" + end) + (Make_hash_index_key (Sc_rollup.Inbox_merkelized_payload_hashes.Hash)) + (struct + type t = Sc_rollup.Inbox_message.t list + + let name = "messages_list" + + let encoding = + Data_encoding.(list @@ dynamic_size Sc_rollup.Inbox_message.encoding) + + module Header = struct + type t = Block_hash.t + + let name = "messages_block" + + let encoding = Block_hash.encoding + + let fixed_size = + WithExceptions.Option.get ~loc:__LOC__ + @@ Data_encoding.Binary.fixed_length encoding + end + end) + +module Empty_header = struct + type t = unit + + let name = "empty" + + let encoding = Data_encoding.unit + + let fixed_size = 0 +end + +module Add_empty_header = struct + module Header = Empty_header + + let header _ = () +end + +(** Versioned inboxes *) +module Inboxes = + Indexed_store.Make_simple_indexed_file + (struct + let name = "inboxes" + end) + (Make_hash_index_key (Sc_rollup.Inbox.Hash)) + (struct + type t = Sc_rollup.Inbox.t + + let to_repr inbox = + inbox + |> Data_encoding.Binary.to_string_exn Sc_rollup.Inbox.encoding + |> Data_encoding.Binary.of_string_exn Sc_rollup_inbox_repr.encoding + + let of_repr inbox = + inbox + |> Data_encoding.Binary.to_string_exn Sc_rollup_inbox_repr.encoding + |> Data_encoding.Binary.of_string_exn Sc_rollup.Inbox.encoding + + let encoding = + Data_encoding.conv + (fun x -> to_repr x |> Sc_rollup_inbox_repr.to_versioned) + (fun x -> Sc_rollup_inbox_repr.of_versioned x |> of_repr) + Sc_rollup_inbox_repr.versioned_encoding + + let name = "inbox" + + include Add_empty_header + end) + +(** Versioned commitments *) +module Commitments = + Indexed_store.Make_simple_indexed_file + (struct + let name = "commitments" + end) + (Make_hash_index_key (Sc_rollup.Commitment.Hash)) + (struct + type t = Sc_rollup.Commitment.t + + let to_repr commitment = + commitment + |> Data_encoding.Binary.to_string_exn Sc_rollup.Commitment.encoding + |> Data_encoding.Binary.of_string_exn Sc_rollup_commitment_repr.encoding + + let of_repr commitment = + commitment + |> Data_encoding.Binary.to_string_exn Sc_rollup_commitment_repr.encoding + |> Data_encoding.Binary.of_string_exn Sc_rollup.Commitment.encoding + + let encoding = + Data_encoding.conv + (fun x -> to_repr x |> Sc_rollup_commitment_repr.to_versioned) + (fun x -> Sc_rollup_commitment_repr.of_versioned x |> of_repr) + Sc_rollup_commitment_repr.versioned_encoding + + let name = "commitment" + + include Add_empty_header + end) + +type nonrec 'a store = { + l2_blocks : 'a L2_blocks.t; + messages : 'a Messages.t; + inboxes : 'a Inboxes.t; + commitments : 'a Commitments.t; + commitments_published_at_level : 'a Commitments_published_at_level.t; + l2_head : 'a L2_head.t; + last_finalized_level : 'a Last_finalized_level.t; + levels_to_hashes : 'a Levels_to_hashes.t; + irmin_store : 'a Irmin_store.t; +} + +type 'a t = ([< `Read | `Write > `Read] as 'a) store + +type rw = Store_sigs.rw t + +type ro = Store_sigs.ro t + +let readonly + ({ + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head; + last_finalized_level; + levels_to_hashes; + irmin_store; + } : + _ t) : ro = + { + l2_blocks = L2_blocks.readonly l2_blocks; + messages = Messages.readonly messages; + inboxes = Inboxes.readonly inboxes; + commitments = Commitments.readonly commitments; + commitments_published_at_level = + Commitments_published_at_level.readonly commitments_published_at_level; + l2_head = L2_head.readonly l2_head; + last_finalized_level = Last_finalized_level.readonly last_finalized_level; + levels_to_hashes = Levels_to_hashes.readonly levels_to_hashes; + irmin_store = Irmin_store.readonly irmin_store; + } + +let close + ({ + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head = _; + last_finalized_level = _; + levels_to_hashes; + irmin_store; + } : + _ t) = + let open Lwt_result_syntax in + let+ () = L2_blocks.close l2_blocks + and+ () = Messages.close messages + and+ () = Inboxes.close inboxes + and+ () = Commitments.close commitments + and+ () = Commitments_published_at_level.close commitments_published_at_level + and+ () = Levels_to_hashes.close levels_to_hashes + and+ () = Irmin_store.close irmin_store in + () + +let load (type a) (mode : a mode) ~l2_blocks_cache_size data_dir : + a store tzresult Lwt.t = + let open Lwt_result_syntax in + let path name = Filename.concat data_dir name in + let cache_size = l2_blocks_cache_size in + let* l2_blocks = L2_blocks.load mode ~path:(path "l2_blocks") ~cache_size in + let* messages = Messages.load mode ~path:(path "messages") ~cache_size in + let* inboxes = Inboxes.load mode ~path:(path "inboxes") ~cache_size in + let* commitments = + Commitments.load mode ~path:(path "commitments") ~cache_size + in + let* commitments_published_at_level = + Commitments_published_at_level.load + mode + ~path:(path "commitments_published_at_level") + in + let* l2_head = L2_head.load mode ~path:(path "l2_head") in + let* last_finalized_level = + Last_finalized_level.load mode ~path:(path "last_finalized_level") + in + let* levels_to_hashes = + Levels_to_hashes.load mode ~path:(path "levels_to_hashes") + in + let+ irmin_store = Irmin_store.load mode (path "irmin_store") in + { + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head; + last_finalized_level; + levels_to_hashes; + irmin_store; + } + +let iter_l2_blocks ({l2_blocks; l2_head; _} : _ t) f = + let open Lwt_result_syntax in + let* head = L2_head.read l2_head in + match head with + | None -> + (* No reachable head, nothing to do *) + return_unit + | Some head -> + let rec loop hash = + let* block = L2_blocks.read l2_blocks hash in + match block with + | None -> + (* The block does not exist, the known chain stops here, so do we. *) + return_unit + | Some (block, header) -> + let* () = f {block with header} in + loop header.predecessor + in + loop head.header.block_hash diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/store_v2.mli b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_v2.mli new file mode 100644 index 0000000000000000000000000000000000000000..dcb851d224f3f0e29439276667ee4f00a82f959b --- /dev/null +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_v2.mli @@ -0,0 +1,72 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2022 Nomadic Labs, *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(** This version of the store is used for the rollup nodes for protocols for and + after Nairobi, i.e. >= 17. *) + +open Protocol +open Alpha_context +open Indexed_store + +include module type of struct + include Store_v1 +end + +(** Storage for persisting messages downloaded from the L1 node. *) +module Messages : + INDEXED_FILE + with type key := Sc_rollup.Inbox_merkelized_payload_hashes.Hash.t + and type value := Sc_rollup.Inbox_message.t list + and type header := Block_hash.t + +(** Storage for persisting inboxes. *) +module Inboxes : + SIMPLE_INDEXED_FILE + with type key := Sc_rollup.Inbox.Hash.t + and type value := Sc_rollup.Inbox.t + and type header := unit + +(** Storage containing commitments and corresponding commitment hashes that the + rollup node has knowledge of. *) +module Commitments : + SIMPLE_INDEXED_FILE + with type key := Sc_rollup.Commitment.Hash.t + and type value := Sc_rollup.Commitment.t + and type header := unit + +type +'a store = { + l2_blocks : 'a L2_blocks.t; + messages : 'a Messages.t; + inboxes : 'a Inboxes.t; + commitments : 'a Commitments.t; + commitments_published_at_level : 'a Commitments_published_at_level.t; + l2_head : 'a L2_head.t; + last_finalized_level : 'a Last_finalized_level.t; + levels_to_hashes : 'a Levels_to_hashes.t; + irmin_store : 'a Irmin_store.t; +} + +include Store_sig.S with type 'a store := 'a store diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/store_version.ml b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_version.ml index cdb8252b3bae35556ae7b323e08b33118bb870e1..db71f628911302ba849e2d0fdec8895320b2db1b 100644 --- a/src/proto_016_PtMumbai/lib_sc_rollup_node/store_version.ml +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_version.ml @@ -23,18 +23,20 @@ (* *) (*****************************************************************************) -type t = V0 | V1 +type t = V0 | V1 | V2 let pp ppf v = - Format.pp_print_string ppf @@ match v with V0 -> "v0" | V1 -> "v1" + Format.pp_print_string ppf + @@ match v with V0 -> "v0" | V1 -> "v1" | V2 -> "v2" let encoding = let open Data_encoding in conv - (function V0 -> 0 | V1 -> 1) + (function V0 -> 0 | V1 -> 1 | V2 -> 2) (function | 0 -> V0 | 1 -> V1 + | 2 -> V2 | v -> Format.ksprintf Stdlib.failwith "Unsupported store version %d" v) (obj1 (req "store_version" int31)) diff --git a/src/proto_016_PtMumbai/lib_sc_rollup_node/store_version.mli b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_version.mli index 870c76d05ba28a4e2039bdfa9b4e1db939aaa694..1f2f8b4a26cc96d70357839ccee9be8f7a91acfb 100644 --- a/src/proto_016_PtMumbai/lib_sc_rollup_node/store_version.mli +++ b/src/proto_016_PtMumbai/lib_sc_rollup_node/store_version.mli @@ -23,7 +23,7 @@ (* *) (*****************************************************************************) -type t = V0 | V1 +type t = V0 | V1 | V2 (** Pretty-printer for store versions *) val pp : Format.formatter -> t -> unit diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/daemon.ml b/src/proto_017_PtNairob/lib_sc_rollup_node/daemon.ml index c19b0e3a49418d543b4bba2a0b635ca067b93cd3..849a8fd47f025175df6604f2e93c91033eed5349 100644 --- a/src/proto_017_PtNairob/lib_sc_rollup_node/daemon.ml +++ b/src/proto_017_PtNairob/lib_sc_rollup_node/daemon.ml @@ -599,7 +599,7 @@ module Make (PVM : Pvm.S) = struct node_ctxt ~is_first_block ~predecessor - ~level:head.level + head messages in let* ctxt, _num_messages, num_ticks, initial_tick = diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/inbox.ml b/src/proto_017_PtNairob/lib_sc_rollup_node/inbox.ml index 6c2b59e39ab67190c07ff37bb6ac7b98d1680930..ede23e940dfffd4c12662237cc59bc18f66fb4d9 100644 --- a/src/proto_017_PtNairob/lib_sc_rollup_node/inbox.ml +++ b/src/proto_017_PtNairob/lib_sc_rollup_node/inbox.ml @@ -119,8 +119,9 @@ let add_messages ~is_first_block ~predecessor_timestamp ~predecessor inbox messages_with_protocol_internal_messages ) let process_messages (node_ctxt : _ Node_context.t) ~is_first_block - ~(predecessor : Layer1.header) ~level messages = + ~(predecessor : Layer1.header) (head : Layer1.header) messages = let open Lwt_result_syntax in + let level = head.level in let* inbox = Node_context.inbox_of_head node_ctxt (Layer1.head_of_header predecessor) in @@ -147,12 +148,8 @@ let process_messages (node_ctxt : _ Node_context.t) ~is_first_block Node_context.save_messages node_ctxt witness_hash - { - is_first_block; - predecessor = predecessor.hash; - predecessor_timestamp; - messages; - } + ~block_hash:head.hash + messages in let* inbox_hash = Node_context.save_inbox node_ctxt inbox in return @@ -191,7 +188,7 @@ let process_head (node_ctxt : _ Node_context.t) ~(predecessor : Layer1.header) node_ctxt ~is_first_block ~predecessor - ~level:head.level + head collected_messages in let* () = same_inbox_as_layer_1 node_ctxt head.hash inbox in diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/inbox.mli b/src/proto_017_PtNairob/lib_sc_rollup_node/inbox.mli index 34d19b287ab30b4e9f54076a3e245194c007a9b6..24a413e58ae068083026ed903ab7c282ee7ed3ed 100644 --- a/src/proto_017_PtNairob/lib_sc_rollup_node/inbox.mli +++ b/src/proto_017_PtNairob/lib_sc_rollup_node/inbox.mli @@ -89,7 +89,7 @@ module Internal_for_tests : sig Node_context.rw -> is_first_block:bool -> predecessor:Layer1.header -> - level:int32 -> + Layer1.header -> Sc_rollup.Inbox_message.t list -> (Sc_rollup.Inbox.Hash.t * Sc_rollup.Inbox.t diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/node_context.ml b/src/proto_017_PtNairob/lib_sc_rollup_node/node_context.ml index 9739dc6477ad71a77e334e7da45d06844b0da148..9a63f68e36f9dba3928ee5840cabff5f0b893f5c 100644 --- a/src/proto_017_PtNairob/lib_sc_rollup_node/node_context.ml +++ b/src/proto_017_PtNairob/lib_sc_rollup_node/node_context.ml @@ -553,10 +553,13 @@ let get_predecessor node_ctxt (hash, level) = (* [head] is not already known in the L2 chain *) Layer1.get_predecessor node_ctxt.l1_ctxt (hash, level) -let header_of_head node_ctxt Layer1.{hash; level} = +let header_of_hash node_ctxt hash = let open Lwt_result_syntax in let+ header = Layer1.fetch_tezos_shell_header node_ctxt.cctxt hash in - {Layer1.hash; level; header} + {Layer1.hash; level = header.level; header} + +let header_of_head node_ctxt Layer1.{hash; level = _} = + header_of_hash node_ctxt hash let get_tezos_reorg_for_new_head node_ctxt old_head new_head = let open Lwt_result_syntax in @@ -672,17 +675,19 @@ let block_with_tick ({store; _} as node_ctxt) ~max_level tick = let get_commitment {store; _} commitment_hash = let open Lwt_result_syntax in - let* commitment = Store.Commitments.find store.commitments commitment_hash in + let* commitment = Store.Commitments.read store.commitments commitment_hash in match commitment with | None -> failwith "Could not retrieve commitment %a" Sc_rollup.Commitment.Hash.pp commitment_hash - | Some c -> return c + | Some (c, ()) -> return c let find_commitment {store; _} hash = - Store.Commitments.find store.commitments hash + let open Lwt_result_syntax in + let+ commitment = Store.Commitments.read store.commitments hash in + Option.map fst commitment let commitment_exists {store; _} hash = Store.Commitments.mem store.commitments hash @@ -690,7 +695,9 @@ let commitment_exists {store; _} hash = let save_commitment {store; _} commitment = let open Lwt_result_syntax in let hash = Sc_rollup.Commitment.hash_uncarbonated commitment in - let+ () = Store.Commitments.add store.commitments hash commitment in + let+ () = + Store.Commitments.append store.commitments ~key:hash ~value:commitment + in hash let commitment_published_at_level {store; _} commitment = @@ -792,55 +799,71 @@ type messages_info = { messages : Sc_rollup.Inbox_message.t list; } -let get_messages {store; _} messages_hash = +let find_messages node_ctxt messages_hash = let open Lwt_result_syntax in - let* msg = Store.Messages.read store.messages messages_hash in + let* msg = Store.Messages.read node_ctxt.store.messages messages_hash in match msg with + | None -> return_none + | Some (messages, block_hash) -> + let* header = header_of_hash node_ctxt block_hash in + let* pred_header = header_of_hash node_ctxt header.header.predecessor in + let* grand_parent_header = + header_of_hash node_ctxt pred_header.header.predecessor + in + let is_first_block = + pred_header.header.proto_level <> grand_parent_header.header.proto_level + in + return_some + { + is_first_block; + predecessor = pred_header.hash; + predecessor_timestamp = pred_header.header.timestamp; + messages; + } + +let get_messages_aux find node_ctxt hash = + let open Lwt_result_syntax in + let* res = find node_ctxt hash in + match res with | None -> failwith "Could not retrieve messages with payloads merkelized hash %a" Sc_rollup.Inbox_merkelized_payload_hashes.Hash.pp - messages_hash - | Some - ( messages, - (is_first_block, predecessor, predecessor_timestamp, _num_messages) ) -> - return {is_first_block; predecessor; predecessor_timestamp; messages} + hash + | Some res -> return res -let find_messages {store; _} hash = - let open Lwt_result_syntax in - let+ msgs = Store.Messages.read store.messages hash in - Option.map - (fun ( messages, - (is_first_block, predecessor, predecessor_timestamp, _num_messages) - ) -> {is_first_block; predecessor; predecessor_timestamp; messages}) - msgs +let get_messages node_ctxt = get_messages_aux find_messages node_ctxt + +let get_messages_without_proto_messages node_ctxt = + get_messages_aux + (fun node_ctxt messages_hash -> + let open Lwt_result_syntax in + let* msg = Store.Messages.read node_ctxt.store.messages messages_hash in + match msg with + | None -> return_none + | Some (messages, _block_hash) -> return_some messages) + node_ctxt let get_num_messages {store; _} hash = let open Lwt_result_syntax in - let* header = Store.Messages.header store.messages hash in - match header with + let* msg = Store.Messages.read store.messages hash in + match msg with | None -> failwith "Could not retrieve number of messages for inbox witness %a" Sc_rollup.Inbox_merkelized_payload_hashes.Hash.pp hash - | Some (_first_block, _predecessor, _predecessor_timestamp, num_messages) -> - return num_messages - -let save_messages {store; _} key - {is_first_block; predecessor; predecessor_timestamp; messages} = - Store.Messages.append - store.messages - ~key - ~header: - (is_first_block, predecessor, predecessor_timestamp, List.length messages) - ~value:messages + | Some (messages, _block_hash) -> return (List.length messages) + +let save_messages {store; _} key ~block_hash messages = + Store.Messages.append store.messages ~key ~header:block_hash ~value:messages let get_full_l2_block node_ctxt block_hash = let open Lwt_result_syntax in let* block = get_l2_block node_ctxt block_hash in let* inbox = get_inbox node_ctxt block.header.inbox_hash - and* {messages; _} = get_messages node_ctxt block.header.inbox_witness + and* messages = + get_messages_without_proto_messages node_ctxt block.header.inbox_witness and* commitment = Option.map_es (get_commitment node_ctxt) block.header.commitment_hash in diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/node_context.mli b/src/proto_017_PtNairob/lib_sc_rollup_node/node_context.mli index 6bf49eabb421df050c441678a2722eda4786cc06..84c0970b2534b43b01dfa3bc80b352599d5c51a6 100644 --- a/src/proto_017_PtNairob/lib_sc_rollup_node/node_context.mli +++ b/src/proto_017_PtNairob/lib_sc_rollup_node/node_context.mli @@ -353,19 +353,19 @@ val find_messages : Sc_rollup.Inbox_merkelized_payload_hashes.Hash.t -> messages_info option tzresult Lwt.t -(** [get_num_messages t witness_hash] retrieves (without reading all the messages - from disk) the number of messages for the inbox witness [witness_hash] - stored by the rollup node. *) +(** [get_num_messages t witness_hash] retrieves the number of messages for the + inbox witness [witness_hash] stored by the rollup node. *) val get_num_messages : _ t -> Sc_rollup.Inbox_merkelized_payload_hashes.Hash.t -> int tzresult Lwt.t -(** [save_messages t payloads_hash messages] associates the list of [messages] - to the [payloads_hash]. The payload hash must be computed by calling, - e.g. {!Sc_rollup.Inbox.add_all_messages}. *) +(** [save_messages t payloads_hash ~block_hash messages] associates the list of + [messages] to the [payloads_hash]. The payload hash must be computed by + calling, e.g. {!Sc_rollup.Inbox.add_all_messages}. *) val save_messages : rw -> Sc_rollup.Inbox_merkelized_payload_hashes.Hash.t -> - messages_info -> + block_hash:Block_hash.t -> + Sc_rollup.Inbox_message.t list -> unit tzresult Lwt.t (** {3 DAL} *) diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/store.ml b/src/proto_017_PtNairob/lib_sc_rollup_node/store.ml index 90d354f79c800bfd9c773751bf1b6cd5f9b4392a..f89bd6e8cfb551522c566991a8cede8bb4e94e69 100644 --- a/src/proto_017_PtNairob/lib_sc_rollup_node/store.ml +++ b/src/proto_017_PtNairob/lib_sc_rollup_node/store.ml @@ -24,4 +24,4 @@ (* *) (*****************************************************************************) -include Store_v1 +include Store_v2 diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/store_migration.ml b/src/proto_017_PtNairob/lib_sc_rollup_node/store_migration.ml index 2c3fd7901eca6c2b8475346f51737f314f14ecf7..3805963a8cac526f8018a2e8066a367fb1e22169 100644 --- a/src/proto_017_PtNairob/lib_sc_rollup_node/store_migration.ml +++ b/src/proto_017_PtNairob/lib_sc_rollup_node/store_migration.ml @@ -224,7 +224,7 @@ let maybe_run_migration ~storage_dir = in Format.printf "Store migration completed@.") -module V0_to_V1 = struct +module V1_migrations = struct let convert_store_messages (messages, (block_hash, timestamp, number_of_messages)) = ( messages, @@ -293,7 +293,7 @@ module V0_to_V1 = struct let*! () = migrate_dal_processed_slots_irmin v1_store in return_unit - include + module From_v0 = Make (Store_v0) (Store_v1) (struct let migrate_block_action = migrate_messages @@ -301,3 +301,138 @@ module V0_to_V1 = struct let final_actions = final_actions end) end + +module V2_migrations = struct + let messages_store_location ~storage_dir = + let open Filename.Infix in + storage_dir // "messages" + + let commitments_store_location ~storage_dir = + let open Filename.Infix in + storage_dir // "commitments" + + let inboxes_store_location ~storage_dir = + let open Filename.Infix in + storage_dir // "inboxes" + + let migrate_messages read_messages (v2_store : _ Store_v2.t) + (l2_block : Sc_rollup_block.t) = + let open Lwt_result_syntax in + let* v1_messages = read_messages l2_block.header.inbox_witness in + match v1_messages with + | None -> return_unit + | Some (messages, _v1_header) -> + let header = l2_block.header.block_hash in + Store_v2.Messages.append + v2_store.messages + ~key:l2_block.header.inbox_witness + ~header + ~value:messages + + let migrate_commitment read_commitment (v2_store : _ Store_v2.t) + (l2_block : Sc_rollup_block.t) = + let open Lwt_result_syntax in + match l2_block.header.commitment_hash with + | None -> return_unit + | Some commitment_hash -> ( + let* v1_commitment = read_commitment commitment_hash in + match v1_commitment with + | None -> return_unit + | Some commitment -> + Store_v2.Commitments.append + v2_store.commitments + ~key:commitment_hash + ~value:commitment) + + let migrate_inbox read_inbox (v2_store : _ Store_v2.t) + (l2_block : Sc_rollup_block.t) = + let open Lwt_result_syntax in + let* v1_inbox = read_inbox l2_block.header.inbox_hash in + match v1_inbox with + | None -> return_unit + | Some (inbox, ()) -> + Store_v2.Inboxes.append + v2_store.inboxes + ~key:l2_block.header.inbox_hash + ~value:inbox + + let final_actions ~storage_dir ~tmp_dir _ _ = + let open Lwt_result_syntax in + let*! () = + Lwt_utils_unix.remove_dir (messages_store_location ~storage_dir) + in + let*! () = + Lwt_utils_unix.remove_dir (commitments_store_location ~storage_dir) + in + let*! () = + Lwt_utils_unix.remove_dir (inboxes_store_location ~storage_dir) + in + let*! () = + Lwt_unix.rename + (messages_store_location ~storage_dir:tmp_dir) + (messages_store_location ~storage_dir) + in + let*! () = + Lwt_unix.rename + (commitments_store_location ~storage_dir:tmp_dir) + (commitments_store_location ~storage_dir) + in + let*! () = + Lwt_unix.rename + (inboxes_store_location ~storage_dir:tmp_dir) + (inboxes_store_location ~storage_dir) + in + return_unit + + module From_v1 = + Make (Store_v1) (Store_v2) + (struct + let migrate_block_action v1_store v2_store l2_block = + let open Lwt_result_syntax in + let* () = + migrate_messages + Store_v1.(Messages.read v1_store.messages) + v2_store + l2_block + and* () = + migrate_commitment + Store_v1.(Commitments.find v1_store.commitments) + v2_store + l2_block + and* () = + migrate_inbox + Store_v1.(Inboxes.read v1_store.inboxes) + v2_store + l2_block + in + return_unit + + let final_actions = final_actions + end) + + module From_v0 = + Make (Store_v0) (Store_v2) + (struct + let migrate_block_action v0_store v2_store l2_block = + let open Lwt_result_syntax in + let* () = + migrate_messages + Store_v0.(Messages.read v0_store.messages) + v2_store + l2_block + and* () = + migrate_commitment + Store_v0.(Commitments.find v0_store.commitments) + v2_store + l2_block + and* () = + migrate_inbox + Store_v0.(Inboxes.read v0_store.inboxes) + v2_store + l2_block + in + return_unit + + let final_actions = final_actions + end) +end diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/store_v2.ml b/src/proto_017_PtNairob/lib_sc_rollup_node/store_v2.ml new file mode 100644 index 0000000000000000000000000000000000000000..c8363e787effc9c091ed4925dd7a36edc0c282b4 --- /dev/null +++ b/src/proto_017_PtNairob/lib_sc_rollup_node/store_v2.ml @@ -0,0 +1,269 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2021 Nomadic Labs, *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +open Protocol +open Alpha_context +include Store_sigs +include Store_utils +include Store_v1 + +let version = Store_version.V2 + +module Make_hash_index_key (H : Environment.S.HASH) = +Indexed_store.Make_index_key (struct + include Indexed_store.Make_fixed_encodable (H) + + let equal = H.equal +end) + +(** Unaggregated messages per block *) +module Messages = + Indexed_store.Make_indexed_file + (struct + let name = "messages" + end) + (Make_hash_index_key (Sc_rollup.Inbox_merkelized_payload_hashes.Hash)) + (struct + type t = Sc_rollup.Inbox_message.t list + + let name = "messages_list" + + let encoding = + Data_encoding.(list @@ dynamic_size Sc_rollup.Inbox_message.encoding) + + module Header = struct + type t = Block_hash.t + + let name = "messages_block" + + let encoding = Block_hash.encoding + + let fixed_size = + WithExceptions.Option.get ~loc:__LOC__ + @@ Data_encoding.Binary.fixed_length encoding + end + end) + +module Empty_header = struct + type t = unit + + let name = "empty" + + let encoding = Data_encoding.unit + + let fixed_size = 0 +end + +module Add_empty_header = struct + module Header = Empty_header + + let header _ = () +end + +(** Versioned inboxes *) +module Inboxes = + Indexed_store.Make_simple_indexed_file + (struct + let name = "inboxes" + end) + (Make_hash_index_key (Sc_rollup.Inbox.Hash)) + (struct + type t = Sc_rollup.Inbox.t + + let to_repr inbox = + inbox + |> Data_encoding.Binary.to_string_exn Sc_rollup.Inbox.encoding + |> Data_encoding.Binary.of_string_exn Sc_rollup_inbox_repr.encoding + + let of_repr inbox = + inbox + |> Data_encoding.Binary.to_string_exn Sc_rollup_inbox_repr.encoding + |> Data_encoding.Binary.of_string_exn Sc_rollup.Inbox.encoding + + let encoding = + Data_encoding.conv + (fun x -> to_repr x |> Sc_rollup_inbox_repr.to_versioned) + (fun x -> Sc_rollup_inbox_repr.of_versioned x |> of_repr) + Sc_rollup_inbox_repr.versioned_encoding + + let name = "inbox" + + include Add_empty_header + end) + +(** Versioned commitments *) +module Commitments = + Indexed_store.Make_simple_indexed_file + (struct + let name = "commitments" + end) + (Make_hash_index_key (Sc_rollup.Commitment.Hash)) + (struct + type t = Sc_rollup.Commitment.t + + let to_repr commitment = + commitment + |> Data_encoding.Binary.to_string_exn Sc_rollup.Commitment.encoding + |> Data_encoding.Binary.of_string_exn Sc_rollup_commitment_repr.encoding + + let of_repr commitment = + commitment + |> Data_encoding.Binary.to_string_exn Sc_rollup_commitment_repr.encoding + |> Data_encoding.Binary.of_string_exn Sc_rollup.Commitment.encoding + + let encoding = + Data_encoding.conv + (fun x -> to_repr x |> Sc_rollup_commitment_repr.to_versioned) + (fun x -> Sc_rollup_commitment_repr.of_versioned x |> of_repr) + Sc_rollup_commitment_repr.versioned_encoding + + let name = "commitment" + + include Add_empty_header + end) + +type nonrec 'a store = { + l2_blocks : 'a L2_blocks.t; + messages : 'a Messages.t; + inboxes : 'a Inboxes.t; + commitments : 'a Commitments.t; + commitments_published_at_level : 'a Commitments_published_at_level.t; + l2_head : 'a L2_head.t; + last_finalized_level : 'a Last_finalized_level.t; + levels_to_hashes : 'a Levels_to_hashes.t; + irmin_store : 'a Irmin_store.t; +} + +type 'a t = ([< `Read | `Write > `Read] as 'a) store + +type rw = Store_sigs.rw t + +type ro = Store_sigs.ro t + +let readonly + ({ + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head; + last_finalized_level; + levels_to_hashes; + irmin_store; + } : + _ t) : ro = + { + l2_blocks = L2_blocks.readonly l2_blocks; + messages = Messages.readonly messages; + inboxes = Inboxes.readonly inboxes; + commitments = Commitments.readonly commitments; + commitments_published_at_level = + Commitments_published_at_level.readonly commitments_published_at_level; + l2_head = L2_head.readonly l2_head; + last_finalized_level = Last_finalized_level.readonly last_finalized_level; + levels_to_hashes = Levels_to_hashes.readonly levels_to_hashes; + irmin_store = Irmin_store.readonly irmin_store; + } + +let close + ({ + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head = _; + last_finalized_level = _; + levels_to_hashes; + irmin_store; + } : + _ t) = + let open Lwt_result_syntax in + let+ () = L2_blocks.close l2_blocks + and+ () = Messages.close messages + and+ () = Inboxes.close inboxes + and+ () = Commitments.close commitments + and+ () = Commitments_published_at_level.close commitments_published_at_level + and+ () = Levels_to_hashes.close levels_to_hashes + and+ () = Irmin_store.close irmin_store in + () + +let load (type a) (mode : a mode) ~l2_blocks_cache_size data_dir : + a store tzresult Lwt.t = + let open Lwt_result_syntax in + let path name = Filename.concat data_dir name in + let cache_size = l2_blocks_cache_size in + let* l2_blocks = L2_blocks.load mode ~path:(path "l2_blocks") ~cache_size in + let* messages = Messages.load mode ~path:(path "messages") ~cache_size in + let* inboxes = Inboxes.load mode ~path:(path "inboxes") ~cache_size in + let* commitments = + Commitments.load mode ~path:(path "commitments") ~cache_size + in + let* commitments_published_at_level = + Commitments_published_at_level.load + mode + ~path:(path "commitments_published_at_level") + in + let* l2_head = L2_head.load mode ~path:(path "l2_head") in + let* last_finalized_level = + Last_finalized_level.load mode ~path:(path "last_finalized_level") + in + let* levels_to_hashes = + Levels_to_hashes.load mode ~path:(path "levels_to_hashes") + in + let+ irmin_store = Irmin_store.load mode (path "irmin_store") in + { + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head; + last_finalized_level; + levels_to_hashes; + irmin_store; + } + +let iter_l2_blocks ({l2_blocks; l2_head; _} : _ t) f = + let open Lwt_result_syntax in + let* head = L2_head.read l2_head in + match head with + | None -> + (* No reachable head, nothing to do *) + return_unit + | Some head -> + let rec loop hash = + let* block = L2_blocks.read l2_blocks hash in + match block with + | None -> + (* The block does not exist, the known chain stops here, so do we. *) + return_unit + | Some (block, header) -> + let* () = f {block with header} in + loop header.predecessor + in + loop head.header.block_hash diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/store_v2.mli b/src/proto_017_PtNairob/lib_sc_rollup_node/store_v2.mli new file mode 100644 index 0000000000000000000000000000000000000000..dcb851d224f3f0e29439276667ee4f00a82f959b --- /dev/null +++ b/src/proto_017_PtNairob/lib_sc_rollup_node/store_v2.mli @@ -0,0 +1,72 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2022 Nomadic Labs, *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(** This version of the store is used for the rollup nodes for protocols for and + after Nairobi, i.e. >= 17. *) + +open Protocol +open Alpha_context +open Indexed_store + +include module type of struct + include Store_v1 +end + +(** Storage for persisting messages downloaded from the L1 node. *) +module Messages : + INDEXED_FILE + with type key := Sc_rollup.Inbox_merkelized_payload_hashes.Hash.t + and type value := Sc_rollup.Inbox_message.t list + and type header := Block_hash.t + +(** Storage for persisting inboxes. *) +module Inboxes : + SIMPLE_INDEXED_FILE + with type key := Sc_rollup.Inbox.Hash.t + and type value := Sc_rollup.Inbox.t + and type header := unit + +(** Storage containing commitments and corresponding commitment hashes that the + rollup node has knowledge of. *) +module Commitments : + SIMPLE_INDEXED_FILE + with type key := Sc_rollup.Commitment.Hash.t + and type value := Sc_rollup.Commitment.t + and type header := unit + +type +'a store = { + l2_blocks : 'a L2_blocks.t; + messages : 'a Messages.t; + inboxes : 'a Inboxes.t; + commitments : 'a Commitments.t; + commitments_published_at_level : 'a Commitments_published_at_level.t; + l2_head : 'a L2_head.t; + last_finalized_level : 'a Last_finalized_level.t; + levels_to_hashes : 'a Levels_to_hashes.t; + irmin_store : 'a Irmin_store.t; +} + +include Store_sig.S with type 'a store := 'a store diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/store_version.ml b/src/proto_017_PtNairob/lib_sc_rollup_node/store_version.ml index cdb8252b3bae35556ae7b323e08b33118bb870e1..db71f628911302ba849e2d0fdec8895320b2db1b 100644 --- a/src/proto_017_PtNairob/lib_sc_rollup_node/store_version.ml +++ b/src/proto_017_PtNairob/lib_sc_rollup_node/store_version.ml @@ -23,18 +23,20 @@ (* *) (*****************************************************************************) -type t = V0 | V1 +type t = V0 | V1 | V2 let pp ppf v = - Format.pp_print_string ppf @@ match v with V0 -> "v0" | V1 -> "v1" + Format.pp_print_string ppf + @@ match v with V0 -> "v0" | V1 -> "v1" | V2 -> "v2" let encoding = let open Data_encoding in conv - (function V0 -> 0 | V1 -> 1) + (function V0 -> 0 | V1 -> 1 | V2 -> 2) (function | 0 -> V0 | 1 -> V1 + | 2 -> V2 | v -> Format.ksprintf Stdlib.failwith "Unsupported store version %d" v) (obj1 (req "store_version" int31)) diff --git a/src/proto_017_PtNairob/lib_sc_rollup_node/store_version.mli b/src/proto_017_PtNairob/lib_sc_rollup_node/store_version.mli index 870c76d05ba28a4e2039bdfa9b4e1db939aaa694..1f2f8b4a26cc96d70357839ccee9be8f7a91acfb 100644 --- a/src/proto_017_PtNairob/lib_sc_rollup_node/store_version.mli +++ b/src/proto_017_PtNairob/lib_sc_rollup_node/store_version.mli @@ -23,7 +23,7 @@ (* *) (*****************************************************************************) -type t = V0 | V1 +type t = V0 | V1 | V2 (** Pretty-printer for store versions *) val pp : Format.formatter -> t -> unit diff --git a/src/proto_alpha/lib_sc_rollup_node/daemon.ml b/src/proto_alpha/lib_sc_rollup_node/daemon.ml index 48f0eeda36d7e86df33ad15e1f45adb13ed89963..09fa40360b9559d351c5bb9ff49ac0cd25fe2d3e 100644 --- a/src/proto_alpha/lib_sc_rollup_node/daemon.ml +++ b/src/proto_alpha/lib_sc_rollup_node/daemon.ml @@ -600,7 +600,7 @@ module Make (PVM : Pvm.S) = struct node_ctxt ~is_first_block ~predecessor - ~level:head.level + head messages in let* ctxt, _num_messages, num_ticks, initial_tick = diff --git a/src/proto_alpha/lib_sc_rollup_node/inbox.ml b/src/proto_alpha/lib_sc_rollup_node/inbox.ml index 85fafb360157adbc2895a7f7ab009b4c80d17067..51b8f497d3260a2eb501ec4220104cd9cf8d4453 100644 --- a/src/proto_alpha/lib_sc_rollup_node/inbox.ml +++ b/src/proto_alpha/lib_sc_rollup_node/inbox.ml @@ -119,8 +119,9 @@ let add_messages ~is_first_block ~predecessor_timestamp ~predecessor inbox messages_with_protocol_internal_messages ) let process_messages (node_ctxt : _ Node_context.t) ~is_first_block - ~(predecessor : Layer1.header) ~level messages = + ~(predecessor : Layer1.header) (head : Layer1.header) messages = let open Lwt_result_syntax in + let level = head.level in let* inbox = Node_context.inbox_of_head node_ctxt (Layer1.head_of_header predecessor) in @@ -147,12 +148,8 @@ let process_messages (node_ctxt : _ Node_context.t) ~is_first_block Node_context.save_messages node_ctxt witness_hash - { - is_first_block; - predecessor = predecessor.hash; - predecessor_timestamp; - messages; - } + ~block_hash:head.hash + messages in let* inbox_hash = Node_context.save_inbox node_ctxt inbox in return @@ -191,7 +188,7 @@ let process_head (node_ctxt : _ Node_context.t) ~(predecessor : Layer1.header) node_ctxt ~is_first_block ~predecessor - ~level:head.level + head collected_messages in let* () = same_inbox_as_layer_1 node_ctxt head.hash inbox in diff --git a/src/proto_alpha/lib_sc_rollup_node/inbox.mli b/src/proto_alpha/lib_sc_rollup_node/inbox.mli index 8cd5f4f61ce724811cf67a05664fe6d86240dda7..9fc76a974e4a00f8c78d6935eb9a4e220987865f 100644 --- a/src/proto_alpha/lib_sc_rollup_node/inbox.mli +++ b/src/proto_alpha/lib_sc_rollup_node/inbox.mli @@ -89,7 +89,7 @@ module Internal_for_tests : sig Node_context.rw -> is_first_block:bool -> predecessor:Layer1.header -> - level:int32 -> + Layer1.header -> Sc_rollup.Inbox_message.t list -> (Sc_rollup.Inbox.Hash.t * Sc_rollup.Inbox.t diff --git a/src/proto_alpha/lib_sc_rollup_node/node_context.ml b/src/proto_alpha/lib_sc_rollup_node/node_context.ml index 3eee7f9296693948eb84f01ee0d8ffdfdd0f8c98..504e27f36ab15739d95f3794eaccec8d633f07d6 100644 --- a/src/proto_alpha/lib_sc_rollup_node/node_context.ml +++ b/src/proto_alpha/lib_sc_rollup_node/node_context.ml @@ -556,10 +556,13 @@ let get_predecessor node_ctxt (hash, level) = (* [head] is not already known in the L2 chain *) Layer1.get_predecessor node_ctxt.l1_ctxt (hash, level) -let header_of_head node_ctxt Layer1.{hash; level} = +let header_of_hash node_ctxt hash = let open Lwt_result_syntax in let+ header = Layer1.fetch_tezos_shell_header node_ctxt.cctxt hash in - {Layer1.hash; level; header} + {Layer1.hash; level = header.level; header} + +let header_of_head node_ctxt Layer1.{hash; level = _} = + header_of_hash node_ctxt hash let get_tezos_reorg_for_new_head node_ctxt old_head new_head = let open Lwt_result_syntax in @@ -675,17 +678,19 @@ let block_with_tick ({store; _} as node_ctxt) ~max_level tick = let get_commitment {store; _} commitment_hash = let open Lwt_result_syntax in - let* commitment = Store.Commitments.find store.commitments commitment_hash in + let* commitment = Store.Commitments.read store.commitments commitment_hash in match commitment with | None -> failwith "Could not retrieve commitment %a" Sc_rollup.Commitment.Hash.pp commitment_hash - | Some c -> return c + | Some (c, ()) -> return c let find_commitment {store; _} hash = - Store.Commitments.find store.commitments hash + let open Lwt_result_syntax in + let+ commitment = Store.Commitments.read store.commitments hash in + Option.map fst commitment let commitment_exists {store; _} hash = Store.Commitments.mem store.commitments hash @@ -693,7 +698,9 @@ let commitment_exists {store; _} hash = let save_commitment {store; _} commitment = let open Lwt_result_syntax in let hash = Sc_rollup.Commitment.hash_uncarbonated commitment in - let+ () = Store.Commitments.add store.commitments hash commitment in + let+ () = + Store.Commitments.append store.commitments ~key:hash ~value:commitment + in hash let commitment_published_at_level {store; _} commitment = @@ -795,55 +802,71 @@ type messages_info = { messages : Sc_rollup.Inbox_message.t list; } -let get_messages {store; _} messages_hash = +let find_messages node_ctxt messages_hash = let open Lwt_result_syntax in - let* msg = Store.Messages.read store.messages messages_hash in + let* msg = Store.Messages.read node_ctxt.store.messages messages_hash in match msg with + | None -> return_none + | Some (messages, block_hash) -> + let* header = header_of_hash node_ctxt block_hash in + let* pred_header = header_of_hash node_ctxt header.header.predecessor in + let* grand_parent_header = + header_of_hash node_ctxt pred_header.header.predecessor + in + let is_first_block = + pred_header.header.proto_level <> grand_parent_header.header.proto_level + in + return_some + { + is_first_block; + predecessor = pred_header.hash; + predecessor_timestamp = pred_header.header.timestamp; + messages; + } + +let get_messages_aux find node_ctxt hash = + let open Lwt_result_syntax in + let* res = find node_ctxt hash in + match res with | None -> failwith "Could not retrieve messages with payloads merkelized hash %a" Sc_rollup.Inbox_merkelized_payload_hashes.Hash.pp - messages_hash - | Some - ( messages, - (is_first_block, predecessor, predecessor_timestamp, _num_messages) ) -> - return {is_first_block; predecessor; predecessor_timestamp; messages} + hash + | Some res -> return res -let find_messages {store; _} hash = - let open Lwt_result_syntax in - let+ msgs = Store.Messages.read store.messages hash in - Option.map - (fun ( messages, - (is_first_block, predecessor, predecessor_timestamp, _num_messages) - ) -> {is_first_block; predecessor; predecessor_timestamp; messages}) - msgs +let get_messages node_ctxt = get_messages_aux find_messages node_ctxt + +let get_messages_without_proto_messages node_ctxt = + get_messages_aux + (fun node_ctxt messages_hash -> + let open Lwt_result_syntax in + let* msg = Store.Messages.read node_ctxt.store.messages messages_hash in + match msg with + | None -> return_none + | Some (messages, _block_hash) -> return_some messages) + node_ctxt let get_num_messages {store; _} hash = let open Lwt_result_syntax in - let* header = Store.Messages.header store.messages hash in - match header with + let* msg = Store.Messages.read store.messages hash in + match msg with | None -> failwith "Could not retrieve number of messages for inbox witness %a" Sc_rollup.Inbox_merkelized_payload_hashes.Hash.pp hash - | Some (_first_block, _predecessor, _predecessor_timestamp, num_messages) -> - return num_messages - -let save_messages {store; _} key - {is_first_block; predecessor; predecessor_timestamp; messages} = - Store.Messages.append - store.messages - ~key - ~header: - (is_first_block, predecessor, predecessor_timestamp, List.length messages) - ~value:messages + | Some (messages, _block_hash) -> return (List.length messages) + +let save_messages {store; _} key ~block_hash messages = + Store.Messages.append store.messages ~key ~header:block_hash ~value:messages let get_full_l2_block node_ctxt block_hash = let open Lwt_result_syntax in let* block = get_l2_block node_ctxt block_hash in let* inbox = get_inbox node_ctxt block.header.inbox_hash - and* {messages; _} = get_messages node_ctxt block.header.inbox_witness + and* messages = + get_messages_without_proto_messages node_ctxt block.header.inbox_witness and* commitment = Option.map_es (get_commitment node_ctxt) block.header.commitment_hash in diff --git a/src/proto_alpha/lib_sc_rollup_node/node_context.mli b/src/proto_alpha/lib_sc_rollup_node/node_context.mli index 90dd35e0b217b3c4a588f7c5087dee4067021f9c..addf262fad5cff8ecea6c78dca1717be80ede47a 100644 --- a/src/proto_alpha/lib_sc_rollup_node/node_context.mli +++ b/src/proto_alpha/lib_sc_rollup_node/node_context.mli @@ -358,19 +358,19 @@ val find_messages : Sc_rollup.Inbox_merkelized_payload_hashes.Hash.t -> messages_info option tzresult Lwt.t -(** [get_num_messages t witness_hash] retrieves (without reading all the messages - from disk) the number of messages for the inbox witness [witness_hash] - stored by the rollup node. *) +(** [get_num_messages t witness_hash] retrieves the number of messages for the + inbox witness [witness_hash] stored by the rollup node. *) val get_num_messages : _ t -> Sc_rollup.Inbox_merkelized_payload_hashes.Hash.t -> int tzresult Lwt.t -(** [save_messages t payloads_hash messages] associates the list of [messages] - to the [payloads_hash]. The payload hash must be computed by calling, - e.g. {!Sc_rollup.Inbox.add_all_messages}. *) +(** [save_messages t payloads_hash ~block_hash messages] associates the list of + [messages] to the [payloads_hash]. The payload hash must be computed by + calling, e.g. {!Sc_rollup.Inbox.add_all_messages}. *) val save_messages : rw -> Sc_rollup.Inbox_merkelized_payload_hashes.Hash.t -> - messages_info -> + block_hash:Block_hash.t -> + Sc_rollup.Inbox_message.t list -> unit tzresult Lwt.t (** {3 DAL} *) diff --git a/src/proto_alpha/lib_sc_rollup_node/store.ml b/src/proto_alpha/lib_sc_rollup_node/store.ml index 90d354f79c800bfd9c773751bf1b6cd5f9b4392a..f89bd6e8cfb551522c566991a8cede8bb4e94e69 100644 --- a/src/proto_alpha/lib_sc_rollup_node/store.ml +++ b/src/proto_alpha/lib_sc_rollup_node/store.ml @@ -24,4 +24,4 @@ (* *) (*****************************************************************************) -include Store_v1 +include Store_v2 diff --git a/src/proto_alpha/lib_sc_rollup_node/store_migration.ml b/src/proto_alpha/lib_sc_rollup_node/store_migration.ml index 2c3fd7901eca6c2b8475346f51737f314f14ecf7..3805963a8cac526f8018a2e8066a367fb1e22169 100644 --- a/src/proto_alpha/lib_sc_rollup_node/store_migration.ml +++ b/src/proto_alpha/lib_sc_rollup_node/store_migration.ml @@ -224,7 +224,7 @@ let maybe_run_migration ~storage_dir = in Format.printf "Store migration completed@.") -module V0_to_V1 = struct +module V1_migrations = struct let convert_store_messages (messages, (block_hash, timestamp, number_of_messages)) = ( messages, @@ -293,7 +293,7 @@ module V0_to_V1 = struct let*! () = migrate_dal_processed_slots_irmin v1_store in return_unit - include + module From_v0 = Make (Store_v0) (Store_v1) (struct let migrate_block_action = migrate_messages @@ -301,3 +301,138 @@ module V0_to_V1 = struct let final_actions = final_actions end) end + +module V2_migrations = struct + let messages_store_location ~storage_dir = + let open Filename.Infix in + storage_dir // "messages" + + let commitments_store_location ~storage_dir = + let open Filename.Infix in + storage_dir // "commitments" + + let inboxes_store_location ~storage_dir = + let open Filename.Infix in + storage_dir // "inboxes" + + let migrate_messages read_messages (v2_store : _ Store_v2.t) + (l2_block : Sc_rollup_block.t) = + let open Lwt_result_syntax in + let* v1_messages = read_messages l2_block.header.inbox_witness in + match v1_messages with + | None -> return_unit + | Some (messages, _v1_header) -> + let header = l2_block.header.block_hash in + Store_v2.Messages.append + v2_store.messages + ~key:l2_block.header.inbox_witness + ~header + ~value:messages + + let migrate_commitment read_commitment (v2_store : _ Store_v2.t) + (l2_block : Sc_rollup_block.t) = + let open Lwt_result_syntax in + match l2_block.header.commitment_hash with + | None -> return_unit + | Some commitment_hash -> ( + let* v1_commitment = read_commitment commitment_hash in + match v1_commitment with + | None -> return_unit + | Some commitment -> + Store_v2.Commitments.append + v2_store.commitments + ~key:commitment_hash + ~value:commitment) + + let migrate_inbox read_inbox (v2_store : _ Store_v2.t) + (l2_block : Sc_rollup_block.t) = + let open Lwt_result_syntax in + let* v1_inbox = read_inbox l2_block.header.inbox_hash in + match v1_inbox with + | None -> return_unit + | Some (inbox, ()) -> + Store_v2.Inboxes.append + v2_store.inboxes + ~key:l2_block.header.inbox_hash + ~value:inbox + + let final_actions ~storage_dir ~tmp_dir _ _ = + let open Lwt_result_syntax in + let*! () = + Lwt_utils_unix.remove_dir (messages_store_location ~storage_dir) + in + let*! () = + Lwt_utils_unix.remove_dir (commitments_store_location ~storage_dir) + in + let*! () = + Lwt_utils_unix.remove_dir (inboxes_store_location ~storage_dir) + in + let*! () = + Lwt_unix.rename + (messages_store_location ~storage_dir:tmp_dir) + (messages_store_location ~storage_dir) + in + let*! () = + Lwt_unix.rename + (commitments_store_location ~storage_dir:tmp_dir) + (commitments_store_location ~storage_dir) + in + let*! () = + Lwt_unix.rename + (inboxes_store_location ~storage_dir:tmp_dir) + (inboxes_store_location ~storage_dir) + in + return_unit + + module From_v1 = + Make (Store_v1) (Store_v2) + (struct + let migrate_block_action v1_store v2_store l2_block = + let open Lwt_result_syntax in + let* () = + migrate_messages + Store_v1.(Messages.read v1_store.messages) + v2_store + l2_block + and* () = + migrate_commitment + Store_v1.(Commitments.find v1_store.commitments) + v2_store + l2_block + and* () = + migrate_inbox + Store_v1.(Inboxes.read v1_store.inboxes) + v2_store + l2_block + in + return_unit + + let final_actions = final_actions + end) + + module From_v0 = + Make (Store_v0) (Store_v2) + (struct + let migrate_block_action v0_store v2_store l2_block = + let open Lwt_result_syntax in + let* () = + migrate_messages + Store_v0.(Messages.read v0_store.messages) + v2_store + l2_block + and* () = + migrate_commitment + Store_v0.(Commitments.find v0_store.commitments) + v2_store + l2_block + and* () = + migrate_inbox + Store_v0.(Inboxes.read v0_store.inboxes) + v2_store + l2_block + in + return_unit + + let final_actions = final_actions + end) +end diff --git a/src/proto_alpha/lib_sc_rollup_node/store_v2.ml b/src/proto_alpha/lib_sc_rollup_node/store_v2.ml new file mode 100644 index 0000000000000000000000000000000000000000..c8363e787effc9c091ed4925dd7a36edc0c282b4 --- /dev/null +++ b/src/proto_alpha/lib_sc_rollup_node/store_v2.ml @@ -0,0 +1,269 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2021 Nomadic Labs, *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +open Protocol +open Alpha_context +include Store_sigs +include Store_utils +include Store_v1 + +let version = Store_version.V2 + +module Make_hash_index_key (H : Environment.S.HASH) = +Indexed_store.Make_index_key (struct + include Indexed_store.Make_fixed_encodable (H) + + let equal = H.equal +end) + +(** Unaggregated messages per block *) +module Messages = + Indexed_store.Make_indexed_file + (struct + let name = "messages" + end) + (Make_hash_index_key (Sc_rollup.Inbox_merkelized_payload_hashes.Hash)) + (struct + type t = Sc_rollup.Inbox_message.t list + + let name = "messages_list" + + let encoding = + Data_encoding.(list @@ dynamic_size Sc_rollup.Inbox_message.encoding) + + module Header = struct + type t = Block_hash.t + + let name = "messages_block" + + let encoding = Block_hash.encoding + + let fixed_size = + WithExceptions.Option.get ~loc:__LOC__ + @@ Data_encoding.Binary.fixed_length encoding + end + end) + +module Empty_header = struct + type t = unit + + let name = "empty" + + let encoding = Data_encoding.unit + + let fixed_size = 0 +end + +module Add_empty_header = struct + module Header = Empty_header + + let header _ = () +end + +(** Versioned inboxes *) +module Inboxes = + Indexed_store.Make_simple_indexed_file + (struct + let name = "inboxes" + end) + (Make_hash_index_key (Sc_rollup.Inbox.Hash)) + (struct + type t = Sc_rollup.Inbox.t + + let to_repr inbox = + inbox + |> Data_encoding.Binary.to_string_exn Sc_rollup.Inbox.encoding + |> Data_encoding.Binary.of_string_exn Sc_rollup_inbox_repr.encoding + + let of_repr inbox = + inbox + |> Data_encoding.Binary.to_string_exn Sc_rollup_inbox_repr.encoding + |> Data_encoding.Binary.of_string_exn Sc_rollup.Inbox.encoding + + let encoding = + Data_encoding.conv + (fun x -> to_repr x |> Sc_rollup_inbox_repr.to_versioned) + (fun x -> Sc_rollup_inbox_repr.of_versioned x |> of_repr) + Sc_rollup_inbox_repr.versioned_encoding + + let name = "inbox" + + include Add_empty_header + end) + +(** Versioned commitments *) +module Commitments = + Indexed_store.Make_simple_indexed_file + (struct + let name = "commitments" + end) + (Make_hash_index_key (Sc_rollup.Commitment.Hash)) + (struct + type t = Sc_rollup.Commitment.t + + let to_repr commitment = + commitment + |> Data_encoding.Binary.to_string_exn Sc_rollup.Commitment.encoding + |> Data_encoding.Binary.of_string_exn Sc_rollup_commitment_repr.encoding + + let of_repr commitment = + commitment + |> Data_encoding.Binary.to_string_exn Sc_rollup_commitment_repr.encoding + |> Data_encoding.Binary.of_string_exn Sc_rollup.Commitment.encoding + + let encoding = + Data_encoding.conv + (fun x -> to_repr x |> Sc_rollup_commitment_repr.to_versioned) + (fun x -> Sc_rollup_commitment_repr.of_versioned x |> of_repr) + Sc_rollup_commitment_repr.versioned_encoding + + let name = "commitment" + + include Add_empty_header + end) + +type nonrec 'a store = { + l2_blocks : 'a L2_blocks.t; + messages : 'a Messages.t; + inboxes : 'a Inboxes.t; + commitments : 'a Commitments.t; + commitments_published_at_level : 'a Commitments_published_at_level.t; + l2_head : 'a L2_head.t; + last_finalized_level : 'a Last_finalized_level.t; + levels_to_hashes : 'a Levels_to_hashes.t; + irmin_store : 'a Irmin_store.t; +} + +type 'a t = ([< `Read | `Write > `Read] as 'a) store + +type rw = Store_sigs.rw t + +type ro = Store_sigs.ro t + +let readonly + ({ + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head; + last_finalized_level; + levels_to_hashes; + irmin_store; + } : + _ t) : ro = + { + l2_blocks = L2_blocks.readonly l2_blocks; + messages = Messages.readonly messages; + inboxes = Inboxes.readonly inboxes; + commitments = Commitments.readonly commitments; + commitments_published_at_level = + Commitments_published_at_level.readonly commitments_published_at_level; + l2_head = L2_head.readonly l2_head; + last_finalized_level = Last_finalized_level.readonly last_finalized_level; + levels_to_hashes = Levels_to_hashes.readonly levels_to_hashes; + irmin_store = Irmin_store.readonly irmin_store; + } + +let close + ({ + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head = _; + last_finalized_level = _; + levels_to_hashes; + irmin_store; + } : + _ t) = + let open Lwt_result_syntax in + let+ () = L2_blocks.close l2_blocks + and+ () = Messages.close messages + and+ () = Inboxes.close inboxes + and+ () = Commitments.close commitments + and+ () = Commitments_published_at_level.close commitments_published_at_level + and+ () = Levels_to_hashes.close levels_to_hashes + and+ () = Irmin_store.close irmin_store in + () + +let load (type a) (mode : a mode) ~l2_blocks_cache_size data_dir : + a store tzresult Lwt.t = + let open Lwt_result_syntax in + let path name = Filename.concat data_dir name in + let cache_size = l2_blocks_cache_size in + let* l2_blocks = L2_blocks.load mode ~path:(path "l2_blocks") ~cache_size in + let* messages = Messages.load mode ~path:(path "messages") ~cache_size in + let* inboxes = Inboxes.load mode ~path:(path "inboxes") ~cache_size in + let* commitments = + Commitments.load mode ~path:(path "commitments") ~cache_size + in + let* commitments_published_at_level = + Commitments_published_at_level.load + mode + ~path:(path "commitments_published_at_level") + in + let* l2_head = L2_head.load mode ~path:(path "l2_head") in + let* last_finalized_level = + Last_finalized_level.load mode ~path:(path "last_finalized_level") + in + let* levels_to_hashes = + Levels_to_hashes.load mode ~path:(path "levels_to_hashes") + in + let+ irmin_store = Irmin_store.load mode (path "irmin_store") in + { + l2_blocks; + messages; + inboxes; + commitments; + commitments_published_at_level; + l2_head; + last_finalized_level; + levels_to_hashes; + irmin_store; + } + +let iter_l2_blocks ({l2_blocks; l2_head; _} : _ t) f = + let open Lwt_result_syntax in + let* head = L2_head.read l2_head in + match head with + | None -> + (* No reachable head, nothing to do *) + return_unit + | Some head -> + let rec loop hash = + let* block = L2_blocks.read l2_blocks hash in + match block with + | None -> + (* The block does not exist, the known chain stops here, so do we. *) + return_unit + | Some (block, header) -> + let* () = f {block with header} in + loop header.predecessor + in + loop head.header.block_hash diff --git a/src/proto_alpha/lib_sc_rollup_node/store_v2.mli b/src/proto_alpha/lib_sc_rollup_node/store_v2.mli new file mode 100644 index 0000000000000000000000000000000000000000..dcb851d224f3f0e29439276667ee4f00a82f959b --- /dev/null +++ b/src/proto_alpha/lib_sc_rollup_node/store_v2.mli @@ -0,0 +1,72 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2022 Nomadic Labs, *) +(* Copyright (c) 2023 Functori, *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(** This version of the store is used for the rollup nodes for protocols for and + after Nairobi, i.e. >= 17. *) + +open Protocol +open Alpha_context +open Indexed_store + +include module type of struct + include Store_v1 +end + +(** Storage for persisting messages downloaded from the L1 node. *) +module Messages : + INDEXED_FILE + with type key := Sc_rollup.Inbox_merkelized_payload_hashes.Hash.t + and type value := Sc_rollup.Inbox_message.t list + and type header := Block_hash.t + +(** Storage for persisting inboxes. *) +module Inboxes : + SIMPLE_INDEXED_FILE + with type key := Sc_rollup.Inbox.Hash.t + and type value := Sc_rollup.Inbox.t + and type header := unit + +(** Storage containing commitments and corresponding commitment hashes that the + rollup node has knowledge of. *) +module Commitments : + SIMPLE_INDEXED_FILE + with type key := Sc_rollup.Commitment.Hash.t + and type value := Sc_rollup.Commitment.t + and type header := unit + +type +'a store = { + l2_blocks : 'a L2_blocks.t; + messages : 'a Messages.t; + inboxes : 'a Inboxes.t; + commitments : 'a Commitments.t; + commitments_published_at_level : 'a Commitments_published_at_level.t; + l2_head : 'a L2_head.t; + last_finalized_level : 'a Last_finalized_level.t; + levels_to_hashes : 'a Levels_to_hashes.t; + irmin_store : 'a Irmin_store.t; +} + +include Store_sig.S with type 'a store := 'a store diff --git a/src/proto_alpha/lib_sc_rollup_node/store_version.ml b/src/proto_alpha/lib_sc_rollup_node/store_version.ml index cdb8252b3bae35556ae7b323e08b33118bb870e1..db71f628911302ba849e2d0fdec8895320b2db1b 100644 --- a/src/proto_alpha/lib_sc_rollup_node/store_version.ml +++ b/src/proto_alpha/lib_sc_rollup_node/store_version.ml @@ -23,18 +23,20 @@ (* *) (*****************************************************************************) -type t = V0 | V1 +type t = V0 | V1 | V2 let pp ppf v = - Format.pp_print_string ppf @@ match v with V0 -> "v0" | V1 -> "v1" + Format.pp_print_string ppf + @@ match v with V0 -> "v0" | V1 -> "v1" | V2 -> "v2" let encoding = let open Data_encoding in conv - (function V0 -> 0 | V1 -> 1) + (function V0 -> 0 | V1 -> 1 | V2 -> 2) (function | 0 -> V0 | 1 -> V1 + | 2 -> V2 | v -> Format.ksprintf Stdlib.failwith "Unsupported store version %d" v) (obj1 (req "store_version" int31)) diff --git a/src/proto_alpha/lib_sc_rollup_node/store_version.mli b/src/proto_alpha/lib_sc_rollup_node/store_version.mli index 870c76d05ba28a4e2039bdfa9b4e1db939aaa694..1f2f8b4a26cc96d70357839ccee9be8f7a91acfb 100644 --- a/src/proto_alpha/lib_sc_rollup_node/store_version.mli +++ b/src/proto_alpha/lib_sc_rollup_node/store_version.mli @@ -23,7 +23,7 @@ (* *) (*****************************************************************************) -type t = V0 | V1 +type t = V0 | V1 | V2 (** Pretty-printer for store versions *) val pp : Format.formatter -> t -> unit