From 24b12c1035442be8a300189fc2984a38b7db26db Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Wed, 9 Oct 2024 20:54:39 +0200 Subject: [PATCH 1/6] Rollup node/Store: migrations part of store lib with less dependencies --- manifest/product_octez.ml | 14 ++++++- src/lib_smart_rollup_node/dune | 28 +++++++++++++- src/lib_smart_rollup_node/store_migration.ml | 37 ++++++++----------- src/lib_smart_rollup_node/store_migration.mli | 3 +- src/lib_smart_rollup_node/store_v0.ml | 25 ++++++++++++- src/lib_smart_rollup_node/store_v2.ml | 9 ++++- src/lib_smart_rollup_node/store_v2.mli | 5 ++- 7 files changed, 90 insertions(+), 31 deletions(-) diff --git a/manifest/product_octez.ml b/manifest/product_octez.ml index b99bf8215619..a85dfbb7a520 100644 --- a/manifest/product_octez.ml +++ b/manifest/product_octez.ml @@ -4851,7 +4851,19 @@ let rollup_node_sqlite_migrations = ] let octez_smart_rollup_node_store_lib_modules = - ["store_version"; "sql_store"; "store_v5"; "store"] + [ + "store_version"; + "sql_store"; + "store_sig"; + "store_v0"; + "store_v1"; + "store_v2"; + "store_v3"; + "store_v4"; + "store_v5"; + "store"; + "store_migration"; + ] let octez_smart_rollup_node_store_lib = octez_l2_lib diff --git a/src/lib_smart_rollup_node/dune b/src/lib_smart_rollup_node/dune index edf7ada75411..9d2bd259b031 100644 --- a/src/lib_smart_rollup_node/dune +++ b/src/lib_smart_rollup_node/dune @@ -21,7 +21,18 @@ -open Tezos_layer2_store -open Octez_sqlite -open Octez_smart_rollup) - (modules store_version sql_store store_v5 store)) + (modules + store_version + sql_store + store_sig + store_v0 + store_v1 + store_v2 + store_v3 + store_v4 + store_v5 + store + store_migration)) (library (name octez_smart_rollup_node) @@ -66,4 +77,17 @@ -open Octez_crawler -open Tezos_workers -open Octez_smart_rollup) - (modules (:standard \ store_version sql_store store_v5 store))) + (modules + (:standard + \ + store_version + sql_store + store_sig + store_v0 + store_v1 + store_v2 + store_v3 + store_v4 + store_v5 + store + store_migration))) diff --git a/src/lib_smart_rollup_node/store_migration.ml b/src/lib_smart_rollup_node/store_migration.ml index b0e1f841e5bb..2d56ff012fa2 100644 --- a/src/lib_smart_rollup_node/store_migration.ml +++ b/src/lib_smart_rollup_node/store_migration.ml @@ -27,6 +27,8 @@ open Store_version type version_result = Version_known | Unintialized_version +let storage_dir data_dir = Filename.concat data_dir "storage" + let messages_store_location ~storage_dir = let open Filename.Infix in storage_dir // "messages" @@ -180,7 +182,7 @@ struct in let*! () = Lwt_utils_unix.remove_dir tmp_dir in Store_version.write_version_file - ~dir:(Configuration.default_storage_dir data_dir) + ~dir:(storage_dir data_dir) S_dest.version in Lwt.finalize run_migration cleanup @@ -194,7 +196,7 @@ module Wrap_old (S : Store_sig.S) : STORE with type 'a t = 'a S.t = struct let load mode ~data_dir = load mode - (Configuration.default_storage_dir data_dir) + (storage_dir data_dir) ~index_buffer_size:100 ~l2_blocks_cache_size:1 end @@ -227,11 +229,10 @@ let migration_path ~from ~dest = in path [] from dest -let maybe_run_migration metadata ~data_dir = +let maybe_run_migration metadata last_version ~data_dir = let open Lwt_result_syntax in - let storage_dir = Configuration.default_storage_dir data_dir in + let storage_dir = storage_dir data_dir in let* current_version = version_of_store ~storage_dir in - let last_version = Store.version in match (current_version, last_version) with | None, _ -> (* Store not initialized, write last version *) @@ -264,7 +265,7 @@ let maybe_run_migration metadata ~data_dir = module V1_migrations = struct let messages_store_location ~data_dir = let open Filename.Infix in - Configuration.default_storage_dir data_dir // "messages" + storage_dir data_dir // "messages" let convert_store_messages (messages, (block_hash, timestamp, number_of_messages)) = @@ -297,12 +298,7 @@ module V1_migrations = struct Tezos_base.Time.( System.now () |> System.to_protocol |> Protocol.to_seconds) in - let author = - Format.asprintf - "Rollup node %a" - Tezos_version_parser.pp - Tezos_version_value.Current_git_info.octez_version - in + let author = "Rollup node" in let message = "Migration store from v0 to v1" in Irmin_store.Raw_irmin.Info.v ~author ~message date in @@ -344,15 +340,15 @@ end module V2_migrations = struct let messages_store_location ~data_dir = let open Filename.Infix in - Configuration.default_storage_dir data_dir // "messages" + storage_dir data_dir // "messages" let commitments_store_location ~data_dir = let open Filename.Infix in - Configuration.default_storage_dir data_dir // "commitments" + storage_dir data_dir // "commitments" let inboxes_store_location ~data_dir = let open Filename.Infix in - Configuration.default_storage_dir data_dir // "inboxes" + storage_dir data_dir // "inboxes" let migrate_messages read_messages (v2_store : _ Store_v2.t) (l2_block : Sc_rollup_block.t) = @@ -475,7 +471,7 @@ end module V3_migrations = struct let levels_store_location ~data_dir = let open Filename.Infix in - Configuration.default_storage_dir data_dir // "levels_to_hashes" + storage_dir data_dir // "levels_to_hashes" let recompute_level (v3_store : _ Store_v3.t) (l2_block : Sc_rollup_block.t) = Store_v3.Levels_to_hashes.add @@ -507,7 +503,7 @@ end module V4_migrations = struct let messages_store_location ~data_dir = let open Filename.Infix in - Configuration.default_storage_dir data_dir // "messages" + storage_dir data_dir // "messages" let migrate_messages (v3_store : _ Store_v3.t) (v4_store : _ Store_v4.t) (l2_block : Sc_rollup_block.t) = @@ -784,7 +780,8 @@ module V5_sqlite_migrations = struct let* () = match history_mode with | None -> return_unit - | Some m -> Store_v5.State.History_mode.set v5_store m + | Some Archive -> Store_v5.State.History_mode.set v5_store Archive + | Some Full -> Store_v5.State.History_mode.set v5_store Full in return_unit @@ -793,9 +790,7 @@ module V5_sqlite_migrations = struct let* () = migrate_outbox_messages v4_store v5_store in let* () = migrate_protocols v4_store v5_store in let* () = migrate_rollup_node_state v4_store v5_store in - let*! () = - Lwt_utils_unix.remove_dir (Configuration.default_storage_dir data_dir) - in + let*! () = Lwt_utils_unix.remove_dir (storage_dir data_dir) in let mv file = let src = Filename.concat tmp_dir file in let*! exists = Lwt_unix.file_exists src in diff --git a/src/lib_smart_rollup_node/store_migration.mli b/src/lib_smart_rollup_node/store_migration.mli index fabbf80c02ec..a51fffa477ce 100644 --- a/src/lib_smart_rollup_node/store_migration.mli +++ b/src/lib_smart_rollup_node/store_migration.mli @@ -27,4 +27,5 @@ needed. If there is no possible migration path registered to go from the current version to the last {!Store.version}, this function resolves with an error. *) -val maybe_run_migration : Metadata.t -> data_dir:string -> unit tzresult Lwt.t +val maybe_run_migration : + Metadata.t -> Store_version.t -> data_dir:string -> unit tzresult Lwt.t diff --git a/src/lib_smart_rollup_node/store_v0.ml b/src/lib_smart_rollup_node/store_v0.ml index 2ad68841f832..a764e7b32e7b 100644 --- a/src/lib_smart_rollup_node/store_v0.ml +++ b/src/lib_smart_rollup_node/store_v0.ml @@ -65,6 +65,27 @@ Indexed_store.Make_index_key (struct let equal = H.equal end) +module Block_key = struct + include Block_hash + + let hash_size = 31 + + let t = + let open Repr in + map + (bytes_of (`Fixed hash_size)) + (fun b -> Block_hash.of_bytes_exn b) + (fun bh -> Block_hash.to_bytes bh) + + let encode bh = Block_hash.to_string bh + + let encoded_size = Block_hash.size (* in bytes *) + + let decode str off = + let str = String.sub str off encoded_size in + Block_hash.of_string_exn str +end + (** L2 blocks *) module L2_blocks = Indexed_store.Make_indexed_file @@ -72,7 +93,7 @@ module L2_blocks = let name = "l2_blocks" end) (struct - include Tezos_store_shared.Block_key + include Block_key let pp = Block_hash.pp end) @@ -227,7 +248,7 @@ module Levels_to_hashes = let equal = Int32.equal end)) - (Tezos_store_shared.Block_key) + (Block_key) (** stores slots whose data have been considered and pages stored to disk (if they are confirmed). *) diff --git a/src/lib_smart_rollup_node/store_v2.ml b/src/lib_smart_rollup_node/store_v2.ml index 1e1f0132ab95..154637228bb4 100644 --- a/src/lib_smart_rollup_node/store_v2.ml +++ b/src/lib_smart_rollup_node/store_v2.ml @@ -395,12 +395,17 @@ module Last_context_split = Indexed_store.Make_singleton (struct let encoding = Data_encoding.int32 end) +type history_mode = Archive | Full + +let history_mode_encoding : history_mode Data_encoding.t = + Data_encoding.string_enum [("archive", Archive); ("full", Full)] + module History_mode = Indexed_store.Make_singleton (struct - type t = Configuration.history_mode + type t = history_mode let name = "history_mode" - let encoding = Configuration.history_mode_encoding + let encoding = history_mode_encoding end) type 'a store = { diff --git a/src/lib_smart_rollup_node/store_v2.mli b/src/lib_smart_rollup_node/store_v2.mli index d533314bdb5d..9da77a17cb45 100644 --- a/src/lib_smart_rollup_node/store_v2.mli +++ b/src/lib_smart_rollup_node/store_v2.mli @@ -156,9 +156,10 @@ end (** Level at which context was last split. *) module Last_context_split : SINGLETON_STORE with type value := int32 +type history_mode = Archive | Full + (** History mode of the rollup node. *) -module History_mode : - SINGLETON_STORE with type value := Configuration.history_mode +module History_mode : SINGLETON_STORE with type value := history_mode type +'a store = { l2_blocks : 'a L2_blocks.t; -- GitLab From 575ef21780b9c21d760c15a09f21b912e5d5f89b Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Wed, 9 Oct 2024 20:56:02 +0200 Subject: [PATCH 2/6] Rollup node/SQL Store: don't return level in messages query --- etherlink/bin_node/lib_dev/evm_context.ml | 2 +- src/lib_smart_rollup_node/node_context.ml | 4 +--- src/lib_smart_rollup_node/snapshots.ml | 2 +- src/lib_smart_rollup_node/sql_store.ml | 4 ++-- src/lib_smart_rollup_node/sql_store.mli | 2 +- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/etherlink/bin_node/lib_dev/evm_context.ml b/etherlink/bin_node/lib_dev/evm_context.ml index ba11db067902..18758df76e9f 100644 --- a/etherlink/bin_node/lib_dev/evm_context.ml +++ b/etherlink/bin_node/lib_dev/evm_context.ml @@ -1213,7 +1213,7 @@ module State = struct block in let* messages = Store.Messages.find store block.header.inbox_witness in - let*? _level, messages = + let*? messages = Option.to_result ~none:[error_of_fmt "No messages found for block %a" Block_hash.pp hash] messages diff --git a/src/lib_smart_rollup_node/node_context.ml b/src/lib_smart_rollup_node/node_context.ml index 9deb49a2d8a5..d727068209a7 100644 --- a/src/lib_smart_rollup_node/node_context.ml +++ b/src/lib_smart_rollup_node/node_context.ml @@ -642,9 +642,7 @@ let get_inbox_by_block_hash node_ctxt hash = inbox_of_head node_ctxt {hash; level} let find_messages {store; _} payload_hash = - let open Lwt_result_syntax in - let+ res = Store.Messages.find store payload_hash in - Option.map snd res + Store.Messages.find store payload_hash let get_messages node_ctxt messages_hash = let open Lwt_result_syntax in diff --git a/src/lib_smart_rollup_node/snapshots.ml b/src/lib_smart_rollup_node/snapshots.ml index 06ce699d7fa3..5bab286a24fb 100644 --- a/src/lib_smart_rollup_node/snapshots.ml +++ b/src/lib_smart_rollup_node/snapshots.ml @@ -185,7 +185,7 @@ let check_block_data_and_get_content (store : _ Store.t) context hash = let* b = Store.L2_blocks.find store hash in let*? {header; _} = check_some hash "L2 block" b in let* messages = Store.Messages.find store header.inbox_witness in - let*? _level, _messages = check_some hash "messages" messages in + let*? _messages = check_some hash "messages" messages in let* inbox = Store.Inboxes.find store header.inbox_hash in let*? inbox = check_some hash "inbox" inbox in let* commitment = diff --git a/src/lib_smart_rollup_node/sql_store.ml b/src/lib_smart_rollup_node/sql_store.ml index dfe55416d51f..cccbc9afb040 100644 --- a/src/lib_smart_rollup_node/sql_store.ml +++ b/src/lib_smart_rollup_node/sql_store.ml @@ -562,9 +562,9 @@ module Messages = struct |sql} let select = - (payload_hashes_hash ->? t2 level messages_list) + (payload_hashes_hash ->? messages_list) @@ {sql| - SELECT inbox_level, message_list + SELECT message_list FROM messages WHERE payload_hashes_hash = ? |sql} diff --git a/src/lib_smart_rollup_node/sql_store.mli b/src/lib_smart_rollup_node/sql_store.mli index 734ccf769a09..9c7a2daa1d36 100644 --- a/src/lib_smart_rollup_node/sql_store.mli +++ b/src/lib_smart_rollup_node/sql_store.mli @@ -147,7 +147,7 @@ module Messages : sig ?conn:Sqlite.conn -> _ t -> Merkelized_payload_hashes_hash.t -> - (int32 * string list) option tzresult Lwt.t + string list option tzresult Lwt.t end (** Storage for persisting outbox messages. *) -- GitLab From 3f249746c64a339e00ba0b7a96e892711e96c644 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Wed, 9 Oct 2024 20:50:18 +0200 Subject: [PATCH 3/6] Rollup node/Store: expose standalone migration function for v4 -> v5 --- src/lib_smart_rollup_node/store_migration.ml | 46 ++++++++++++------- src/lib_smart_rollup_node/store_migration.mli | 15 ++++++ 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/src/lib_smart_rollup_node/store_migration.ml b/src/lib_smart_rollup_node/store_migration.ml index 2d56ff012fa2..7342ef700a04 100644 --- a/src/lib_smart_rollup_node/store_migration.ml +++ b/src/lib_smart_rollup_node/store_migration.ml @@ -136,10 +136,34 @@ struct Store_version.pp S_dest.version + let migrate_between_stores metadata ~data_dir ~dest_data_dir source_store + dest_store = + let open Lwt_result_syntax in + let* () = + S_from.iter_l2_blocks + ~progress: + (Format.asprintf + "Migrating store from %a to %a" + Store_version.pp + S_from.version + Store_version.pp + S_dest.version) + metadata + source_store + (Actions.migrate_block_action source_store dest_store) + in + let* () = + Actions.final_actions + ~data_dir + ~tmp_dir:dest_data_dir + source_store + dest_store + in + Store_version.write_version_file ~dir:(storage_dir data_dir) S_dest.version + let migrate metadata ~data_dir = let open Lwt_result_syntax in let* source_store = S_from.load Read_only ~data_dir in - let tmp_dir = tmp_dir ~data_dir in let*! tmp_dir_exists = Lwt_utils_unix.dir_exists tmp_dir in let*? () = @@ -165,25 +189,15 @@ struct in let run_migration () = let* () = - S_from.iter_l2_blocks - ~progress: - (Format.asprintf - "Migrating store from %a to %a" - Store_version.pp - S_from.version - Store_version.pp - S_dest.version) + migrate_between_stores metadata + ~data_dir + ~dest_data_dir:tmp_dir source_store - (Actions.migrate_block_action source_store dest_store) - in - let* () = - Actions.final_actions ~data_dir ~tmp_dir source_store dest_store + dest_store in let*! () = Lwt_utils_unix.remove_dir tmp_dir in - Store_version.write_version_file - ~dir:(storage_dir data_dir) - S_dest.version + return_unit in Lwt.finalize run_migration cleanup diff --git a/src/lib_smart_rollup_node/store_migration.mli b/src/lib_smart_rollup_node/store_migration.mli index a51fffa477ce..85b7004613d9 100644 --- a/src/lib_smart_rollup_node/store_migration.mli +++ b/src/lib_smart_rollup_node/store_migration.mli @@ -29,3 +29,18 @@ error. *) val maybe_run_migration : Metadata.t -> Store_version.t -> data_dir:string -> unit tzresult Lwt.t + +(** {2 Version specific standalone migration functions} *) + +module V5_sqlite_migrations : sig + module From_v4 : sig + (** Migration between a store V4 and an SQL store (V5) in place. *) + val migrate_between_stores : + Metadata.t -> + data_dir:string -> + dest_data_dir:string -> + Store_sigs.ro Store_v4.store -> + Store_sigs.rw Store_v5.t -> + unit tzresult Lwt.t + end +end -- GitLab From 1226abd2980dc9cf1e723d9112b767b4865b8cca Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Wed, 9 Oct 2024 20:57:05 +0200 Subject: [PATCH 4/6] Rollup node/Store: run v4 -> v5 migration in the background --- etherlink/bin_node/lib_dev/evm_context.ml | 2 + src/lib_smart_rollup_node/node_context.ml | 3 +- src/lib_smart_rollup_node/node_context.mli | 8 +- .../node_context_loader.ml | 20 +- src/lib_smart_rollup_node/snapshots.ml | 6 + src/lib_smart_rollup_node/store.ml | 793 +++++++++++++++++- .../wasm_2_0_0_utilities.ml | 2 + 7 files changed, 804 insertions(+), 30 deletions(-) diff --git a/etherlink/bin_node/lib_dev/evm_context.ml b/etherlink/bin_node/lib_dev/evm_context.ml index 18758df76e9f..5ae0182ff4c2 100644 --- a/etherlink/bin_node/lib_dev/evm_context.ml +++ b/etherlink/bin_node/lib_dev/evm_context.ml @@ -1701,6 +1701,7 @@ let init_context_from_rollup_node ~data_dir ~rollup_node_data_dir = let* rollup_node_store = Store.init Read_only ~data_dir:rollup_node_data_dir in + let rollup_node_store = Store.Normal rollup_node_store in let* final_l2_block = Store.L2_blocks.find_finalized rollup_node_store in let* final_l2_block = match final_l2_block with @@ -1838,6 +1839,7 @@ let reconstruct ~data_dir ~rollup_node_data_dir ~boot_sector = let* rollup_node_store = Store.init Read_only ~data_dir:rollup_node_data_dir in + let rollup_node_store = Store.Normal rollup_node_store in let* finalized = Store.State.Finalized_level.get rollup_node_store in let* finalized_level = match finalized with diff --git a/src/lib_smart_rollup_node/node_context.ml b/src/lib_smart_rollup_node/node_context.ml index d727068209a7..9c5f0c4ee652 100644 --- a/src/lib_smart_rollup_node/node_context.ml +++ b/src/lib_smart_rollup_node/node_context.ml @@ -37,8 +37,7 @@ type 'a store = 'a Store.t constraint 'a = [< `Read | `Write > `Read] module Node_store = struct let close (s : 'a store) = Store.close s - let load : 'a Store_sigs.mode -> data_dir:string -> 'a store tzresult Lwt.t = - Store.init + let init_with_migration = Store.init_with_migration let check_and_set_history_mode (type a) (mode : a Store_sigs.mode) (store : a Store.t) (history_mode : Configuration.history_mode option) = diff --git a/src/lib_smart_rollup_node/node_context.mli b/src/lib_smart_rollup_node/node_context.mli index 1fe449a731d1..78eaab21d906 100644 --- a/src/lib_smart_rollup_node/node_context.mli +++ b/src/lib_smart_rollup_node/node_context.mli @@ -39,9 +39,11 @@ type 'a store constraint 'a = [< `Read | `Write > `Read] (** Exposed functions to manipulate Node_context store outside of this module *) module Node_store : sig - (** [load mode ~data_dir] loads a store form the data persisted in [data_dir] - as described in. *) - val load : 'a Store_sigs.mode -> data_dir:string -> 'a store tzresult Lwt.t + val init_with_migration : + 'a Store_sigs.mode -> + Metadata.t -> + data_dir:string -> + 'a store tzresult Lwt.t (** [close store] closes the store *) val close : 'a store -> unit Lwt.t diff --git a/src/lib_smart_rollup_node/node_context_loader.ml b/src/lib_smart_rollup_node/node_context_loader.ml index 532551b235ba..ff226422b510 100644 --- a/src/lib_smart_rollup_node/node_context_loader.ml +++ b/src/lib_smart_rollup_node/node_context_loader.ml @@ -80,11 +80,12 @@ let init (cctxt : #Client_context.full) ~data_dir ~irmin_cache_size } in let* () = update_metadata metadata ~data_dir in - let* () = Store_migration.maybe_run_migration metadata ~data_dir in + let* store = + Node_context.Node_store.init_with_migration mode metadata ~data_dir + in let dal_cctxt = Option.map Dal_node_client.make_unix_cctxt dal_node_endpoint in - let* store = Node_context.Node_store.load mode ~data_dir in let*? (module Plugin : Protocol_plugin_sig.S) = Protocol_plugins.proto_plugin_for_protocol current_protocol.hash in @@ -380,7 +381,19 @@ module Internal_for_tests = struct } in let* lockfile = lock ~data_dir in - let* store = Node_context.Node_store.load Read_write ~data_dir in + let genesis_info = {level = 0l; commitment_hash = Commitment.Hash.zero} in + let metadata = + Metadata. + { + rollup_address; + context_version = Context.Version.version; + kind; + genesis_info; + } + in + let* store = + Node_context.Node_store.init_with_migration Read_write metadata ~data_dir + in let*? (module Plugin : Protocol_plugin_sig.S) = Protocol_plugins.proto_plugin_for_protocol current_protocol.hash in @@ -392,7 +405,6 @@ module Internal_for_tests = struct (Configuration.default_context_dir data_dir) ~cache_size:irmin_cache_size in - let genesis_info = {level = 0l; commitment_hash = Commitment.Hash.zero} in let l1_ctxt = Layer1.Internal_for_tests.dummy cctxt in let lcc = Reference.new_ {commitment = Commitment.Hash.zero; level = 0l} in let lpc = Reference.new_ None in diff --git a/src/lib_smart_rollup_node/snapshots.ml b/src/lib_smart_rollup_node/snapshots.ml index 5bab286a24fb..c2cc002333a7 100644 --- a/src/lib_smart_rollup_node/snapshots.ml +++ b/src/lib_smart_rollup_node/snapshots.ml @@ -121,6 +121,7 @@ let pre_export_checks_and_get_snapshot_header cctxt ~no_checks ~data_dir = in let*? () = Context.Version.check metadata.context_version in let* store = Store.init Read_only ~data_dir in + let store = Store.Normal store in let* head = get_head store in let level = head.Sc_rollup_block.header.level in let* (module Plugin) = @@ -502,6 +503,7 @@ let with_modify_data_dir cctxt ~data_dir ~apply_unsafe_patches let context_dir = Configuration.default_context_dir data_dir in let* () = check_store_version store_dir in let* store = Store.init Read_write ~data_dir in + let store = Store.Normal store in let* head = get_head store in let* (module Plugin) = Protocol_plugins.proto_plugin_for_level_with_store store head.header.level @@ -598,6 +600,7 @@ let post_checks ?(apply_unsafe_patches = false) ~action ~message snapshot_header (* Load context and stores in read-only to run checks. *) let* () = check_store_version store_dir in let* store = Store.init Read_only ~data_dir:dest in + let store = Store.Normal store in let* head = get_head store in let* (module Plugin) = Protocol_plugins.proto_plugin_for_level_with_store store head.header.level @@ -810,6 +813,7 @@ let export_compact cctxt ~no_checks ~compression ~data_dir ~dest ~filename = let*! () = Lwt_utils_unix.create_dir tmp_context_dir in let context_dir = Configuration.default_context_dir data_dir in let* store = Store.init Read_only ~data_dir in + let store = Store.Normal store in let* metadata = Metadata.read_metadata_file ~dir:data_dir in let*? metadata = match metadata with @@ -876,6 +880,7 @@ let pre_import_checks cctxt ~no_checks ~data_dir (snapshot_header : Header.t) = let open Lwt_result_syntax in (* Load stores in read-only to make simple checks. *) let* store = Store.init Read_write ~data_dir in + let store = Store.Normal store in let* metadata = Metadata.read_metadata_file ~dir:data_dir in let* history_mode = Store.State.History_mode.get store in let* head = Store.L2_blocks.find_head store in @@ -958,6 +963,7 @@ let correct_history_mode ~data_dir (snapshot_header : Header.t) | Some Archive, Archive | Some Full, Full -> return_unit | Some Full, Archive -> let* store = Store.init Read_write ~data_dir in + let store = Store.Normal store in Store.State.History_mode.set store Full let import ~apply_unsafe_patches ~no_checks ~force cctxt ~data_dir diff --git a/src/lib_smart_rollup_node/store.ml b/src/lib_smart_rollup_node/store.ml index 14d957b86e5a..5af23ea6601d 100644 --- a/src/lib_smart_rollup_node/store.ml +++ b/src/lib_smart_rollup_node/store.ml @@ -1,27 +1,778 @@ (*****************************************************************************) (* *) -(* Open Source License *) -(* Copyright (c) 2021 Nomadic Labs, *) -(* Copyright (c) 2023 Functori, *) -(* *) -(* Permission is hereby granted, free of charge, to any person obtaining a *) -(* copy of this software and associated documentation files (the "Software"),*) -(* to deal in the Software without restriction, including without limitation *) -(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) -(* and/or sell copies of the Software, and to permit persons to whom the *) -(* Software is furnished to do so, subject to the following conditions: *) -(* *) -(* The above copyright notice and this permission notice shall be included *) -(* in all copies or substantial portions of the Software. *) -(* *) -(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) -(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) -(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) -(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) -(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) -(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) -(* DEALINGS IN THE SOFTWARE. *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2024 Functori *) (* *) (*****************************************************************************) +(** The store module is a pointer to the latest version. However, in order to + run long migrations in the background, this module allows to have a store + which is a hybrid version of the last two versions. *) + include Store_v5 + +(* TODO: When the V5 store has been in production for long enough, all the below + code can be removed. *) + +(** The hybrid store. *) +type 'a hybrid = { + v4 : Store_v4.ro; (** A read-only old V4 store. *) + v5 : 'a Store_v5.t; + (** A read-write V5 store to which we migrate data and which we use to + store new data. *) + migration_done : bool ref; + (** A flag to indicate when the migration has terminated. When this is the + case we only query the v5 store. *) +} + +(** The type of stores. *) +type nonrec 'a t = + | Hybrid of 'a hybrid + (** Either a hybrid store when a bacground migration is running. *) + | Normal of 'a t (** Or a "normal" store, for the nominal case. *) + +type rw = Store_sigs.rw t + +type ro = Store_sigs.ro t + +let with_rw (type a) (mode : a Store_sigs.mode) (f : unit -> rw tzresult Lwt.t) + : a t tzresult Lwt.t = + match mode with + | Read_only -> + failwith + "The rollup node store needs a migration but was opened in read-only \ + mode." + | Read_write -> f () + +(** Initialize a store and run the corresponding migrations if needed. For the + v4 -> v5 transition, the migration is run in the background. *) +let init_with_migration (type a) (mode : a Store_sigs.mode) metadata ~data_dir : + a t tzresult Lwt.t = + let open Lwt_result_syntax in + let storage_dir = Filename.concat data_dir "storage" in + let* current_version = Store_version.read_version_file ~dir:storage_dir in + match (current_version, version) with + | Some V4, V5_sqlite -> + with_rw mode @@ fun () -> + Format.eprintf + "Hybrid V4/V5 store while migration runs in the background@." ; + let migration_done = ref false in + let* v4 = + Store_v4.load + Read_only + ~index_buffer_size:1000 + ~l2_blocks_cache_size:64 + storage_dir + and* v5 = Store_v5.init Read_write ~data_dir in + Lwt.dont_wait + (fun () -> + let*! r = + protect @@ fun () -> + (* Migrate in place *) + Store_migration.V5_sqlite_migrations.From_v4.migrate_between_stores + metadata + ~data_dir + ~dest_data_dir:data_dir + v4 + v5 + in + Format.eprintf "Migration thread terminated@." ; + let*! (Ok () | Error _) = Store_v4.close v4 in + (match r with + | Ok () -> migration_done := true + | Error e -> Format.eprintf "Migration error: %a@." pp_print_trace e) ; + Lwt.return_unit) + (fun exn -> + Format.eprintf "Migration exception: %s@." (Printexc.to_string exn)) ; + let store = {migration_done; v4; v5} in + return (Hybrid store) + | _ -> + let* () = + Store_migration.maybe_run_migration metadata version ~data_dir + in + let+ store = init mode ~data_dir in + Normal store + +let close = function + | Normal s -> close s + | Hybrid {v4; v5; _} -> + let open Lwt_syntax in + let* () = close v5 and* _ = Store_v4.close v4 in + return_unit + +let readonly = function + | Normal s -> Normal (readonly s) + | Hybrid {v4; v5; migration_done} -> + Hybrid {v4 = Store_v4.readonly v4; v5 = readonly v5; migration_done} + +let gc = function + | Normal s -> gc s + | Hybrid {v5; _} -> + (* Only run GC on new store, the old one will be removed *) + gc v5 + +let read_from = function + | Hybrid {v5; migration_done = {contents = true}; _} -> + (* If the migration is done we only read in the V5 store. *) + Normal v5 + | store -> store + +let write_to = function + | Normal v5 | Hybrid {v5; _} -> + (* Always write to the V5 store. *) + v5 + +(** [x @?~~> y] executes [x] first and then [y] if [x] resolves to [None] (where + [x] is an access to the V5 store and [y] to the V4 store). *) +let ( @?~~> ) t1 t2 = + let open Lwt_result_syntax in + let*! r = t1 in + (match r with + | Ok _ -> () + | Error e -> Format.eprintf "Store v5 error: %a@." pp_print_trace e) ; + let*? r in + match r with + | Some r -> return_some r + | None -> + let*! r = t2 in + (match r with + | Ok _ -> () + | Error e -> Format.eprintf "Store v4 error: %a@." pp_print_trace e) ; + Lwt.return r + +module Commitments = struct + let store s c = Commitments.store (write_to s) c + + let find s h = + match read_from s with + | Normal s -> Commitments.find s h + | Hybrid {v4; v5; _} -> + Commitments.find v5 h + @?~~> (Store_v4.Commitments.read v4.commitments h + |> Lwt_result.map @@ Option.map fst) + + let find_lcc s = + match read_from s with + | Normal s -> Commitments.find_lcc s () + | Hybrid {v4; v5; _} -> ( + Commitments.find_lcc v5 () + @?~~> + let open Lwt_result_syntax in + let* r = Store_v4.Lcc.read v4.lcc in + match r with + | None -> return_none + | Some {commitment; _} -> + Store_v4.Commitments.read v4.commitments commitment + |> Lwt_result.map @@ Option.map fst) + + let find_lpc s = + match read_from s with + | Normal s -> Commitments.find_lpc s + | Hybrid {v4; v5; _} -> + Commitments.find_lpc v5 @?~~> Store_v4.Lpc.read v4.lpc +end + +module Commitments_published_at_levels = struct + include Commitments_published_at_levels + + let register s h levels = register (write_to s) h levels + + let get s h = + match read_from s with + | Normal s -> get s h + | Hybrid {v4; v5; _} -> ( + get v5 h + @?~~> + let open Lwt_result_syntax in + let+ r = + Store_v4.Commitments_published_at_level.find + v4.commitments_published_at_level + h + in + match r with + | None -> None + | Some {first_published_at_level; published_at_level} -> + Some {first_published_at_level; published_at_level}) + + let get_first_published_level s h = + let open Lwt_result_syntax in + let+ r = get s h in + match r with + | None -> None + | Some {first_published_at_level; _} -> Some first_published_at_level +end + +module Inboxes = struct + let store s i = Inboxes.store (write_to s) i + + let find s h = + match read_from s with + | Normal s -> Inboxes.find s h + | Hybrid {v4; v5; _} -> + Inboxes.find v5 h + @?~~> (Store_v4.Inboxes.read v4.inboxes h + |> Lwt_result.map @@ Option.map fst) + + let find_by_block_hash s h = + match read_from s with + | Normal s -> Inboxes.find_by_block_hash s h + | Hybrid {v4; v5; _} -> ( + Inboxes.find_by_block_hash v5 h + @?~~> + let open Lwt_result_syntax in + let* b = Store_v4.L2_blocks.header v4.l2_blocks h in + match b with + | None -> return_none + | Some {inbox_hash; _} -> + Store_v4.Inboxes.read v4.inboxes inbox_hash + |> Lwt_result.map @@ Option.map fst) +end + +module Messages = struct + let store s i = Messages.store (write_to s) i + + let find s h = + match read_from s with + | Normal s -> Messages.find s h + | Hybrid {v4; v5; _} -> + Messages.find v5 h + @?~~> (Store_v4.Messages.read v4.messages h + |> Lwt_result.map @@ Option.map fst) +end + +module Outbox_messages = struct + let pending s ~min_level ~max_level = + match read_from s with + | Normal s -> Outbox_messages.pending s ~min_level ~max_level + | Hybrid {v4; v5; _} -> + let open Lwt_result_syntax in + let* p1 = Outbox_messages.pending v5 ~min_level ~max_level + and* p2 = + Store_v4.Outbox_messages.pending + v4.outbox_messages + ~min_level + ~max_level + in + let rec merge acc p1 p2 = + match (p1, p2) with + | (l1, m1) :: r1, (l2, m2) :: r2 when l1 = l2 -> + (* Pending lists (for both stores) are obtained by taking whole + messages for this level and removing the executed ones. The + pending messages are the ones that are still pending both in v4 + and v5. *) + let m = + List.filter (fun x -> List.mem ~equal:Int.equal x m2) m1 + in + merge ((l1, m) :: acc) r1 r2 + | (l1, m1) :: r1, (l2, _) :: _ when l1 < l2 -> + merge ((l1, m1) :: acc) r1 p2 + | (_, _) :: _, (l2, m2) :: r2 -> merge ((l2, m2) :: acc) p1 r2 + | [], _ -> List.rev_append p2 acc |> List.rev + | _, [] -> List.rev_append p1 acc |> List.rev + in + merge [] p1 p2 |> return + + let register_outbox_messages s ~outbox_level ~indexes = + Outbox_messages.register_outbox_messages (write_to s) ~outbox_level ~indexes + + let set_outbox_message_executed s ~outbox_level ~index = + let open Lwt_result_syntax in + match read_from s with + | Normal s -> + Outbox_messages.set_outbox_message_executed s ~outbox_level ~index + | Hybrid {v4; v5; _} -> + (* copy over pending messages to v5 *) + let* pending = + Store_v4.Outbox_messages.pending + v4.outbox_messages + ~min_level:outbox_level + ~max_level:outbox_level + in + let* () = + match pending with + | [] -> return_unit + | [(outbox_level, indexes)] -> + let*? indexes = Bitset.from_list indexes in + Outbox_messages.register_outbox_messages v5 ~outbox_level ~indexes + | _ -> + (* Only 1 level requested, we can't have more than one element. *) + assert false + in + Outbox_messages.set_outbox_message_executed v5 ~outbox_level ~index +end + +module Protocols = struct + include Protocols + + let store s info = store (write_to s) info + + let find s h = + match read_from s with + | Normal s -> find s h + | Hybrid {v4; v5; _} -> ( + find v5 h + @?~~> + let open Lwt_result_syntax in + let* protos = Store_v4.Protocols.read v4.protocols in + match protos with + | None -> return_none + | Some protos -> + List.find_map + (fun Store_v4.Protocols.{level; proto_level; protocol} -> + if Protocol_hash.equal protocol h then + let level = + match level with + | First_known l -> First_known l + | Activation_level l -> Activation_level l + in + Some {level; proto_level; protocol} + else None) + protos + |> return) + + let proto_of_level s level = + match read_from s with + | Normal s -> proto_of_level s level + | Hybrid {v4; v5; _} -> ( + proto_of_level v5 level + @?~~> + let open Lwt_result_syntax in + let* protos = Store_v4.Protocols.read v4.protocols in + match protos with + | None -> return_none + | Some protos -> + let rec find = function + | [] -> None + | Store_v4.Protocols. + { + level = (First_known l | Activation_level l) as xl; + proto_level; + protocol; + } + :: _ + when level > l -> + let level = + match xl with + | First_known l -> First_known l + | Activation_level l -> Activation_level l + in + Some {level; proto_level; protocol} + | _ :: rest -> find rest + in + find protos |> return) + + let last s = + match read_from s with + | Normal s -> last s + | Hybrid {v4; v5; _} -> ( + last v5 + @?~~> + let open Lwt_result_syntax in + let* protos = Store_v4.Protocols.read v4.protocols in + match protos with + | None -> return_none + | Some [] -> return_none + | Some (Store_v4.Protocols.{level; proto_level; protocol} :: _) -> + let level = + match level with + | First_known l -> First_known l + | Activation_level l -> Activation_level l + in + return_some {level; proto_level; protocol}) +end + +module Dal_slots_headers = struct + let store s block header = Dal_slots_headers.store (write_to s) block header + + let find_slot_header s block ~slot_index = + match read_from s with + | Normal s -> Dal_slots_headers.find_slot_header s block ~slot_index + | Hybrid {v4; v5; _} -> + Dal_slots_headers.find_slot_header v5 block ~slot_index + @?~~> Store_v4.Dal_slots_headers.find + v4.irmin_store + ~primary_key:block + ~secondary_key:slot_index + + let list_slot_headers s block = + match read_from s with + | Normal s -> Dal_slots_headers.list_slot_headers s block + | Hybrid {v4; v5; _} -> ( + let open Lwt_result_syntax in + let* v5_headers = Dal_slots_headers.list_slot_headers v5 block in + match v5_headers with + | _ :: _ -> return v5_headers + | [] -> + Store_v4.Dal_slots_headers.list_values + v4.irmin_store + ~primary_key:block) + + let list_slot_indexes s block = + match read_from s with + | Normal s -> Dal_slots_headers.list_slot_indexes s block + | Hybrid {v4; v5; _} -> ( + let open Lwt_result_syntax in + let* v5_indexes = Dal_slots_headers.list_slot_indexes v5 block in + match v5_indexes with + | _ :: _ -> return v5_indexes + | [] -> + Store_v4.Dal_slots_headers.list_secondary_keys + v4.irmin_store + ~primary_key:block) +end + +module Dal_slots_statuses = struct + let store s block index status = + Dal_slots_statuses.store (write_to s) block index status + + let find_slot_status s block ~slot_index = + match read_from s with + | Normal s -> Dal_slots_statuses.find_slot_status s block ~slot_index + | Hybrid {v4; v5; _} -> + Dal_slots_statuses.find_slot_status v5 block ~slot_index + @?~~> Store_v4.Dal_slots_statuses.find + v4.irmin_store + ~primary_key:block + ~secondary_key:slot_index + + let list_slot_statuses s block = + match read_from s with + | Normal s -> Dal_slots_statuses.list_slot_statuses s block + | Hybrid {v4; v5; _} -> ( + let open Lwt_result_syntax in + let* v5_statuses = Dal_slots_statuses.list_slot_statuses v5 block in + match v5_statuses with + | _ :: _ -> return v5_statuses + | [] -> + Store_v4.Dal_slots_statuses.list_secondary_keys_with_values + v4.irmin_store + ~primary_key:block) +end + +module L2_levels = struct + let store s level hash = L2_levels.store (write_to s) level hash + + let find s level = + match read_from s with + | Normal s -> L2_levels.find s level + | Hybrid {v4; v5; _} -> + L2_levels.find v5 level + @?~~> Store_v4.Levels_to_hashes.find v4.levels_to_hashes level +end + +module L2_blocks = struct + open L2_blocks + + let store s block = store (write_to s) block + + let find s hash = + match read_from s with + | Normal s -> find s hash + | Hybrid {v4; v5; _} -> + find v5 hash + @?~~> + let open Lwt_result_syntax in + let+ res = Store_v4.L2_blocks.read v4.l2_blocks hash in + Option.map + (fun (block, header) -> Sc_rollup_block.{block with header}) + res + + let find_by_level s level = + match read_from s with + | Normal s -> find_by_level s level + | Hybrid {v4; v5; _} -> ( + find_by_level v5 level + @?~~> + let open Lwt_result_syntax in + let* hash = Store_v4.Levels_to_hashes.find v4.levels_to_hashes level in + match hash with + | None -> return_none + | Some hash -> + let+ res = Store_v4.L2_blocks.read v4.l2_blocks hash in + Option.map + (fun (block, header) -> Sc_rollup_block.{block with header}) + res) + + let find_level s hash = + match read_from s with + | Normal s -> find_level s hash + | Hybrid {v4; v5; _} -> + find_level v5 hash + @?~~> + let open Lwt_result_syntax in + let+ res = Store_v4.L2_blocks.header v4.l2_blocks hash in + Option.map (fun header -> header.Sc_rollup_block.level) res + + let find_context s hash = + match read_from s with + | Normal s -> find_context s hash + | Hybrid {v4; v5; _} -> + find_context v5 hash + @?~~> + let open Lwt_result_syntax in + let+ res = Store_v4.L2_blocks.header v4.l2_blocks hash in + Option.map (fun header -> header.Sc_rollup_block.context) res + + let find_head s = + match read_from s with + | Normal s -> find_head s + | Hybrid {v4; v5; _} -> find_head v5 @?~~> Store_v4.L2_head.read v4.l2_head + + let find_finalized s = + match read_from s with + | Normal s -> find_finalized s + | Hybrid {v4; v5; _} -> ( + find_finalized v5 + @?~~> + let open Lwt_result_syntax in + let* level = + Store_v4.Last_finalized_level.read v4.last_finalized_level + in + match level with + | None -> return_none + | Some level -> ( + let* hash = + Store_v4.Levels_to_hashes.find v4.levels_to_hashes level + in + match hash with + | None -> return_none + | Some hash -> + let+ res = Store_v4.L2_blocks.read v4.l2_blocks hash in + Option.map + (fun (block, header) -> Sc_rollup_block.{block with header}) + res)) + + let find_predecessor s hash = + match read_from s with + | Normal s -> find_predecessor s hash + | Hybrid {v4; v5; _} -> + find_predecessor v5 hash + @?~~> + let open Lwt_result_syntax in + let+ res = Store_v4.L2_blocks.header v4.l2_blocks hash in + Option.map + (fun header -> + (header.Sc_rollup_block.predecessor, Int32.pred header.level)) + res + + let find_full s hash = + match read_from s with + | Normal s -> find_full s hash + | Hybrid {v4; v5; _} -> ( + find_full v5 hash + @?~~> + let open Lwt_result_syntax in + let* res = Store_v4.L2_blocks.read v4.l2_blocks hash in + match res with + | None -> return_none + | Some (block, header) -> ( + let* inbox = Store_v4.Inboxes.read v4.inboxes header.inbox_hash + and* messages = + Store_v4.Messages.read v4.messages header.inbox_witness + and* commitment = + match header.commitment_hash with + | None -> return_none + | Some c -> + Store_v4.Commitments.read v4.commitments c + |> Lwt_result.map @@ Option.map fst + in + match (inbox, messages) with + | Some (inbox, ()), Some (messages, _) -> + return_some + { + block with + header; + content = + Sc_rollup_block. + {inbox; messages; commitment; outbox = None}; + } + | _ -> return_none)) +end + +module State = struct + include State + + module Finalized_level = struct + open Finalized_level + + let set s v = set (write_to s) v + + let get s = + match read_from s with + | Normal s -> get s + | Hybrid {v4; v5; _} -> ( + get v5 + @?~~> + let open Lwt_result_syntax in + let* level = + Store_v4.Last_finalized_level.read v4.last_finalized_level + in + match level with + | None -> return_none + | Some level -> ( + let+ hash = + Store_v4.Levels_to_hashes.find v4.levels_to_hashes level + in + match hash with None -> None | Some hash -> Some (hash, level))) + end + + module LCC = struct + open LCC + + let set s v = set (write_to s) v + + let get s = + match read_from s with + | Normal s -> get s + | Hybrid {v4; v5; _} -> ( + get v5 + @?~~> + let open Lwt_result_syntax in + let+ lcc = Store_v4.Lcc.read v4.lcc in + match lcc with + | None -> None + | Some {commitment; level} -> Some (commitment, level)) + end + + module LPC = struct + open LPC + + let set s v = set (write_to s) v + + let get s = + match read_from s with + | Normal s -> get s + | Hybrid {v4; v5; _} -> ( + get v5 + @?~~> + let open Lwt_result_syntax in + let+ lpc = Store_v4.Lpc.read v4.lpc in + match lpc with + | None -> None + | Some commitment -> + let hash = Commitment.hash commitment in + Some (hash, commitment.inbox_level)) + end + + module Last_gc_target = struct + open Last_gc_target + + let set s v = set (write_to s) v + + let get s = + match read_from s with + | Normal s -> get s + | Hybrid {v4; v5; _} -> ( + get v5 + @?~~> + let open Lwt_result_syntax in + let+ gc_levels = Store_v4.Gc_levels.read v4.gc_levels in + match gc_levels with + | None -> None + | Some {first_available_level; _} -> Some first_available_level) + end + + module Last_gc_triggered_at = struct + open Last_gc_triggered_at + + let set s v = set (write_to s) v + + let get s = + match read_from s with + | Normal s -> get s + | Hybrid {v4; v5; _} -> ( + get v5 + @?~~> + let open Lwt_result_syntax in + let+ gc_levels = Store_v4.Gc_levels.read v4.gc_levels in + match gc_levels with + | None -> None + | Some {last_gc_level; _} -> Some last_gc_level) + end + + module Last_successful_gc_target = struct + open Last_successful_gc_target + + let set s v = set (write_to s) v + + let get s = + match read_from s with + | Normal s -> get s + | Hybrid {v4; v5; _} -> ( + get v5 + @?~~> + let open Lwt_result_syntax in + let+ gc_levels = Store_v4.Gc_levels.read v4.successful_gc_levels in + match gc_levels with + | None -> None + | Some {first_available_level; _} -> Some first_available_level) + end + + module Last_successful_gc_triggered_at = struct + open Last_successful_gc_triggered_at + + let set s v = set (write_to s) v + + let get s = + match read_from s with + | Normal s -> get s + | Hybrid {v4; v5; _} -> ( + get v5 + @?~~> + let open Lwt_result_syntax in + let+ gc_levels = Store_v4.Gc_levels.read v4.successful_gc_levels in + match gc_levels with + | None -> None + | Some {last_gc_level; _} -> Some last_gc_level) + end + + module Last_context_split = struct + open Last_context_split + + let set s v = set (write_to s) v + + let get s = + match read_from s with + | Normal s -> get s + | Hybrid {v4; v5; _} -> + get v5 + @?~~> Store_v4.Last_context_split.read v4.last_context_split_level + end + + module History_mode = struct + open History_mode + + let set s v = set (write_to s) v + + let get s = + match read_from s with + | Normal s -> get s + | Hybrid {v4; v5; _} -> ( + get v5 + @?~~> + let open Lwt_result_syntax in + let+ m = Store_v4.History_mode.read v4.history_mode in + match m with + | None -> None + | Some Full -> Some Full + | Some Archive -> Some Archive) + end + + module L2_head = struct + open L2_head + + let set s v = set (write_to s) v + + let get s = + match read_from s with + | Normal s -> get s + | Hybrid {v4; v5; _} -> ( + get v5 + @?~~> + let open Lwt_result_syntax in + let+ block = Store_v4.L2_head.read v4.l2_head in + match block with + | None -> None + | Some block -> Some (block.header.block_hash, block.header.level)) + end +end diff --git a/src/lib_smart_rollup_node/wasm_2_0_0_utilities.ml b/src/lib_smart_rollup_node/wasm_2_0_0_utilities.ml index aeaf8fff7134..46c110d39a2f 100644 --- a/src/lib_smart_rollup_node/wasm_2_0_0_utilities.ml +++ b/src/lib_smart_rollup_node/wasm_2_0_0_utilities.ml @@ -94,6 +94,7 @@ let generate_durable_storage ~(plugin : (module Protocol_plugin_sig.S)) tree = let dump_durable_storage ~block ~data_dir ~file = let open Lwt_result_syntax in let* store = Store.init Read_only ~data_dir in + let store = Store.Normal store in let get name load = let* value = load () in match value with @@ -153,6 +154,7 @@ let patch_durable_storage ~data_dir ~key ~value = (* Loads the state of the head. *) let* _lock = Node_context_loader.lock ~data_dir in let* store = Store.init Read_write ~data_dir in + let store = Store.Normal store in let* ({header = {block_hash; level = block_level; _}; _} as l2_block) = let* r = Store.L2_blocks.find_head store in match r with -- GitLab From f7c3724c7d655f74bbb461e0fe5c282ee0e51a2a Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Wed, 9 Oct 2024 22:06:47 +0200 Subject: [PATCH 5/6] Rollup node/Store: add a command to migrate store manually --- .../main_smart_rollup_node.ml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/bin_smart_rollup_node/main_smart_rollup_node.ml b/src/bin_smart_rollup_node/main_smart_rollup_node.ml index 2deef106c157..0a8d63d0eee2 100644 --- a/src/bin_smart_rollup_node/main_smart_rollup_node.ml +++ b/src/bin_smart_rollup_node/main_smart_rollup_node.ml @@ -451,6 +451,32 @@ let patch_durable_storage = As a reminder, patching the state is an advanced and unsafe \ procedure.") +let migrate_store = + let open Tezos_clic in + command + ~group + ~desc:"Migrate the rollup node store to the latest supported version." + (args1 data_dir_arg) + (prefixes ["migrate"; "store"] @@ stop) + (fun data_dir _cctxt -> + let open Lwt_result_syntax in + let* metadata = Metadata.read_metadata_file ~dir:data_dir in + let*? metadata = + match metadata with + | None -> + error_with + "No metadata for the rollup node in %s. Is the rollup node \ + initialized?" + data_dir + | Some m -> Ok m + in + let* _fd = Node_context_loader.lock ~data_dir in + let* () = + let open Octez_smart_rollup_node_store in + Store_migration.maybe_run_migration metadata Store.version ~data_dir + in + return_unit) + let export_snapshot ( data_dir, dest, @@ -620,6 +646,7 @@ let sc_rollup_commands () = dump_metrics; dump_durable_storage; patch_durable_storage; + migrate_store; export_snapshot_auto_name; export_snapshot_named; import_snapshot; -- GitLab From 119388b1bf16462fc442a563d45ad78fbbb327a1 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Thu, 17 Oct 2024 14:07:33 +0200 Subject: [PATCH 6/6] Doc: changelog --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2ee10e2d6dfb..833aa9c6fc2f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -126,7 +126,7 @@ Smart Rollup node - Storage now uses SQLite as a backend instead of the custom indexed-file based store. This change makes the rollup node more robust but entails a migration of the store data. (MRs :gl:`!15053`, :gl:`!15026`, :gl:`!15059`, - :gl:`!15073`, :gl:`!15218`) + :gl:`!15073`, :gl:`!15218`, :gl:`!15257`) Smart Rollup WASM Debugger -------------------------- -- GitLab