diff --git a/src/bin_dal_node/main.ml b/src/bin_dal_node/main.ml index da755e5e0e63188ff5927538d7bb03251faff3e5..73d0c1911824842e9a39703b5c873e76643b33e7 100644 --- a/src/bin_dal_node/main.ml +++ b/src/bin_dal_node/main.ml @@ -109,8 +109,7 @@ let run subcommand cli_options = Lwt_main.run @@ wrap_with_error @@ Lwt_utils_unix.with_tempdir "store" @@ fun data_dir -> - let* store = Dal_store_sqlite3.init ~data_dir ~perm:`Read_write () in - let* schemas = Dal_store_sqlite3.(use store Schemas.get_all) in + let* schemas = Store.Skip_list_cells.schemas data_dir in let output = String.concat ";\n\n" schemas in Format.printf "%s\n" output ; return_unit diff --git a/src/bin_dal_node/store.ml b/src/bin_dal_node/store.ml index 5c04a8041b1c6e11a34de2b9660f954fbba9677c..da7723673eea62c6942e061e8b2f66cfff51e134 100644 --- a/src/bin_dal_node/store.ml +++ b/src/bin_dal_node/store.ml @@ -551,7 +551,7 @@ type t = { last_processed_level : Last_processed_level.t; first_seen_level : First_seen_level.t; storage_backend : Storage_backend.kind; - sqlite3 : Dal_store_sqlite3.t; + sqlite3 : Dal_store_sqlite3.Skip_list_cells.t; } let cache {cache; _} = cache @@ -573,6 +573,37 @@ let slot_header_statuses {slot_header_statuses; _} = slot_header_statuses let slots {slots; _} = slots +let init_sqlite_skip_list_cells_store ?(perm = `Read_write) data_dir = + let open Lwt_result_syntax in + let open Filename.Infix in + let skip_list_cells_data_dir = data_dir // Stores_dirs.skip_list_cells in + let*! () = + (* This occurs when running the command: + ./octez-dal-node debug print store schemas *) + if not (Sys.file_exists skip_list_cells_data_dir) then + Lwt_utils_unix.create_dir (data_dir // Stores_dirs.skip_list_cells) + else Lwt.return_unit + in + let*! () = + (* If a previous sqlite database has been created using the + `--sqlite3-backend` experimental flag. We simply move it to the + new destination path. *) + let previous_path = data_dir // Dal_store_sqlite3.sqlite_file_name in + if Sys.file_exists previous_path then ( + let new_path = + skip_list_cells_data_dir // Dal_store_sqlite3.sqlite_file_name + in + let*! () = Lwt_utils_unix.copy_file ~src:previous_path ~dst:new_path () in + Sys.remove previous_path ; + Lwt.return_unit) + else Lwt.return_unit + in + let*! () = Event.(emit dal_node_sqlite3_store_init ()) in + Dal_store_sqlite3.Skip_list_cells.init + ~data_dir:skip_list_cells_data_dir + ~perm + () + module Skip_list_cells = struct let find t = match t.storage_backend with @@ -588,6 +619,11 @@ module Skip_list_cells = struct match t.storage_backend with | Legacy -> Kvs_skip_list_cells_store.remove t.skip_list_cells | SQLite3 -> Dal_store_sqlite3.Skip_list_cells.remove t.sqlite3 + + let schemas data_dir = + let open Lwt_result_syntax in + let* store = init_sqlite_skip_list_cells_store data_dir in + Dal_store_sqlite3.Skip_list_cells.schemas store end let init_skip_list_cells_store base_dir = @@ -695,9 +731,7 @@ let upgrade_from_v1_to_v2 ~base_dir = in (* Initialize both stores and migrate. *) let* kvs_store = init_skip_list_cells_store base_dir in - let* sql_store = - Dal_store_sqlite3.init ~data_dir:base_dir ~perm:`Read_write () - in + let* sql_store = init_sqlite_skip_list_cells_store base_dir in let* storage_backend_store = Storage_backend.init ~root_dir:base_dir in let*! res = Store_migrations.migrate_skip_list_store kvs_store sql_store in let* () = Kvs_skip_list_cells_store.close kvs_store in @@ -711,10 +745,10 @@ let upgrade_from_v1_to_v2 ~base_dir = ~force:true in (* Remove the Stores_dirs.skip_list_cells directory. *) - let*! () = - Lwt_utils_unix.remove_dir - Filename.Infix.(base_dir // Stores_dirs.skip_list_cells) - in + let open Filename.Infix in + let store_dir = base_dir // Stores_dirs.skip_list_cells in + let*! () = Lwt_utils_unix.remove_dir (store_dir // "hashes") in + let*! () = Lwt_utils_unix.remove_dir (store_dir // "cells") in (* The storage upgrade has been done. *) let*! () = Event.(emit store_upgraded (Version.make 1, Version.make 2)) in return_unit @@ -798,10 +832,7 @@ let init config = let sqlite3_backend = config.experimental_features.sqlite3_backend in Storage_backend.set store ~sqlite3_backend in - let* sqlite3 = - let*! () = Event.(emit dal_node_sqlite3_store_init ()) in - Dal_store_sqlite3.init ~data_dir:base_dir ~perm:`Read_write () - in + let* sqlite3 = init_sqlite_skip_list_cells_store base_dir in let*! () = Event.(emit store_is_ready ()) in return { diff --git a/src/bin_dal_node/store.mli b/src/bin_dal_node/store.mli index 0af3b9ddd408b2ec36a087bf98ed290c82fe8335..9f077b61e96c7b5295ab86f513d478aea6980136 100644 --- a/src/bin_dal_node/store.mli +++ b/src/bin_dal_node/store.mli @@ -164,7 +164,9 @@ val shards : t -> Shards.t [`SQLite3 store] depending on the current storage backend used by the store [t]. *) val skip_list_cells : - t -> [> `KVS of Kvs_skip_list_cells_store.t | `SQLite3 of Dal_store_sqlite3.t] + t -> + [> `KVS of Kvs_skip_list_cells_store.t + | `SQLite3 of Dal_store_sqlite3.Skip_list_cells.t ] (** [slot_header_statuses t] returns the statuses store associated with the store [t]. *) @@ -228,4 +230,8 @@ module Skip_list_cells : sig (** [remove store ~attested_level] removes any data related to [attested_level] from the [store]. *) val remove : t -> attested_level:int32 -> unit tzresult Lwt.t + + (** [schemas data_dir] returns the list of SQL statements allowing + to recreate the tables of the DAL skip list cells store. *) + val schemas : string -> string list tzresult Lwt.t end diff --git a/src/lib_dal_node/dal_plugin.ml b/src/lib_dal_node/dal_plugin.ml index 90c78b50f00d61cccfdc169d5d594468e1c29fba..02c07dad7cc412adaf5af603fce92e981e243291 100644 --- a/src/lib_dal_node/dal_plugin.ml +++ b/src/lib_dal_node/dal_plugin.ml @@ -174,7 +174,8 @@ module type T = sig module RPC : sig val directory : - [< `KVS of Kvs_skip_list_cells_store.t | `SQLite3 of Dal_store_sqlite3.t] -> + [< `KVS of Kvs_skip_list_cells_store.t + | `SQLite3 of Dal_store_sqlite3.Skip_list_cells.t ] -> unit Tezos_rpc.Directory.t end end diff --git a/src/lib_dal_node/dal_plugin.mli b/src/lib_dal_node/dal_plugin.mli index 92e4c4702f6680071c8238c5b09eae86a3c4b18d..fc2fdff12beef9373e3d683c0f583a8dba362294 100644 --- a/src/lib_dal_node/dal_plugin.mli +++ b/src/lib_dal_node/dal_plugin.mli @@ -164,7 +164,8 @@ module type T = sig module RPC : sig (** RPCs directory of the protocol-related part of the DAL node. *) val directory : - [< `KVS of Kvs_skip_list_cells_store.t | `SQLite3 of Dal_store_sqlite3.t] -> + [< `KVS of Kvs_skip_list_cells_store.t + | `SQLite3 of Dal_store_sqlite3.Skip_list_cells.t ] -> unit Tezos_rpc.Directory.t end end diff --git a/src/lib_dal_node/dal_store_sqlite3.ml b/src/lib_dal_node/dal_store_sqlite3.ml index da1069fab6e563f0b0a747adb28422487f806bd1..f923acc78f7c5f310777780e09e3d3e52445b38b 100644 --- a/src/lib_dal_node/dal_store_sqlite3.ml +++ b/src/lib_dal_node/dal_store_sqlite3.ml @@ -7,10 +7,14 @@ (*****************************************************************************) open Filename.Infix -include Sqlite +open Sqlite open Caqti_request.Infix open Caqti_type.Std +let sqlite_file_name = "store.sqlite" + +type conn = Sqlite.conn + module Q = struct let table_exists = (string ->! bool) @@ -151,6 +155,8 @@ let with_connection store conn = fun k -> Sqlite.use store @@ fun conn -> Sqlite.with_connection conn k module Skip_list_cells = struct + type nonrec t = t + open Types module Q = struct @@ -228,9 +234,54 @@ module Skip_list_cells = struct Sqlite.Db.exec conn Q.insert_skip_list_cell (cell_hash, cell)) items - module Internal_for_tests = struct - open Types + let init ~data_dir ~perm () = + let open Lwt_result_syntax in + let path = data_dir // sqlite_file_name in + let*! exists = Lwt_unix.file_exists path in + let migration conn = + Sqlite.assert_in_transaction conn ; + let* () = + if not exists then + let* () = Migrations.create_table conn in + return_unit + else + let* table_exists = Migrations.table_exists conn in + let* () = + when_ (not table_exists) (fun () -> + failwith "A store already exists, but its content is incorrect.") + in + return_unit + in + let* migrations = Migrations.missing_migrations conn in + let*? () = + match (perm, migrations) with + | `Read_only, _ :: _ -> + error_with + "The store has %d missing migrations but was opened in read-only \ + mode." + (List.length migrations) + | _, _ -> Ok () + in + let* () = + List.iter_es + (fun (i, ((module M : Dal_node_migrations.S) as mig)) -> + let start_time = Unix.gettimeofday () in + let* () = Migrations.apply_migration conn i mig in + let end_time = Unix.gettimeofday () in + let duration = end_time -. start_time in + let*! () = + Dal_store_sqlite3_events.applied_migration ~name:M.name ~duration + in + return_unit) + migrations + in + return_unit + in + Sqlite.init ~path ~perm migration + let schemas t = use t Schemas.get_all + + module Internal_for_tests = struct module Q = struct open Dal_proto_types @@ -250,50 +301,3 @@ module Skip_list_cells = struct Sqlite.Db.find conn Q.skip_list_hash_exists skip_list_hash end end - -let sqlite_file_name = "store.sqlite" - -let init ~data_dir ~perm () = - let open Lwt_result_syntax in - let path = data_dir // sqlite_file_name in - let*! exists = Lwt_unix.file_exists path in - let migration conn = - Sqlite.assert_in_transaction conn ; - let* () = - if not exists then - let* () = Migrations.create_table conn in - return_unit - else - let* table_exists = Migrations.table_exists conn in - let* () = - when_ (not table_exists) (fun () -> - failwith "A store already exists, but its content is incorrect.") - in - return_unit - in - let* migrations = Migrations.missing_migrations conn in - let*? () = - match (perm, migrations) with - | `Read_only, _ :: _ -> - error_with - "The store has %d missing migrations but was opened in read-only \ - mode." - (List.length migrations) - | _, _ -> Ok () - in - let* () = - List.iter_es - (fun (i, ((module M : Dal_node_migrations.S) as mig)) -> - let start_time = Unix.gettimeofday () in - let* () = Migrations.apply_migration conn i mig in - let end_time = Unix.gettimeofday () in - let duration = end_time -. start_time in - let*! () = - Dal_store_sqlite3_events.applied_migration ~name:M.name ~duration - in - return_unit) - migrations - in - return_unit - in - Sqlite.init ~path ~perm migration diff --git a/src/lib_dal_node/dal_store_sqlite3.mli b/src/lib_dal_node/dal_store_sqlite3.mli index d84f8630c826142a385a61fd9a0a93bfc26f5c99..47a7178d50ec20487183d23a7213e5ab7b7879fc 100644 --- a/src/lib_dal_node/dal_store_sqlite3.mli +++ b/src/lib_dal_node/dal_store_sqlite3.mli @@ -6,27 +6,12 @@ (* *) (*****************************************************************************) -(** A handler to the DAL node's SQLite3 database. *) -type t = Sqlite.t +(** Name of the SQLite3 file. *) +val sqlite_file_name : string (** A direct connection to the database, allowing to interact with it. *) type conn = Sqlite.conn -(** [use db k] executes [k] with a fresh connection to [db]. *) -val use : t -> (conn -> 'a tzresult Lwt.t) -> 'a tzresult Lwt.t - -(** [init ~data_dir ~perm ()] returns a handler to the DAL node store - located under [data_dir]. If no store is located in [data_dir], an - empty store is created. - - If [perm] is [`Read_only], then SQL requests requiring write access will - fail. With [`Read_write], they will succeed as expected. *) -val init : - data_dir:string -> perm:[`Read_only | `Read_write] -> unit -> t tzresult Lwt.t - -(** Name of the SQLite3 file. *) -val sqlite_file_name : string - module Schemas : sig (** [get_all conn] returns the list of SQL statements allowing to recreate the tables in the current store. *) @@ -36,6 +21,21 @@ end module Skip_list_cells : sig open Dal_proto_types + (** A handler to the DAL node's skip list cells SQLite3 database. *) + type t + + (** [init ~data_dir ~perm ()] returns a handler to the DAL node store + located under [data_dir]. If no store is located in [data_dir], an + empty store is created. + + If [perm] is [`Read_only], then SQL requests requiring write access will + fail. With [`Read_write], they will succeed as expected. *) + val init : + data_dir:string -> + perm:[`Read_only | `Read_write] -> + unit -> + t tzresult Lwt.t + (** [find ?conn store hash] returns the cell associated to [hash] in the [store], if any. Uses the [conn] if provided (defaults to [None]). *) @@ -58,6 +58,10 @@ module Skip_list_cells : sig (defaults to [None]). *) val remove : ?conn:conn -> t -> attested_level:int32 -> unit tzresult Lwt.t + (** [schemas t] returns the list of SQL statements + allowing to recreate the tables of the store [t]. *) + val schemas : t -> string list tzresult Lwt.t + (** Internal functions for testing purpose. *) module Internal_for_tests : sig val skip_list_hash_exists : diff --git a/src/lib_dal_node/store_migrations.mli b/src/lib_dal_node/store_migrations.mli index 65ac43a673a3666e9b4244591f78f99538ce04d5..c497fe85f98dceef3ab5948a46bd92f0559eabca 100644 --- a/src/lib_dal_node/store_migrations.mli +++ b/src/lib_dal_node/store_migrations.mli @@ -10,4 +10,6 @@ [kvs_store] and inserting the corresponding payload in the [sql_store]. *) val migrate_skip_list_store : - Kvs_skip_list_cells_store.t -> Dal_store_sqlite3.t -> unit tzresult Lwt.t + Kvs_skip_list_cells_store.t -> + Dal_store_sqlite3.Skip_list_cells.t -> + unit tzresult Lwt.t diff --git a/src/lib_dal_node/test/test_storage.ml b/src/lib_dal_node/test/test_storage.ml index 32a95f9dde0afdafdec2dfcc3c3362922370ab93..8bac24ea4934986c7ea6f12b8a604acc3bed67ae 100644 --- a/src/lib_dal_node/test/test_storage.ml +++ b/src/lib_dal_node/test/test_storage.ml @@ -372,6 +372,12 @@ let init_skip_list_cells_store node_store_dir = ~padded_encoded_cell_size ~encoded_hash_size +let init_sqlite_skip_list_cells_store ?(perm = `Read_write) data_dir = + Dal_store_sqlite3.Skip_list_cells.init + ~data_dir:(Filename.concat data_dir "skip_list_store") + ~perm + () + (* Required by the KVS store. *) let () = Value_size_hooks.set_number_of_slots number_of_slots @@ -386,7 +392,7 @@ let consistency_test = (* Initialize the KVS and the SQLite stores. *) let data_dir = Tezt.Temp.dir "data-dir" in let* kvs_store = init_skip_list_cells_store data_dir in - let* sql_store = Dal_store_sqlite3.init ~data_dir ~perm:`Read_write () in + let* sql_store = init_sqlite_skip_list_cells_store data_dir in (* Run the actions both for the KVS and the SQL backends. *) let* () = run_kvs kvs_store actions in let* () = run_sqlite3 sql_store actions in @@ -426,7 +432,7 @@ let migration_skip_list_test {state; actions} = let data_dir = Tezt.Temp.dir "data-dir" in let* kvs_store = init_skip_list_cells_store data_dir in let* () = run_kvs kvs_store actions in - let* sql_store = Dal_store_sqlite3.init ~data_dir ~perm:`Read_write () in + let* sql_store = init_sqlite_skip_list_cells_store data_dir in let* () = Store_migrations.migrate_skip_list_store kvs_store sql_store in let* () = handshake state kvs_store sql_store in Kvs_skip_list_cells_store.close kvs_store diff --git a/src/proto_020_PsParisC/lib_dal/RPC_directory.mli b/src/proto_020_PsParisC/lib_dal/RPC_directory.mli index 35f51811cd617ee7e83a04df6aa7590ff2daa473..6dab8ab83655dfeddb12c11096e00428d9b8da24 100644 --- a/src/proto_020_PsParisC/lib_dal/RPC_directory.mli +++ b/src/proto_020_PsParisC/lib_dal/RPC_directory.mli @@ -8,5 +8,6 @@ (** The RPCs directory of the protocol part of DAL nodes. *) val directory : - [< `KVS of Kvs_skip_list_cells_store.t | `SQLite3 of Dal_store_sqlite3.t] -> + [< `KVS of Kvs_skip_list_cells_store.t + | `SQLite3 of Dal_store_sqlite3.Skip_list_cells.t ] -> unit Environment.RPC_directory.t diff --git a/src/proto_021_PsQuebec/lib_dal/RPC_directory.mli b/src/proto_021_PsQuebec/lib_dal/RPC_directory.mli index 35f51811cd617ee7e83a04df6aa7590ff2daa473..6dab8ab83655dfeddb12c11096e00428d9b8da24 100644 --- a/src/proto_021_PsQuebec/lib_dal/RPC_directory.mli +++ b/src/proto_021_PsQuebec/lib_dal/RPC_directory.mli @@ -8,5 +8,6 @@ (** The RPCs directory of the protocol part of DAL nodes. *) val directory : - [< `KVS of Kvs_skip_list_cells_store.t | `SQLite3 of Dal_store_sqlite3.t] -> + [< `KVS of Kvs_skip_list_cells_store.t + | `SQLite3 of Dal_store_sqlite3.Skip_list_cells.t ] -> unit Environment.RPC_directory.t diff --git a/src/proto_alpha/lib_dal/RPC_directory.mli b/src/proto_alpha/lib_dal/RPC_directory.mli index 35f51811cd617ee7e83a04df6aa7590ff2daa473..6dab8ab83655dfeddb12c11096e00428d9b8da24 100644 --- a/src/proto_alpha/lib_dal/RPC_directory.mli +++ b/src/proto_alpha/lib_dal/RPC_directory.mli @@ -8,5 +8,6 @@ (** The RPCs directory of the protocol part of DAL nodes. *) val directory : - [< `KVS of Kvs_skip_list_cells_store.t | `SQLite3 of Dal_store_sqlite3.t] -> + [< `KVS of Kvs_skip_list_cells_store.t + | `SQLite3 of Dal_store_sqlite3.Skip_list_cells.t ] -> unit Environment.RPC_directory.t