From e3bb0600c3ac18ad2077f7240be906416d28467f Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Wed, 24 Apr 2024 13:09:33 +0100 Subject: [PATCH 1/2] Store: Remove version upgrades before 3.0 --- src/lib_node_config/data_version.ml | 49 +++-------------------------- src/lib_store/mocked/store.ml | 3 +- src/lib_store/store.mli | 3 +- src/lib_store/unix/store.ml | 8 ++--- src/lib_store/unix/store.mli | 3 +- 5 files changed, 9 insertions(+), 57 deletions(-) diff --git a/src/lib_node_config/data_version.ml b/src/lib_node_config/data_version.ml index 5f05bc0c6295..9c3cc2323bda 100644 --- a/src/lib_node_config/data_version.ml +++ b/src/lib_node_config/data_version.ml @@ -107,16 +107,6 @@ end (* FIXME https://gitlab.com/tezos/tezos/-/issues/2861 We should enable the semantic versioning instead of applying hardcoded rules.*) -let v_0_6 = Version.make ~major:0 ~minor:6 - -let v_0_7 = Version.make ~major:0 ~minor:7 - -let v_0_8 = Version.make ~major:0 ~minor:8 - -let v_1_0 = Version.make ~major:1 ~minor:0 - -let v_2_0 = Version.make ~major:2 ~minor:0 - let v_3_0 = Version.make ~major:3 ~minor:0 let v_3_1 = Version.make ~major:3 ~minor:1 @@ -135,41 +125,14 @@ let current_version = v_3_1 - we want to deprecate a specific upgrade *) let upgradable_data_version = - let open Lwt_result_syntax in - let v_1_0_upgrade ~data_dir = - let context_root = context_dir data_dir in - (* The upgrade function consist in letting irmin doing its own - file renaming. To do so, it must be done using a RW instance.*) - let*! ctxt = Context.init ~readonly:false context_root in - let*! () = Context.close ctxt in - return_unit - in - let v_3_1_upgrade ~data_dir genesis ~upgrade_to_v3 = + let v_3_1_upgrade ~data_dir genesis = let store_dir = store_dir data_dir in - Store.v_3_1_upgrade ~store_dir genesis ~upgrade_to_v3 + Store.v_3_1_upgrade ~store_dir genesis in [ - ( v_0_6, - fun ~data_dir genesis ~chain_name:_ ~sandbox_parameters:_ -> - let* () = v_1_0_upgrade ~data_dir in - v_3_1_upgrade ~data_dir genesis ~upgrade_to_v3:true ); - ( v_0_7, - fun ~data_dir genesis ~chain_name:_ ~sandbox_parameters:_ -> - let* () = v_1_0_upgrade ~data_dir in - v_3_1_upgrade ~data_dir genesis ~upgrade_to_v3:true ); - ( v_0_8, - fun ~data_dir genesis ~chain_name:_ ~sandbox_parameters:_ -> - let* () = v_1_0_upgrade ~data_dir in - v_3_1_upgrade ~data_dir genesis ~upgrade_to_v3:true ); - ( v_1_0, - fun ~data_dir genesis ~chain_name:_ ~sandbox_parameters:_ -> - v_3_1_upgrade ~data_dir genesis ~upgrade_to_v3:true ); - ( v_2_0, - fun ~data_dir genesis ~chain_name:_ ~sandbox_parameters:_ -> - v_3_1_upgrade ~data_dir genesis ~upgrade_to_v3:true ); ( v_3_0, fun ~data_dir genesis ~chain_name:_ ~sandbox_parameters:_ -> - v_3_1_upgrade ~data_dir genesis ~upgrade_to_v3:false ); + v_3_1_upgrade ~data_dir genesis ); ] type error += Invalid_data_dir_version of Version.t * Version.t @@ -460,11 +423,7 @@ let ensure_data_dir ?(mode = Is_compatible) genesis data_dir = | None -> return_unit (* Enable automatic upgrade to avoid users to manually upgrade. *) - | Some (version, _) - when Version.( - equal version v_3_0 || equal version v_2_0 || equal version v_1_0 - || equal version v_0_6 || equal version v_0_7 || equal version v_0_8) - -> + | Some (version, _) when Version.(equal version v_3_0) -> let* () = upgrade_data_dir ~data_dir genesis ~chain_name:() ~sandbox_parameters:() in diff --git a/src/lib_store/mocked/store.ml b/src/lib_store/mocked/store.ml index 97c38702a887..7bad0760b5fc 100644 --- a/src/lib_store/mocked/store.ml +++ b/src/lib_store/mocked/store.ml @@ -2078,5 +2078,4 @@ end let v_3_0_upgrade ~store_dir:_ _genesis = Lwt_result_syntax.return_unit -let v_3_1_upgrade ~store_dir:_ _genesis ~upgrade_to_v3:_ = - Lwt_result_syntax.return_unit +let v_3_1_upgrade ~store_dir:_ _genesis = Lwt_result_syntax.return_unit diff --git a/src/lib_store/store.mli b/src/lib_store/store.mli index 894c4666fda6..8e3c63e01210 100644 --- a/src/lib_store/store.mli +++ b/src/lib_store/store.mli @@ -1021,8 +1021,7 @@ end val v_3_0_upgrade : store_dir:string -> Genesis.t -> unit tzresult Lwt.t -val v_3_1_upgrade : - store_dir:string -> Genesis.t -> upgrade_to_v3:bool -> unit tzresult Lwt.t +val v_3_1_upgrade : store_dir:string -> Genesis.t -> unit tzresult Lwt.t (**/**) diff --git a/src/lib_store/unix/store.ml b/src/lib_store/unix/store.ml index f3c3de7aa97d..3e0cff3c877e 100644 --- a/src/lib_store/unix/store.ml +++ b/src/lib_store/unix/store.ml @@ -3246,12 +3246,8 @@ let v_3_0_upgrade ~store_dir genesis = let*! () = List.iter_s (fun f -> f ()) !finalizers in return_unit) -let v_3_1_upgrade ~store_dir genesis ~upgrade_to_v3 = - let open Lwt_result_syntax in - let* () = - if upgrade_to_v3 then v_3_0_upgrade ~store_dir genesis else return_unit - in - (* Hypothesis: v_3_0_upgrade has been run before or is run here *) +let v_3_1_upgrade ~store_dir genesis = + (* Hypothesis: The node storage version is 3.0 *) let chain_id = Chain_id.of_block_hash genesis.Genesis.block in let chain_dir = Naming.chain_dir (Naming.store_dir ~dir_path:store_dir) chain_id diff --git a/src/lib_store/unix/store.mli b/src/lib_store/unix/store.mli index 22bf36322b17..7a4e8770f6c1 100644 --- a/src/lib_store/unix/store.mli +++ b/src/lib_store/unix/store.mli @@ -1020,8 +1020,7 @@ end val v_3_0_upgrade : store_dir:string -> Genesis.t -> unit tzresult Lwt.t (** Potentially upgrade to v_3 and then upgrade the block_store_status in v_3_1. *) -val v_3_1_upgrade : - store_dir:string -> Genesis.t -> upgrade_to_v3:bool -> unit tzresult Lwt.t +val v_3_1_upgrade : store_dir:string -> Genesis.t -> unit tzresult Lwt.t (**/**) -- GitLab From fe8c908669816e4da6e8f42041aef87e8b7ac565 Mon Sep 17 00:00:00 2001 From: Gabriel Moise Date: Wed, 24 Apr 2024 13:42:05 +0100 Subject: [PATCH 2/2] Store: Remove version upgrade 3.0 --- src/lib_shell_services/store_errors.ml | 37 +------- src/lib_store/mocked/store.ml | 2 - src/lib_store/store.mli | 2 - src/lib_store/unix/block_store.ml | 121 +------------------------ src/lib_store/unix/block_store.mli | 11 --- src/lib_store/unix/store.ml | 82 ----------------- src/lib_store/unix/store.mli | 6 -- 7 files changed, 3 insertions(+), 258 deletions(-) diff --git a/src/lib_shell_services/store_errors.ml b/src/lib_shell_services/store_errors.ml index eefed9183f8d..8ff79f3f8275 100644 --- a/src/lib_shell_services/store_errors.ml +++ b/src/lib_shell_services/store_errors.ml @@ -1326,13 +1326,7 @@ let () = (fun k -> Corrupted_store k) (* Store upgrade errors *) -type error += - | Cannot_find_chain_dir of string - | V_3_0_upgrade_missing_floating_block of { - block_hash : Block_hash.t; - block_level : Int32.t; - floating_kind : string; - } +type error += Cannot_find_chain_dir of string let () = Error_monad.register_error_kind @@ -1349,31 +1343,4 @@ let () = path) Data_encoding.(obj1 (req "path" string)) (function Cannot_find_chain_dir p -> Some p | _ -> None) - (fun p -> Cannot_find_chain_dir p) ; - register_error_kind - `Permanent - ~id:"block_store.v_3_0_upgrade_missing_floating_block" - ~title:"V.3.0 upgrade missing floating block" - ~description:"Failed to upgrade the floating store" - ~pp:(fun ppf (block_hash, block_level, floating_kind) -> - Format.fprintf - ppf - "Failed to upgrade block %a (level %ld) for %s floating store: block \ - not found in the index." - Block_hash.pp - block_hash - block_level - floating_kind) - Data_encoding.( - obj3 - (req "block_hash" Block_hash.encoding) - (req "block_level" int32) - (req "floating_kind" string)) - (function - | V_3_0_upgrade_missing_floating_block - {block_hash; block_level; floating_kind} -> - Some (block_hash, block_level, floating_kind) - | _ -> None) - (fun (block_hash, block_level, floating_kind) -> - V_3_0_upgrade_missing_floating_block - {block_hash; block_level; floating_kind}) + (fun p -> Cannot_find_chain_dir p) diff --git a/src/lib_store/mocked/store.ml b/src/lib_store/mocked/store.ml index 7bad0760b5fc..5157e186f438 100644 --- a/src/lib_store/mocked/store.ml +++ b/src/lib_store/mocked/store.ml @@ -2076,6 +2076,4 @@ module Unsafe = struct let block_of_repr = Fun.id end -let v_3_0_upgrade ~store_dir:_ _genesis = Lwt_result_syntax.return_unit - let v_3_1_upgrade ~store_dir:_ _genesis = Lwt_result_syntax.return_unit diff --git a/src/lib_store/store.mli b/src/lib_store/store.mli index 8e3c63e01210..28a9f942daaa 100644 --- a/src/lib_store/store.mli +++ b/src/lib_store/store.mli @@ -1019,8 +1019,6 @@ module Chain_traversal : sig (Block.t * Block.t list) Lwt.t end -val v_3_0_upgrade : store_dir:string -> Genesis.t -> unit tzresult Lwt.t - val v_3_1_upgrade : store_dir:string -> Genesis.t -> unit tzresult Lwt.t (**/**) diff --git a/src/lib_store/unix/block_store.ml b/src/lib_store/unix/block_store.ml index f68a9199af87..10599de44ac1 100644 --- a/src/lib_store/unix/block_store.ml +++ b/src/lib_store/unix/block_store.ml @@ -1712,126 +1712,7 @@ let close block_store = Floating_block_store.close (block_store.rw_floating_block_store :: block_store.ro_floating_block_stores) -(***************** Upgrade to V3 *****************) - -let v_3_0_upgrade chain_dir ~cleanups ~finalizers = - let open Lwt_result_syntax in - let get_floating_paths kind = - let legacy_floating_blocks_dir = - Naming.floating_blocks_dir chain_dir kind - in - let legacy_floating_index_dir = - Naming.dir_path - (Naming.floating_blocks_index_dir legacy_floating_blocks_dir) - in - let legacy_floating_blocks_file = - Naming.floating_blocks_file legacy_floating_blocks_dir - in - let new_floating_index_dir = - Naming.dir_path - (Naming.floating_blocks_index_dir legacy_floating_blocks_dir) - ^ ".new" - in - ( Naming.dir_path legacy_floating_blocks_dir, - legacy_floating_index_dir, - legacy_floating_blocks_file, - new_floating_index_dir ) - in - let all_kinds = Naming.[RO; RW; RW_TMP; RO_TMP] in - let upgrade_floating_index kind = - let ( legacy_floating_blocks_dir, - legacy_floating_index_dir, - legacy_floating_blocks_file, - new_floating_index_dir ) = - get_floating_paths kind - in - let*! should_upgrade = Lwt_unix.file_exists legacy_floating_blocks_dir in - if not should_upgrade then return_unit - else - let clean_failed_upgrade () = - let*! exists = Lwt_unix.file_exists new_floating_index_dir in - if exists then Lwt_utils_unix.remove_dir new_floating_index_dir - else Lwt.return_unit - in - let finalize () = - let*! exists = Lwt_unix.file_exists new_floating_index_dir in - if exists then - let*! () = Lwt_utils_unix.remove_dir legacy_floating_index_dir in - Lwt_unix.rename new_floating_index_dir legacy_floating_index_dir - else Lwt.return_unit - in - finalizers := finalize :: !finalizers ; - cleanups := clean_failed_upgrade :: !cleanups ; - let legacy_index = - Floating_block_index.Legacy.v - ~log_size:Floating_block_store.default_floating_blocks_log_size - ~readonly:true - legacy_floating_index_dir - in - let new_index = - Floating_block_index.v - ~log_size:Floating_block_store.default_floating_blocks_log_size - ~readonly:false - new_floating_index_dir - in - let*! fd = - Lwt_unix.openfile - (Naming.file_path legacy_floating_blocks_file) - [Unix.O_CLOEXEC; Unix.O_RDONLY] - 0o444 - in - Lwt.finalize - (fun () -> - (* Iterate over the existing stores and retrieve their context hash. *) - let* () = - Floating_block_store.raw_iterate_fd - (fun (block_b, _len) -> - let block_hash = Block_repr_unix.raw_get_block_hash block_b in - let block_context = Block_repr_unix.raw_get_context block_b in - let* { - Floating_block_index.Legacy.Legacy_block_info.offset; - predecessors; - } = - try - return - @@ Floating_block_index.Legacy.find legacy_index block_hash - with - | Not_found -> - let block_level = - Block_repr_unix.raw_get_block_level block_b - in - let floating_kind = - (function - | Naming.RO -> "RO" - | RW -> "RW" - | RO_TMP -> "RO_TMP" - | RW_TMP -> "RW_TMP" - | Restore _ -> "Restored") - kind - in - tzfail - (V_3_0_upgrade_missing_floating_block - {block_hash; block_level; floating_kind}) - | e -> raise e - in - let resulting_context_hash = block_context in - let new_value = - Floating_block_index.Block_info. - {offset; predecessors; resulting_context_hash} - in - Floating_block_index.replace new_index block_hash new_value ; - return_unit) - fd - in - return_unit) - (fun () -> - Floating_block_index.flush new_index ; - Floating_block_index.close new_index ; - Floating_block_index.Legacy.close legacy_index ; - let*! () = Lwt_unix.close fd in - Lwt.return_unit) - in - protect (fun () -> List.iter_es upgrade_floating_index all_kinds) +(***************** Upgrade to V3.1 *****************) let v_3_1_upgrade chain_dir = let open Lwt_result_syntax in diff --git a/src/lib_store/unix/block_store.mli b/src/lib_store/unix/block_store.mli index f1b0172d57c8..af6d17d6d685 100644 --- a/src/lib_store/unix/block_store.mli +++ b/src/lib_store/unix/block_store.mli @@ -374,16 +374,5 @@ val close : block_store -> unit Lwt.t [block_store] where the merge procedure was interrupted. *) val may_recover_merge : block_store -> unit tzresult Lwt.t -(** Upgrade a v_2 to v_3 block store by retrieving - [resulting_context_hash] of all blocks present in the floating - stores and updating their index. - - {b Warning} Not backward-compatible. *) -val v_3_0_upgrade : - [`Chain_dir] Naming.directory -> - cleanups:(unit -> unit Lwt.t) list ref -> - finalizers:(unit -> unit Lwt.t) list ref -> - unit tzresult Lwt.t - (** Upgrade the block_store_status *) val v_3_1_upgrade : [`Chain_dir] Naming.directory -> unit tzresult Lwt.t diff --git a/src/lib_store/unix/store.ml b/src/lib_store/unix/store.ml index 3e0cff3c877e..b71c90a7ecff 100644 --- a/src/lib_store/unix/store.ml +++ b/src/lib_store/unix/store.ml @@ -3164,88 +3164,6 @@ let make_pp_store (store : store) = pp_testchain_store ()) -let upgrade_protocol_levels ~chain_dir ~cleanups ~finalizers = - let open Lwt_result_syntax in - let cleanup ~tmp_protocol_levels_path ~protocol_levels_path = - let*! exists = Lwt_unix.file_exists tmp_protocol_levels_path in - if exists then Lwt_unix.rename tmp_protocol_levels_path protocol_levels_path - else Lwt.return_unit - in - let protocol_levels_path = - Naming.legacy_protocol_levels_file chain_dir |> Naming.encoded_file_path - in - let tmp_protocol_levels_path = protocol_levels_path ^ ".tmp" in - let*! () = cleanup ~tmp_protocol_levels_path ~protocol_levels_path in - let* legacy_protocol_levels_data = - Stored_data.load (Naming.legacy_protocol_levels_file chain_dir) - in - let*! legacy_protocol_levels = Stored_data.get legacy_protocol_levels_data in - let bindings = Protocol_levels.Legacy.bindings legacy_protocol_levels in - let*! protocol_levels = - List.fold_left_s - (fun map - ( level, - (legacy_activation_block : Protocol_levels.Legacy.activation_block) - ) -> - let protocol_info = - Protocol_levels. - { - protocol = legacy_activation_block.protocol; - activation_block = legacy_activation_block.block; - expect_predecessor_context = - false - (* the shell cannot have stored blocks that use the new semantics *); - } - in - Lwt.return (Protocol_levels.add level protocol_info map)) - Protocol_levels.empty - bindings - in - cleanups := - (fun () -> cleanup ~tmp_protocol_levels_path ~protocol_levels_path) - :: !cleanups ; - finalizers := - (fun () -> - let*! () = - Lwt_unix.rename protocol_levels_path tmp_protocol_levels_path - in - let*! _unit_error = - Stored_data.write_file - (Naming.protocol_levels_file chain_dir) - protocol_levels - in - let*! () = Lwt_unix.unlink tmp_protocol_levels_path in - Lwt.return_unit) - :: !finalizers ; - return_unit - -let v_3_0_upgrade ~store_dir genesis = - let open Lwt_result_syntax in - (* Hypothesis: all present blocks were validated with the - preexisting semantics *) - let*! () = Store_events.(emit upgrade_store_started ()) in - let cleanups : (unit -> unit Lwt.t) list ref = ref [] in - let finalizers : (unit -> unit Lwt.t) list ref = ref [] in - let chain_id = Chain_id.of_block_hash genesis.Genesis.block in - let chain_dir = - Naming.chain_dir (Naming.store_dir ~dir_path:store_dir) chain_id - in - protect - ~on_error:(fun err -> - let*! () = Store_events.(emit upgrade_store_failed) () in - let*! () = List.iter_s (fun f -> f ()) !cleanups in - Lwt.return_error err) - (fun () -> - let* () = - let chain_dir_path = Naming.dir_path chain_dir in - let*! chain_dir_exists = Lwt_unix.file_exists chain_dir_path in - fail_unless chain_dir_exists (Cannot_find_chain_dir chain_dir_path) - in - let* () = upgrade_protocol_levels ~chain_dir ~cleanups ~finalizers in - let* () = Block_store.v_3_0_upgrade chain_dir ~cleanups ~finalizers in - let*! () = List.iter_s (fun f -> f ()) !finalizers in - return_unit) - let v_3_1_upgrade ~store_dir genesis = (* Hypothesis: The node storage version is 3.0 *) let chain_id = Chain_id.of_block_hash genesis.Genesis.block in diff --git a/src/lib_store/unix/store.mli b/src/lib_store/unix/store.mli index 7a4e8770f6c1..c36fe7eda230 100644 --- a/src/lib_store/unix/store.mli +++ b/src/lib_store/unix/store.mli @@ -1013,12 +1013,6 @@ module Chain_traversal : sig (Block.t * Block.t list) Lwt.t end -(** Upgrade a v_2 to v_3 store by rewriting the block store and the - protocol level's table. - - {b Warning} Not backward-compatible. *) -val v_3_0_upgrade : store_dir:string -> Genesis.t -> unit tzresult Lwt.t - (** Potentially upgrade to v_3 and then upgrade the block_store_status in v_3_1. *) val v_3_1_upgrade : store_dir:string -> Genesis.t -> unit tzresult Lwt.t -- GitLab