From 3de16cb866bb0849d8dec70c7929270d4cc7166e Mon Sep 17 00:00:00 2001 From: vbot Date: Wed, 2 Nov 2022 13:19:42 +0100 Subject: [PATCH 01/19] Proto/Env: introduce expected_context_hash --- src/lib_protocol_environment/environment_V0.ml | 2 ++ src/lib_protocol_environment/environment_V1.ml | 2 ++ src/lib_protocol_environment/environment_V2.ml | 2 ++ src/lib_protocol_environment/environment_V3.ml | 2 ++ src/lib_protocol_environment/environment_V4.ml | 2 ++ src/lib_protocol_environment/environment_V5.ml | 2 ++ src/lib_protocol_environment/environment_V6.ml | 2 ++ src/lib_protocol_environment/environment_V7.ml | 2 ++ src/lib_protocol_environment/environment_V8.ml | 2 ++ src/lib_protocol_environment/environment_context.ml | 4 ++++ src/lib_protocol_environment/environment_context.mli | 10 ++++++++++ src/lib_protocol_environment/environment_protocol_T.ml | 2 ++ .../environment_protocol_T_test.ml | 2 ++ 13 files changed, 36 insertions(+) diff --git a/src/lib_protocol_environment/environment_V0.ml b/src/lib_protocol_environment/environment_V0.ml index e5003dab2401..cf82f789c24b 100644 --- a/src/lib_protocol_environment/environment_V0.ml +++ b/src/lib_protocol_environment/environment_V0.ml @@ -834,6 +834,8 @@ struct let set_log_message_consumer _ = () let environment_version = Protocol.V0 + + let expected_context_hash = Resulting_context end class ['chain, 'block] proto_rpc_context (t : Tezos_rpc.Context.t) diff --git a/src/lib_protocol_environment/environment_V1.ml b/src/lib_protocol_environment/environment_V1.ml index 5edfb7f2b71f..7fb677cebec6 100644 --- a/src/lib_protocol_environment/environment_V1.ml +++ b/src/lib_protocol_environment/environment_V1.ml @@ -964,6 +964,8 @@ struct let set_log_message_consumer _ = () let environment_version = Protocol.V1 + + let expected_context_hash = Resulting_context end class ['chain, 'block] proto_rpc_context (t : Tezos_rpc.Context.t) diff --git a/src/lib_protocol_environment/environment_V2.ml b/src/lib_protocol_environment/environment_V2.ml index fa29c2c4cec5..d73776da6daa 100644 --- a/src/lib_protocol_environment/environment_V2.ml +++ b/src/lib_protocol_environment/environment_V2.ml @@ -959,6 +959,8 @@ struct let set_log_message_consumer _ = () let environment_version = Protocol.V2 + + let expected_context_hash = Resulting_context end class ['chain, 'block] proto_rpc_context (t : Tezos_rpc.Context.t) diff --git a/src/lib_protocol_environment/environment_V3.ml b/src/lib_protocol_environment/environment_V3.ml index 62257eb31735..8f3263cd232f 100644 --- a/src/lib_protocol_environment/environment_V3.ml +++ b/src/lib_protocol_environment/environment_V3.ml @@ -1039,6 +1039,8 @@ struct module Lift (P : Updater.PROTOCOL) = struct let environment_version = Protocol.V3 + let expected_context_hash = Resulting_context + include P let value_of_key ~chain_id ~predecessor_context ~predecessor_timestamp diff --git a/src/lib_protocol_environment/environment_V4.ml b/src/lib_protocol_environment/environment_V4.ml index 67f641baf7b1..d6edac7bcf8f 100644 --- a/src/lib_protocol_environment/environment_V4.ml +++ b/src/lib_protocol_environment/environment_V4.ml @@ -1056,6 +1056,8 @@ struct module Lift (P : Updater.PROTOCOL) = struct let environment_version = Protocol.V4 + let expected_context_hash = Resulting_context + include P let value_of_key ~chain_id ~predecessor_context ~predecessor_timestamp diff --git a/src/lib_protocol_environment/environment_V5.ml b/src/lib_protocol_environment/environment_V5.ml index f80ed466cf15..f09cefb20bd8 100644 --- a/src/lib_protocol_environment/environment_V5.ml +++ b/src/lib_protocol_environment/environment_V5.ml @@ -1033,6 +1033,8 @@ struct module Lift (P : Updater.PROTOCOL) = struct let environment_version = Protocol.V5 + let expected_context_hash = Resulting_context + include P let value_of_key ~chain_id ~predecessor_context ~predecessor_timestamp diff --git a/src/lib_protocol_environment/environment_V6.ml b/src/lib_protocol_environment/environment_V6.ml index e3534dac5d7d..2b98d4089d00 100644 --- a/src/lib_protocol_environment/environment_V6.ml +++ b/src/lib_protocol_environment/environment_V6.ml @@ -1123,6 +1123,8 @@ struct module Lift (P : Updater.PROTOCOL) = struct let environment_version = Protocol.V6 + let expected_context_hash = Resulting_context + include P let value_of_key ~chain_id ~predecessor_context ~predecessor_timestamp diff --git a/src/lib_protocol_environment/environment_V7.ml b/src/lib_protocol_environment/environment_V7.ml index 4a4f9fe1da69..53da5215fd67 100644 --- a/src/lib_protocol_environment/environment_V7.ml +++ b/src/lib_protocol_environment/environment_V7.ml @@ -1105,6 +1105,8 @@ struct module Lift (P : Updater.PROTOCOL) = struct let environment_version = Protocol.V7 + let expected_context_hash = Resulting_context + include P let value_of_key ~chain_id ~predecessor_context ~predecessor_timestamp diff --git a/src/lib_protocol_environment/environment_V8.ml b/src/lib_protocol_environment/environment_V8.ml index ca986aacb3ca..ded7c6a081a1 100644 --- a/src/lib_protocol_environment/environment_V8.ml +++ b/src/lib_protocol_environment/environment_V8.ml @@ -1143,6 +1143,8 @@ struct module Lift (P : Updater.PROTOCOL) = struct let environment_version = Protocol.V8 + let expected_context_hash = Predecessor_resulting_context + include P let value_of_key ~chain_id ~predecessor_context ~predecessor_timestamp diff --git a/src/lib_protocol_environment/environment_context.ml b/src/lib_protocol_environment/environment_context.ml index 65066eb54016..fc9e76006595 100644 --- a/src/lib_protocol_environment/environment_context.ml +++ b/src/lib_protocol_environment/environment_context.ml @@ -784,3 +784,7 @@ type rpc_context = { block_header : Block_header.shell_header; context : Context.t; } + +type header_context_hash_semantics = + | Resulting_context + | Predecessor_resulting_context diff --git a/src/lib_protocol_environment/environment_context.mli b/src/lib_protocol_environment/environment_context.mli index afa6a7d4b686..8d5c46031880 100644 --- a/src/lib_protocol_environment/environment_context.mli +++ b/src/lib_protocol_environment/environment_context.mli @@ -234,4 +234,14 @@ type rpc_context = { context : Context.t; } +(** Type of semantics defining the context's hash present in a block + header. *) +type header_context_hash_semantics = + | Resulting_context + (** The block header must contain the hash of the context + resulting of the block's application. *) + | Predecessor_resulting_context + (** The block header must contain the hash of the context + resulting of its predecessor block application. *) + val err_implementation_mismatch : expected:string -> got:string -> 'a diff --git a/src/lib_protocol_environment/environment_protocol_T.ml b/src/lib_protocol_environment/environment_protocol_T.ml index 61b207bfb12e..d5bdb4c87060 100644 --- a/src/lib_protocol_environment/environment_protocol_T.ml +++ b/src/lib_protocol_environment/environment_protocol_T.ml @@ -249,6 +249,8 @@ module type PROTOCOL = sig val environment_version : Protocol.env_version + val expected_context_hash : header_context_hash_semantics + val begin_validation : Context.t -> Tezos_crypto.Chain_id.t -> diff --git a/src/lib_protocol_environment/environment_protocol_T_test.ml b/src/lib_protocol_environment/environment_protocol_T_test.ml index 336d26ffa578..375fa26e267a 100644 --- a/src/lib_protocol_environment/environment_protocol_T_test.ml +++ b/src/lib_protocol_environment/environment_protocol_T_test.ml @@ -54,6 +54,8 @@ module Mock_all_unit : let environment_version = Protocol.V0 + let expected_context_hash = Environment_context.Resulting_context + let init _ = assert false type nonrec validation_state = unit -- GitLab From 587d836c9254fbc3c8e5ae6d917a40dc7a8a8984 Mon Sep 17 00:00:00 2001 From: vbot Date: Thu, 24 Nov 2022 16:49:04 +0100 Subject: [PATCH 02/19] Base: update the block shell header's context documentation --- src/lib_base/block_header.mli | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib_base/block_header.mli b/src/lib_base/block_header.mli index 07903fc29c31..5bdd779fb84c 100644 --- a/src/lib_base/block_header.mli +++ b/src/lib_base/block_header.mli @@ -41,7 +41,9 @@ type shell_header = { then lexicographically. It represents the claimed fitness of the chain ending in this block. *) context : Tezos_crypto.Context_hash.t; - (** Hash of the state of the context after application of this block. *) + (** Hash of the state of the context either before or after + application of this block depending on the protocol + environment's [expected_context_hash] value. *) } val shell_header_encoding : shell_header Data_encoding.t -- GitLab From a033cb980e1c4073e5f2b98d8b02cee8e40c23c1 Mon Sep 17 00:00:00 2001 From: vbot Date: Mon, 21 Nov 2022 11:05:17 +0100 Subject: [PATCH 03/19] Store: open Tezos_crypto by default and make fmt --- manifest/main.ml | 7 + opam/tezos-store.opam | 1 + src/lib_store/dune | 2 + src/lib_store/mocked/block_store.ml | 4 +- src/lib_store/mocked/block_store.mli | 5 +- src/lib_store/mocked/dune | 2 + src/lib_store/mocked/protocol_store.ml | 8 +- src/lib_store/mocked/protocol_store.mli | 18 +- src/lib_store/mocked/store.ml | 165 ++++++-------- src/lib_store/shared/block_key.ml | 12 +- src/lib_store/shared/block_key.mli | 4 +- src/lib_store/shared/block_repr.ml | 32 ++- src/lib_store/shared/block_repr.mli | 29 ++- src/lib_store/shared/dune | 2 + src/lib_store/shared/naming.ml | 10 +- src/lib_store/shared/naming.mli | 10 +- src/lib_store/shared/store_events.ml | 24 +-- src/lib_store/shared/store_types.ml | 31 ++- src/lib_store/shared/store_types.mli | 28 ++- src/lib_store/store.mli | 158 +++++--------- src/lib_store/unix/block_repr_unix.ml | 15 +- src/lib_store/unix/block_repr_unix.mli | 4 +- src/lib_store/unix/block_store.ml | 35 ++- src/lib_store/unix/block_store.mli | 7 +- src/lib_store/unix/cemented_block_store.ml | 3 +- src/lib_store/unix/cemented_block_store.mli | 13 +- src/lib_store/unix/consistency.ml | 18 +- src/lib_store/unix/dune | 8 +- src/lib_store/unix/floating_block_index.ml | 24 +-- src/lib_store/unix/floating_block_index.mli | 10 +- src/lib_store/unix/floating_block_store.ml | 2 +- src/lib_store/unix/floating_block_store.mli | 33 ++- src/lib_store/unix/protocol_store.ml | 15 +- src/lib_store/unix/protocol_store.mli | 18 +- src/lib_store/unix/reconstruction.ml | 31 ++- src/lib_store/unix/reconstruction.mli | 5 +- src/lib_store/unix/snapshots.ml | 157 ++++++-------- src/lib_store/unix/snapshots.mli | 21 +- src/lib_store/unix/snapshots_events.ml | 4 +- src/lib_store/unix/store.ml | 203 +++++++----------- src/lib_store/unix/store.mli | 166 ++++++-------- src/lib_store/unix/test/alpha_utils.ml | 78 +++---- src/lib_store/unix/test/dune | 2 + src/lib_store/unix/test/test_block_store.ml | 11 +- .../unix/test/test_cemented_store.ml | 4 +- src/lib_store/unix/test/test_consistency.ml | 4 +- src/lib_store/unix/test/test_locator.ml | 23 +- src/lib_store/unix/test/test_snapshots.ml | 6 +- src/lib_store/unix/test/test_store.ml | 37 ++-- src/lib_store/unix/test/test_testchain.ml | 10 +- src/lib_store/unix/test/test_utils.ml | 27 ++- 51 files changed, 634 insertions(+), 912 deletions(-) diff --git a/manifest/main.ml b/manifest/main.ml index 61295b23a34a..bb339a26400a 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -2242,6 +2242,7 @@ let octez_store_shared = ~deps: [ octez_base |> open_ |> open_ ~m:"TzPervasives"; + octez_crypto |> open_; octez_shell_services |> open_; aches; aches_lwt; @@ -2277,6 +2278,7 @@ let octez_store_unix = octez_protocol_updater |> open_; octez_stdlib_unix |> open_; octez_stdlib |> open_; + octez_crypto |> open_; lwt_watcher; aches; aches_lwt; @@ -2307,6 +2309,7 @@ let octez_store_unix_reconstruction = [ octez_base |> open_ |> open_ ~m:"TzPervasives"; octez_stdlib_unix |> open_; + octez_crypto |> open_; octez_shell_services |> open_; octez_protocol_updater |> open_; octez_validation |> open_; @@ -2324,6 +2327,7 @@ let octez_store_unix_snapshots = [ octez_base |> open_ |> open_ ~m:"TzPervasives"; octez_stdlib_unix |> open_; + octez_crypto |> open_; octez_shell_services |> open_; octez_context |> open_; octez_validation |> open_; @@ -2345,6 +2349,7 @@ tezos-store is a virtual library that provides two implementations: ~deps: [ octez_base |> open_ |> open_ ~m:"TzPervasives"; + octez_crypto |> open_; octez_rpc; lwt_watcher; octez_shell_services |> open_; @@ -2369,6 +2374,7 @@ let _octez_store_mocked = ~deps: [ octez_base |> open_ ~m:"TzPervasives"; + octez_crypto |> open_; octez_shell_services |> open_; octez_context_memory |> open_; octez_context_ops |> open_; @@ -5315,6 +5321,7 @@ let _octez_store_tests = ~deps: [ octez_base |> open_ ~m:"TzPervasives"; + octez_crypto |> open_; octez_context_ops |> open_; octez_store_shared |> open_; octez_store_unix |> open_; diff --git a/opam/tezos-store.opam b/opam/tezos-store.opam index 22dcf9673e15..7f72fbdc306f 100644 --- a/opam/tezos-store.opam +++ b/opam/tezos-store.opam @@ -10,6 +10,7 @@ license: "MIT" depends: [ "dune" { >= "3.0" } "tezos-base" + "tezos-crypto" "tezos-shell-services" "aches" { >= "1.0.0" } "aches-lwt" { >= "1.0.0" } diff --git a/src/lib_store/dune b/src/lib_store/dune index 79a15e768955..8669675d64aa 100644 --- a/src/lib_store/dune +++ b/src/lib_store/dune @@ -7,6 +7,7 @@ (instrumentation (backend bisect_ppx)) (libraries tezos-base + tezos-crypto tezos-rpc lwt-watcher tezos-shell-services @@ -17,6 +18,7 @@ (:standard) -open Tezos_base -open Tezos_base.TzPervasives + -open Tezos_crypto -open Tezos_shell_services -open Tezos_validation -open Tezos_context_ops diff --git a/src/lib_store/mocked/block_store.ml b/src/lib_store/mocked/block_store.ml index d7ba071c6bc9..aa77977052ce 100644 --- a/src/lib_store/mocked/block_store.ml +++ b/src/lib_store/mocked/block_store.ml @@ -25,9 +25,9 @@ type merge_status = Not_running | Running | Merge_failed of tztrace -type key = Block of (Tezos_crypto.Block_hash.t * int) +type key = Block of (Block_hash.t * int) -module Block_table = Tezos_crypto.Block_hash.Table +module Block_table = Block_hash.Table module Level_table = Hashtbl.Make (struct type t = int32 diff --git a/src/lib_store/mocked/block_store.mli b/src/lib_store/mocked/block_store.mli index f929bd132b59..346f432f48e2 100644 --- a/src/lib_store/mocked/block_store.mli +++ b/src/lib_store/mocked/block_store.mli @@ -157,7 +157,7 @@ type t = block_store A block key may represent an invalid block (wrong hash and/or offset) as it is not ensured to be valid by construction.*) -type key = Block of (Tezos_crypto.Block_hash.t * int) +type key = Block of (Block_hash.t * int) (** The status of the merging thread *) type merge_status = Not_running | Running | Merge_failed of tztrace @@ -181,8 +181,7 @@ val mem : block_store -> key -> bool tzresult Lwt.t (** [get_hash block_store key] retrieves the hash corresponding to the given [key] in [block_store]. Return [None] if the block is unknown. *) -val get_hash : - block_store -> key -> Tezos_crypto.Block_hash.t option tzresult Lwt.t +val get_hash : block_store -> key -> Block_hash.t option tzresult Lwt.t (** [read_block ~read_metadata block_store key] reads the block [key] in [block_store] if present. Return [None] if the block is diff --git a/src/lib_store/mocked/dune b/src/lib_store/mocked/dune index 9ed657a42845..e082ec8c1adb 100644 --- a/src/lib_store/mocked/dune +++ b/src/lib_store/mocked/dune @@ -8,6 +8,7 @@ (instrumentation (backend bisect_ppx)) (libraries tezos-base + tezos-crypto tezos-shell-services tezos-context.memory tezos-context-ops @@ -17,6 +18,7 @@ (flags (:standard) -open Tezos_base.TzPervasives + -open Tezos_crypto -open Tezos_shell_services -open Tezos_context_memory -open Tezos_context_ops diff --git a/src/lib_store/mocked/protocol_store.ml b/src/lib_store/mocked/protocol_store.ml index 72fcb4225f65..ee693aa962f0 100644 --- a/src/lib_store/mocked/protocol_store.ml +++ b/src/lib_store/mocked/protocol_store.ml @@ -23,15 +23,13 @@ (* *) (*****************************************************************************) -module Protocol_table = Tezos_crypto.Protocol_hash.Table +module Protocol_table = Protocol_hash.Table type t = Protocol.t Protocol_table.t -let mem table protocol_hash = - Tezos_crypto.Protocol_hash.Table.mem table protocol_hash +let mem table protocol_hash = Protocol_hash.Table.mem table protocol_hash -let all table = - Protocol_table.to_seq_keys table |> Tezos_crypto.Protocol_hash.Set.of_seq +let all table = Protocol_table.to_seq_keys table |> Protocol_hash.Set.of_seq let raw_store store protocol_hash bytes = if mem store protocol_hash then Lwt.return_none diff --git a/src/lib_store/mocked/protocol_store.mli b/src/lib_store/mocked/protocol_store.mli index 505203ee6bac..c8d281e5e33e 100644 --- a/src/lib_store/mocked/protocol_store.mli +++ b/src/lib_store/mocked/protocol_store.mli @@ -30,33 +30,25 @@ type t (** [mem pstore proto_hash] tests the existence of the protocol indexed by [proto_hash] in the store. *) -val mem : t -> Tezos_crypto.Protocol_hash.t -> bool +val mem : t -> Protocol_hash.t -> bool (** [all pstore] returns the set of all stored protocols in [pstore]. *) -val all : t -> Tezos_crypto.Protocol_hash.Set.t +val all : t -> Protocol_hash.Set.t (** [raw_store pstore proto_hash proto_bytes] stores on disk the protocol [proto_bytes] (encoded bytes) indexed as [proto_hash]. Returns [None] if the protocol already exists. *) -val raw_store : - t -> - Tezos_crypto.Protocol_hash.t -> - bytes -> - Tezos_crypto.Protocol_hash.t option Lwt.t +val raw_store : t -> Protocol_hash.t -> bytes -> Protocol_hash.t option Lwt.t (** [store pstore proto_hash protocol] stores on disk the protocol [protocol] indexed as [proto_hash]. Returns [None] if the protocol already exists. *) -val store : - t -> - Tezos_crypto.Protocol_hash.t -> - Protocol.t -> - Tezos_crypto.Protocol_hash.t option Lwt.t +val store : t -> Protocol_hash.t -> Protocol.t -> Protocol_hash.t option Lwt.t (** [read pstore proto_hash] reads from [pstore] and returns the protocol indexed by [proto_hash]. Returns [None] if the protocol cannot be read. *) -val read : t -> Tezos_crypto.Protocol_hash.t -> Protocol.t option Lwt.t +val read : t -> Protocol_hash.t -> Protocol.t option Lwt.t (** [init store_dir] creates a store relatively to [store_dir] path or loads it if it already exists. *) diff --git a/src/lib_store/mocked/store.ml b/src/lib_store/mocked/store.ml index 7e48faea3a7f..17726f82ba43 100644 --- a/src/lib_store/mocked/store.ml +++ b/src/lib_store/mocked/store.ml @@ -60,13 +60,13 @@ type store = { context_index : Context_ops.index; protocol_store : Protocol_store.t; allow_testchains : bool; - protocol_watcher : Tezos_crypto.Protocol_hash.t Lwt_watcher.input; + protocol_watcher : Protocol_hash.t Lwt_watcher.input; global_block_watcher : (chain_store * block) Lwt_watcher.input; } and chain_store = { global_store : store; - chain_id : Tezos_crypto.Chain_id.t; + chain_id : Chain_id.t; chain_dir : [`Chain_dir] Naming.directory; chain_config : chain_config; block_store : Block_store.t; @@ -75,8 +75,8 @@ and chain_store = { genesis_block_data : block Stored_data.t; block_watcher : block Lwt_watcher.input; block_rpc_directories : - (chain_store * block) Tezos_rpc.Directory.t Tezos_crypto.Protocol_hash.Map.t - Tezos_crypto.Protocol_hash.Table.t; + (chain_store * block) Tezos_rpc.Directory.t Protocol_hash.Map.t + Protocol_hash.Table.t; } and chain_state = { @@ -90,26 +90,20 @@ and chain_state = { (* Following fields are safe to update directly *) protocol_levels_data : Protocol_levels.activation_block Protocol_levels.t Stored_data.t; - invalid_blocks_data : - invalid_block Tezos_crypto.Block_hash.Map.t Stored_data.t; - forked_chains_data : - Tezos_crypto.Block_hash.t Tezos_crypto.Chain_id.Map.t Stored_data.t; + invalid_blocks_data : invalid_block Block_hash.Map.t Stored_data.t; + forked_chains_data : Block_hash.t Chain_id.Map.t Stored_data.t; (* In memory-only: *) current_head : Block_repr.t; active_testchain : testchain option; mempool : Mempool.t; - live_blocks : Tezos_crypto.Block_hash.Set.t; - live_operations : Tezos_crypto.Operation_hash.Set.t; + live_blocks : Block_hash.Set.t; + live_operations : Operation_hash.Set.t; mutable live_data_cache : - (Tezos_crypto.Block_hash.t * Tezos_crypto.Operation_hash.Set.t) Ringo.Ring.t - option; + (Block_hash.t * Operation_hash.Set.t) Ringo.Ring.t option; prechecked_blocks : Block_repr.t Block_lru_cache.t; } -and testchain = { - forked_block : Tezos_crypto.Block_hash.t; - testchain_store : chain_store; -} +and testchain = {forked_block : Block_hash.t; testchain_store : chain_store} and block = Block_repr.t @@ -155,7 +149,7 @@ let locked_is_acceptable_block chain_state (hash, level) = | None -> Lwt.return_true | Some (target_hash, target_level) -> if Compare.Int32.(level = target_level) then - Lwt.return @@ Tezos_crypto.Block_hash.equal hash target_hash + Lwt.return @@ Block_hash.equal hash target_hash else Lwt.return_true module Block = struct @@ -171,8 +165,7 @@ module Block = struct operations_metadata : Block_validation.operation_metadata list list; } - let equal b b' = - Tezos_crypto.Block_hash.equal (Block_repr.hash b) (Block_repr.hash b') + let equal b b' = Block_hash.equal (Block_repr.hash b) (Block_repr.hash b') let descriptor blk = Block_repr.descriptor blk @@ -190,7 +183,7 @@ module Block = struct let locked_is_known_invalid chain_state hash = let open Lwt_syntax in let* invalid_blocks = Stored_data.get chain_state.invalid_blocks_data in - Lwt.return (Tezos_crypto.Block_hash.Map.mem hash invalid_blocks) + Lwt.return (Block_hash.Map.mem hash invalid_blocks) let is_known_invalid {chain_state; _} hash = Shared.use chain_state (fun chain_state -> @@ -221,7 +214,7 @@ module Block = struct let is_genesis chain_store hash = let genesis = genesis chain_store in - Tezos_crypto.Block_hash.equal hash genesis.Genesis.block + Block_hash.equal hash genesis.Genesis.block let read_block {block_store; _} ?(distance = 0) hash = let open Lwt_result_syntax in @@ -419,7 +412,7 @@ module Block = struct in let*! genesis_block = Stored_data.get chain_store.genesis_block_data in let is_main_chain = - Tezos_crypto.Chain_id.equal + Chain_id.equal chain_store.chain_id (WithExceptions.Option.get ~loc:__LOC__ @@ -472,9 +465,7 @@ module Block = struct in let* () = fail_unless - (Tezos_crypto.Context_hash.equal - block_header.shell.context - context_hash) + (Context_hash.equal block_header.shell.context context_hash) (Validation_errors.Inconsistent_hash (context_hash, block_header.shell.context)) in @@ -618,7 +609,7 @@ module Block = struct let open Lwt_syntax in Shared.use chain_state (fun chain_state -> let* invalid_blocks = Stored_data.get chain_state.invalid_blocks_data in - Lwt.return (Tezos_crypto.Block_hash.Map.find hash invalid_blocks)) + Lwt.return (Block_hash.Map.find hash invalid_blocks)) let read_invalid_blocks {chain_state; _} = Shared.use chain_state (fun chain_state -> @@ -634,10 +625,7 @@ module Block = struct chain_state.invalid_blocks_data (fun invalid_blocks -> Lwt.return - (Tezos_crypto.Block_hash.Map.add - hash - {level; errors} - invalid_blocks))) + (Block_hash.Map.add hash {level; errors} invalid_blocks))) in return_unit @@ -646,7 +634,7 @@ module Block = struct Stored_data.update_with chain_state.invalid_blocks_data (fun invalid_blocks -> - Lwt.return (Tezos_crypto.Block_hash.Map.remove hash invalid_blocks))) + Lwt.return (Block_hash.Map.remove hash invalid_blocks))) (** Accessors *) @@ -693,8 +681,8 @@ module Block = struct else Option.map (fun ll -> - Tezos_crypto.Operation_metadata_list_list_hash.compute - (List.map Tezos_crypto.Operation_metadata_list_hash.compute ll)) + Operation_metadata_list_list_hash.compute + (List.map Operation_metadata_list_hash.compute ll)) (Block_repr.operations_metadata_hashes blk) (** Metadata accessors *) @@ -711,10 +699,8 @@ module Block = struct let operations_metadata metadata = Block_repr.operations_metadata metadata let compute_operation_path hashes = - let list_hashes = - List.map Tezos_crypto.Operation_list_hash.compute hashes - in - Tezos_crypto.Operation_list_list_hash.compute_path list_hashes + let list_hashes = List.map Operation_list_hash.compute hashes in + Operation_list_list_hash.compute_path list_hashes let operations_path block i = if i < 0 || validation_passes block <= i then invalid_arg "operations_path" ; @@ -790,27 +776,22 @@ module Chain_traversal = struct let live_blocks chain_store block n = let fold (bacc, oacc) (head_hash, op_hashes) = - let bacc = Tezos_crypto.Block_hash.Set.add head_hash bacc in + let bacc = Block_hash.Set.add head_hash bacc in let oacc = List.fold_left - (List.fold_left (fun oacc op -> - Tezos_crypto.Operation_hash.Set.add op oacc)) + (List.fold_left (fun oacc op -> Operation_hash.Set.add op oacc)) oacc op_hashes in (bacc, oacc) in - let init = - (Tezos_crypto.Block_hash.Set.empty, Tezos_crypto.Operation_hash.Set.empty) - in + let init = (Block_hash.Set.empty, Operation_hash.Set.empty) in folder chain_store block n fold init let live_blocks_with_ring chain_store block n ring = let open Lwt_syntax in let fold acc (head_hash, op_hashes) = - let op_hash_set = - Tezos_crypto.Operation_hash.Set.(of_list (List.flatten op_hashes)) - in + let op_hash_set = Operation_hash.Set.(of_list (List.flatten op_hashes)) in (head_hash, op_hash_set) :: acc in let* l = folder chain_store block n fold [] in @@ -909,7 +890,7 @@ module Chain = struct let*! current_head_descr = Stored_data.get chain_state.current_head_data in - if Tezos_crypto.Block_hash.equal head (fst current_head_descr) then + if Block_hash.equal head (fst current_head_descr) then return (Some {chain_state with mempool}, ()) else return (None, ())) @@ -931,22 +912,20 @@ module Chain = struct match live_data_cache with | Some live_data_cache when update_cache - && Tezos_crypto.Block_hash.equal + && Block_hash.equal (Block.predecessor block) (Block.hash current_head) && Ringo.Ring.capacity live_data_cache = expected_capacity -> ( let most_recent_block = Block.hash block in let most_recent_ops = Block.all_operation_hashes block - |> List.flatten |> Tezos_crypto.Operation_hash.Set.of_list + |> List.flatten |> Operation_hash.Set.of_list in let new_live_blocks = - Tezos_crypto.Block_hash.Set.add most_recent_block live_blocks + Block_hash.Set.add most_recent_block live_blocks in let new_live_operations = - Tezos_crypto.Operation_hash.Set.union - most_recent_ops - live_operations + Operation_hash.Set.union most_recent_ops live_operations in match Ringo.Ring.add_and_return_erased @@ -956,12 +935,10 @@ module Chain = struct | None -> Lwt.return (new_live_blocks, new_live_operations) | Some (last_block, last_ops) -> let diffed_new_live_blocks = - Tezos_crypto.Block_hash.Set.remove last_block new_live_blocks + Block_hash.Set.remove last_block new_live_blocks in let diffed_new_live_operations = - Tezos_crypto.Operation_hash.Set.diff - new_live_operations - last_ops + Operation_hash.Set.diff new_live_operations last_ops in Lwt.return (diffed_new_live_blocks, diffed_new_live_operations)) | _ when update_cache -> @@ -977,12 +954,9 @@ module Chain = struct let live_blocks, live_ops = Ringo.Ring.fold new_cache - ~init: - ( Tezos_crypto.Block_hash.Set.empty, - Tezos_crypto.Operation_hash.Set.empty ) + ~init:(Block_hash.Set.empty, Operation_hash.Set.empty) ~f:(fun (bhs, opss) (bh, ops) -> - ( Tezos_crypto.Block_hash.Set.add bh bhs, - Tezos_crypto.Operation_hash.Set.union ops opss )) + (Block_hash.Set.add bh bhs, Operation_hash.Set.union ops opss)) in Lwt.return (live_blocks, live_ops) | _ -> Chain_traversal.live_blocks chain_store block expected_capacity @@ -1005,7 +979,7 @@ module Chain = struct let open Lwt_syntax in if Compare.Int32.(lvl' > lvl) then Lwt.return_false else if Compare.Int32.(lvl = lvl') then - Lwt.return (Tezos_crypto.Block_hash.equal hash hash') + Lwt.return (Block_hash.equal hash hash') else let* o = Block.read_ancestor_hash_opt @@ -1015,8 +989,7 @@ module Chain = struct in match o with | None -> Lwt.return_false - | Some hash_found -> - Lwt.return (Tezos_crypto.Block_hash.equal hash' hash_found) + | Some hash_found -> Lwt.return (Block_hash.equal hash' hash_found) let is_in_chain chain_store (hash, level) = let open Lwt_syntax in @@ -1323,7 +1296,7 @@ module Chain = struct chain_state.invalid_blocks_data (fun invalid_blocks -> Lwt.return - (Tezos_crypto.Block_hash.Map.filter + (Block_hash.Map.filter (fun _k {level; _} -> level > snd new_checkpoint) invalid_blocks)) in @@ -1406,8 +1379,7 @@ module Chain = struct in match o with | None -> tzfail Missing_last_allowed_fork_level_block - | Some lafl_hash -> - return (Tezos_crypto.Block_hash.equal lafl_hash ancestor))) + | Some lafl_hash -> return (Block_hash.equal lafl_hash ancestor))) let is_valid_for_checkpoint chain_store given_checkpoint = let open Lwt_syntax in @@ -1542,10 +1514,7 @@ module Chain = struct ( best_head, List.filter (fun (hash, _) -> - not - (Tezos_crypto.Block_hash.equal - (Block.hash best_head) - hash)) + not (Block_hash.equal (Block.hash best_head) hash)) all_heads ) in (* Case 1 *) @@ -1650,20 +1619,18 @@ module Chain = struct let* invalid_blocks_data = Stored_data.init (Naming.invalid_blocks_file chain_dir) - ~initial_data:Tezos_crypto.Block_hash.Map.empty + ~initial_data:Block_hash.Map.empty in let* forked_chains_data = Stored_data.init (Naming.forked_chains_file chain_dir) - ~initial_data:Tezos_crypto.Chain_id.Map.empty + ~initial_data:Chain_id.Map.empty in let current_head = genesis_block in let active_testchain = None in let mempool = Mempool.empty in - let live_blocks = - Tezos_crypto.Block_hash.Set.singleton genesis_block.hash - in - let live_operations = Tezos_crypto.Operation_hash.Set.empty in + let live_blocks = Block_hash.Set.singleton genesis_block.hash in + let live_operations = Operation_hash.Set.empty in let live_data_cache = None in let prechecked_blocks = Block_lru_cache.create 10 in return @@ -1741,7 +1708,7 @@ module Chain = struct in let chain_state = Shared.create chain_state in let block_watcher = Lwt_watcher.create_input () in - let block_rpc_directories = Tezos_crypto.Protocol_hash.Table.create 7 in + let block_rpc_directories = Protocol_hash.Table.create 7 in let chain_store : chain_store = { global_store; @@ -1852,7 +1819,7 @@ module Chain = struct let*! o = find_activation_block chain_store ~protocol_level in match o with | Some {block = bh, _; _} -> - if Tezos_crypto.Block_hash.(bh <> Block.hash block) then + if Block_hash.(bh <> Block.hash block) then set_protocol_level chain_store ~protocol_level (block, protocol_hash) else return_unit | None -> @@ -1904,8 +1871,7 @@ module Chain = struct let* o = Block.read_predecessor_opt chain_store block in match o with | None -> Lwt.return_none (* genesis *) - | Some pred - when Tezos_crypto.Block_hash.equal (Block.hash pred) (Block.hash block) -> + | Some pred when Block_hash.equal (Block.hash pred) (Block.hash block) -> Lwt.return_none (* genesis *) | Some pred -> ( let* _, save_point_level = savepoint chain_store in @@ -1922,27 +1888,25 @@ module Chain = struct else Block.protocol_hash_exn chain_store pred in match - Tezos_crypto.Protocol_hash.Table.find - chain_store.block_rpc_directories - protocol + Protocol_hash.Table.find chain_store.block_rpc_directories protocol with | None -> Lwt.return_none | Some map -> let* next_protocol = Block.protocol_hash_exn chain_store block in - Lwt.return (Tezos_crypto.Protocol_hash.Map.find next_protocol map)) + Lwt.return (Protocol_hash.Map.find next_protocol map)) let set_rpc_directory chain_store ~protocol_hash ~next_protocol_hash dir = let map = Option.value - ~default:Tezos_crypto.Protocol_hash.Map.empty - (Tezos_crypto.Protocol_hash.Table.find + ~default:Protocol_hash.Map.empty + (Protocol_hash.Table.find chain_store.block_rpc_directories protocol_hash) in - Tezos_crypto.Protocol_hash.Table.replace + Protocol_hash.Table.replace chain_store.block_rpc_directories protocol_hash - (Tezos_crypto.Protocol_hash.Map.add next_protocol_hash dir map) ; + (Protocol_hash.Map.add next_protocol_hash dir map) ; Lwt.return_unit (* Not implemented as both the store and context garbage collection @@ -2044,7 +2008,7 @@ let init ?patch_context ?commit_genesis ?history_mode ?(readonly = false) patch_context in let store_dir = Naming.store_dir ~dir_path:store_dir in - let chain_id = Tezos_crypto.Chain_id.of_block_hash genesis.Genesis.block in + let chain_id = Chain_id.of_block_hash genesis.Genesis.block in let*! context_index, commit_genesis = let open Tezos_context_memory in match commit_genesis with @@ -2100,7 +2064,7 @@ let get_chain_store store chain_id = let chain_store = main_chain_store store in let rec loop chain_store = let open Lwt_result_syntax in - if Tezos_crypto.Chain_id.equal (Chain.chain_id chain_store) chain_id then + if Chain_id.equal (Chain.chain_id chain_store) chain_id then return chain_store else Shared.use chain_store.chain_state (fun {active_testchain; _} -> @@ -2205,7 +2169,7 @@ let rec make_pp_chain_store (chain_store : chain_store) = proto_level pp_block_descriptor block - Tezos_crypto.Protocol_hash.pp + Protocol_hash.pp protocol (option_pp ~default:"n/a" (fun fmt _ -> Format.fprintf fmt "available")) commit_info @@ -2225,7 +2189,7 @@ let rec make_pp_chain_store (chain_store : chain_store) = highwatermark: %a@ caboose: %a@ target: %a@ @[protocol levels:@ \ %a@]@ @[invalid blocks:@ %a@]@ @[forked chains:@ %a@]@ @[active testchain: %a@]@]" - Tezos_crypto.Chain_id.pp + Chain_id.pp chain_id (Naming.dir_path chain_dir) Data_encoding.Json.pp @@ -2259,22 +2223,19 @@ let rec make_pp_chain_store (chain_store : chain_store) = target (Format.pp_print_list ~pp_sep:Format.pp_print_cut pp_protocol_level) (Protocol_levels.bindings protocol_levels_data) - (Format.pp_print_list - ~pp_sep:Format.pp_print_cut - Tezos_crypto.Block_hash.pp) - (Tezos_crypto.Block_hash.Map.bindings invalid_blocks_data - |> List.map fst) + (Format.pp_print_list ~pp_sep:Format.pp_print_cut Block_hash.pp) + (Block_hash.Map.bindings invalid_blocks_data |> List.map fst) (Format.pp_print_list ~pp_sep:Format.pp_print_cut (fun fmt (chain_id, block_hash) -> Format.fprintf fmt "testchain's chain id: %a, forked block: %a" - Tezos_crypto.Chain_id.pp + Chain_id.pp chain_id - Tezos_crypto.Block_hash.pp + Block_hash.pp block_hash)) - (Tezos_crypto.Chain_id.Map.bindings forked_chains_data) + (Chain_id.Map.bindings forked_chains_data) pp_testchain_opt ()) diff --git a/src/lib_store/shared/block_key.ml b/src/lib_store/shared/block_key.ml index 6f0c77a72433..63fa6ee5a781 100644 --- a/src/lib_store/shared/block_key.ml +++ b/src/lib_store/shared/block_key.ml @@ -23,7 +23,7 @@ (* *) (*****************************************************************************) -include Tezos_crypto.Block_hash +include Block_hash let hash_size = 31 @@ -31,13 +31,13 @@ let t = let open Repr in map (bytes_of (`Fixed hash_size)) - (fun b -> Tezos_crypto.Block_hash.of_bytes_exn b) - (fun bh -> Tezos_crypto.Block_hash.to_bytes bh) + (fun b -> Block_hash.of_bytes_exn b) + (fun bh -> Block_hash.to_bytes bh) -let encode bh = Tezos_crypto.Block_hash.to_string bh +let encode bh = Block_hash.to_string bh -let encoded_size = Tezos_crypto.Block_hash.size (* in bytes *) +let encoded_size = Block_hash.size (* in bytes *) let decode str off = let str = String.sub str off encoded_size in - Tezos_crypto.Block_hash.of_string_exn str + Block_hash.of_string_exn str diff --git a/src/lib_store/shared/block_key.mli b/src/lib_store/shared/block_key.mli index 3ddac652e166..684d58d70366 100644 --- a/src/lib_store/shared/block_key.mli +++ b/src/lib_store/shared/block_key.mli @@ -23,9 +23,9 @@ (* *) (*****************************************************************************) -(** Super-set of {!Tezos_crypto.Block_hash} to satisfy {!Index.Key} interface. *) +(** Super-set of {!Block_hash} to satisfy {!Index.Key} interface. *) -include module type of Tezos_crypto.Block_hash +include module type of Block_hash val t : t Repr.t diff --git a/src/lib_store/shared/block_repr.ml b/src/lib_store/shared/block_repr.ml index 7e8de39e81ca..498cfe792361 100644 --- a/src/lib_store/shared/block_repr.ml +++ b/src/lib_store/shared/block_repr.ml @@ -28,9 +28,8 @@ open Store_errors type contents = { header : Block_header.t; operations : Operation.t list list; - block_metadata_hash : Tezos_crypto.Block_metadata_hash.t option; - operations_metadata_hashes : - Tezos_crypto.Operation_metadata_hash.t list list option; + block_metadata_hash : Block_metadata_hash.t option; + operations_metadata_hashes : Operation_metadata_hash.t list list option; } type metadata = { @@ -50,14 +49,14 @@ type legacy_metadata = { } type legacy_block = { - legacy_hash : Tezos_crypto.Block_hash.t; + legacy_hash : Block_hash.t; legacy_contents : contents; mutable legacy_metadata : legacy_metadata option; (* allows updating metadata field when loading cemented metadata *) } type block = { - hash : Tezos_crypto.Block_hash.t; + hash : Block_hash.t; contents : contents; mutable metadata : metadata option; (* allows updating metadata field when loading cemented metadata *) @@ -75,7 +74,7 @@ let create_genesis_block ~genesis context = timestamp = genesis.Genesis.time; fitness = []; validation_passes = 0; - operations_hash = Tezos_crypto.Operation_list_list_hash.empty; + operations_hash = Operation_list_list_hash.empty; context; } in @@ -117,10 +116,10 @@ let contents_encoding = (obj4 (req "header" (dynamic_size Block_header.encoding)) (req "operations" (list (list (dynamic_size Operation.encoding)))) - (opt "block_metadata_hash" Tezos_crypto.Block_metadata_hash.encoding) + (opt "block_metadata_hash" Block_metadata_hash.encoding) (opt "operations_metadata_hashes" - (list (list Tezos_crypto.Operation_metadata_hash.encoding)))) + (list (list Operation_metadata_hash.encoding)))) let metadata_encoding : metadata Data_encoding.t = let open Data_encoding in @@ -203,7 +202,7 @@ let encoding = (dynamic_size ~kind:`Uint30 (obj3 - (req "hash" Tezos_crypto.Block_hash.encoding) + (req "hash" Block_hash.encoding) (req "contents" contents_encoding) (varopt "metadata" metadata_encoding))) @@ -218,7 +217,7 @@ let legacy_encoding = (dynamic_size ~kind:`Uint30 (obj3 - (req "legacy_hash" Tezos_crypto.Block_hash.encoding) + (req "legacy_hash" Block_hash.encoding) (req "legacy_contents" contents_encoding) (varopt "legacy_metadata" legacy_metadata_encoding))) @@ -282,11 +281,10 @@ let check_block_consistency ?genesis_hash ?pred_block block = let result_hash = Block_header.hash block_header in let* () = fail_unless - (Tezos_crypto.Block_hash.equal block_hash result_hash + (Block_hash.equal block_hash result_hash || match genesis_hash with - | Some genesis_hash -> - Tezos_crypto.Block_hash.equal block_hash genesis_hash + | Some genesis_hash -> Block_hash.equal block_hash genesis_hash | None -> false) (Inconsistent_block_hash { @@ -300,7 +298,7 @@ let check_block_consistency ?genesis_hash ?pred_block block = | None -> return_unit | Some pred_block -> fail_unless - (Tezos_crypto.Block_hash.equal (hash pred_block) (predecessor block) + (Block_hash.equal (hash pred_block) (predecessor block) && Compare.Int32.(level block = Int32.succ (level pred_block))) (Inconsistent_block_predecessor { @@ -311,14 +309,14 @@ let check_block_consistency ?genesis_hash ?pred_block block = }) in let computed_operations_hash = - Tezos_crypto.Operation_list_list_hash.compute + Operation_list_list_hash.compute (List.map - Tezos_crypto.Operation_list_hash.compute + Operation_list_hash.compute (List.map (List.map Operation.hash) (operations block))) in let* () = fail_unless - (Tezos_crypto.Operation_list_list_hash.equal + (Operation_list_list_hash.equal computed_operations_hash (operations_hash block)) (Store_errors.Inconsistent_operations_hash diff --git a/src/lib_store/shared/block_repr.mli b/src/lib_store/shared/block_repr.mli index eb1aa25d928a..7df90d4183c4 100644 --- a/src/lib_store/shared/block_repr.mli +++ b/src/lib_store/shared/block_repr.mli @@ -35,9 +35,8 @@ open Store_types type contents = { header : Block_header.t; operations : Operation.t list list; - block_metadata_hash : Tezos_crypto.Block_metadata_hash.t option; - operations_metadata_hashes : - Tezos_crypto.Operation_metadata_hash.t list list option; + block_metadata_hash : Block_metadata_hash.t option; + operations_metadata_hashes : Operation_metadata_hash.t list list option; } (** The type for a block's [metadata] stored on disk. This @@ -67,7 +66,7 @@ type metadata = { users to re-use the same structure to store freshly loaded metadata. *) type block = { - hash : Tezos_crypto.Block_hash.t; + hash : Block_hash.t; contents : contents; mutable metadata : metadata option; } @@ -85,7 +84,7 @@ type legacy_metadata = { } type legacy_block = { - legacy_hash : Tezos_crypto.Block_hash.t; + legacy_hash : Block_hash.t; legacy_contents : contents; mutable legacy_metadata : legacy_metadata option; } @@ -101,7 +100,7 @@ val legacy_encoding : legacy_block Data_encoding.t (** [create_genesis_block ~genesis context_hash] creates a default genesis block for the given [genesis] and its [context_hash] that contains metadata. *) -val create_genesis_block : genesis:Genesis.t -> Tezos_crypto.Context_hash.t -> t +val create_genesis_block : genesis:Genesis.t -> Context_hash.t -> t (** Encoding for {!contents}. *) val contents_encoding : contents Data_encoding.t @@ -128,7 +127,7 @@ val descriptor : t -> block_descriptor (** [hash block] returns the stored [block]'s hash. It is not guaranteed to be the same as [Block_header.hash (header block)] (e.g. in sandbox, the genesis block might have a fake hash). *) -val hash : t -> Tezos_crypto.Block_hash.t +val hash : t -> Block_hash.t (** [operations block] returns the list of list of operations contained in the [block]. *) @@ -144,24 +143,23 @@ val level : t -> Int32.t val proto_level : t -> int -val predecessor : t -> Tezos_crypto.Block_hash.t +val predecessor : t -> Block_hash.t val timestamp : t -> Time.Protocol.t val validation_passes : t -> int -val operations_hash : t -> Tezos_crypto.Operation_list_list_hash.t +val operations_hash : t -> Operation_list_list_hash.t val fitness : t -> Fitness.t -val context : t -> Tezos_crypto.Context_hash.t +val context : t -> Context_hash.t val protocol_data : t -> Bytes.t -val block_metadata_hash : t -> Tezos_crypto.Block_metadata_hash.t option +val block_metadata_hash : t -> Block_metadata_hash.t option -val operations_metadata_hashes : - t -> Tezos_crypto.Operation_metadata_hash.t list list option +val operations_metadata_hashes : t -> Operation_metadata_hash.t list list option (** {2 Metadata accessors} *) @@ -190,10 +188,7 @@ val operations_metadata : - Are the stored operations hashes consistent regarding the stored operations hashes? *) val check_block_consistency : - ?genesis_hash:Tezos_crypto.Block_hash.t -> - ?pred_block:t -> - t -> - unit tzresult Lwt.t + ?genesis_hash:Block_hash.t -> ?pred_block:t -> t -> unit tzresult Lwt.t (** [decode_metadata data] decodes metadata from [data] encoded either with the new encoding or the legacy one. *) diff --git a/src/lib_store/shared/dune b/src/lib_store/shared/dune index 013763451a51..e8e738998887 100644 --- a/src/lib_store/shared/dune +++ b/src/lib_store/shared/dune @@ -7,6 +7,7 @@ (instrumentation (backend bisect_ppx)) (libraries tezos-base + tezos-crypto tezos-shell-services aches aches-lwt @@ -15,6 +16,7 @@ (:standard) -open Tezos_base -open Tezos_base.TzPervasives + -open Tezos_crypto -open Tezos_shell_services -open Tezos_validation) (modules naming block_repr store_types store_events block_key block_level)) diff --git a/src/lib_store/shared/naming.ml b/src/lib_store/shared/naming.ml index 7f2ffde233e8..50822947d8a5 100644 --- a/src/lib_store/shared/naming.ml +++ b/src/lib_store/shared/naming.ml @@ -60,12 +60,10 @@ let store_dir ~dir_path = {dir_path} let protocol_store_dir base_dir = mk_dir base_dir "protocols" let protocol_file dir proto_hash = - mk_file dir (Tezos_crypto.Protocol_hash.to_b58check proto_hash) + mk_file dir (Protocol_hash.to_b58check proto_hash) let chain_dir dir chain_id = - mk_dir - dir - (Format.asprintf "chain_%a" Tezos_crypto.Chain_id.pp_short chain_id) + mk_dir dir (Format.asprintf "chain_%a" Chain_id.pp_short chain_id) let chain_config_file dir = make_encoded_file ~json:true dir ~filename:"config.json" chain_config_encoding @@ -110,13 +108,13 @@ let invalid_blocks_file dir = make_encoded_file dir ~filename:"invalid_blocks" - (Tezos_crypto.Block_hash.Map.encoding invalid_block_encoding) + (Block_hash.Map.encoding invalid_block_encoding) let forked_chains_file dir = make_encoded_file dir ~filename:"forked_chains" - (Tezos_crypto.Chain_id.Map.encoding Tezos_crypto.Block_hash.encoding) + (Chain_id.Map.encoding Block_hash.encoding) let savepoint_file dir = make_encoded_file dir ~filename:"savepoint" block_descriptor_encoding diff --git a/src/lib_store/shared/naming.mli b/src/lib_store/shared/naming.mli index 63dc914de62d..6c974e22ec31 100644 --- a/src/lib_store/shared/naming.mli +++ b/src/lib_store/shared/naming.mli @@ -62,13 +62,13 @@ val protocol_store_dir : (** Protocol file *) val protocol_file : - [`Protocol_dir] directory -> Tezos_crypto.Protocol_hash.t -> [`Protocol] file + [`Protocol_dir] directory -> Protocol_hash.t -> [`Protocol] file (** {2 Chain directory} *) val chain_dir : [< `Store_dir | `Testchains_dir] directory -> - Tezos_crypto.Chain_id.t -> + Chain_id.t -> [`Chain_dir] directory val lock_file : [`Chain_dir] directory -> [`Lockfile] file @@ -111,13 +111,11 @@ val target_file : val invalid_blocks_file : [`Chain_dir] directory -> - ([`Invalid_blocks], invalid_block Tezos_crypto.Block_hash.Map.t) encoded_file + ([`Invalid_blocks], invalid_block Block_hash.Map.t) encoded_file val forked_chains_file : [`Chain_dir] directory -> - ( [`Forked_chains], - Tezos_crypto.Block_hash.t Tezos_crypto.Chain_id.Map.t ) - encoded_file + ([`Forked_chains], Block_hash.t Chain_id.Map.t) encoded_file (** {2 Block store} *) diff --git a/src/lib_store/shared/store_events.ml b/src/lib_store/shared/store_events.ml index ff91d1aed8db..553cff1d4a24 100644 --- a/src/lib_store/shared/store_events.ml +++ b/src/lib_store/shared/store_events.ml @@ -182,12 +182,12 @@ let fork_testchain = "the test chain {chain_id} for protocol {protocol_hash} with genesis \ block hash {genesis_hash} was initialized from {fork_block} and is now \ registered in the store" - ~pp1:Tezos_crypto.Chain_id.pp - ("chain_id", Tezos_crypto.Chain_id.encoding) - ~pp2:Tezos_crypto.Protocol_hash.pp - ("protocol_hash", Tezos_crypto.Protocol_hash.encoding) - ~pp3:Tezos_crypto.Block_hash.pp - ("genesis_hash", Tezos_crypto.Block_hash.encoding) + ~pp1:Chain_id.pp + ("chain_id", Chain_id.encoding) + ~pp2:Protocol_hash.pp + ("protocol_hash", Protocol_hash.encoding) + ~pp3:Block_hash.pp + ("genesis_hash", Block_hash.encoding) ~pp4:pp_block_descriptor ("fork_block", block_descriptor_encoding) @@ -384,8 +384,8 @@ let restore_protocol_activation = ~name:"restore_protocol_activation" ~msg:"protocol {protocol_level} ({protocol_hash}) was successfully restored" ("protocol_level", Data_encoding.int31) - ~pp2:Tezos_crypto.Protocol_hash.pp - ("protocol_hash", Tezos_crypto.Protocol_hash.encoding) + ~pp2:Protocol_hash.pp + ("protocol_hash", Protocol_hash.encoding) let update_protocol_table = declare_4 @@ -396,11 +396,11 @@ let update_protocol_table = "the protocol table was updated: protocol {proto_hash} (level \ {proto_level}) was activated on block {block_hash} (level \ {block_level})" - ("proto_hash", Tezos_crypto.Protocol_hash.encoding) - ~pp1:Tezos_crypto.Protocol_hash.pp_short + ("proto_hash", Protocol_hash.encoding) + ~pp1:Protocol_hash.pp_short ("proto_level", Data_encoding.int31) - ("block_hash", Tezos_crypto.Block_hash.encoding) - ~pp3:Tezos_crypto.Block_hash.pp + ("block_hash", Block_hash.encoding) + ~pp3:Block_hash.pp ("block_level", Data_encoding.int32) ~pp4:pp_int32 diff --git a/src/lib_store/shared/store_types.ml b/src/lib_store/shared/store_types.ml index 749011b58860..941a219fa745 100644 --- a/src/lib_store/shared/store_types.ml +++ b/src/lib_store/shared/store_types.ml @@ -23,14 +23,14 @@ (* *) (*****************************************************************************) -type block_descriptor = Tezos_crypto.Block_hash.t * int32 +type block_descriptor = Block_hash.t * int32 let block_descriptor_encoding = let open Data_encoding in - tup2 Tezos_crypto.Block_hash.encoding int32 + tup2 Block_hash.encoding int32 let pp_block_descriptor fmt (hash, level) = - Format.fprintf fmt "%a (level: %ld)" Tezos_crypto.Block_hash.pp hash level + Format.fprintf fmt "%a (level: %ld)" Block_hash.pp hash level type chain_config = { history_mode : History_mode.t; @@ -61,7 +61,7 @@ let invalid_block_encoding = module Block_lru_cache = Aches_lwt.Lache.Make_option - (Aches.Rache.Transfer (Aches.Rache.LRU) (Tezos_crypto.Block_hash)) + (Aches.Rache.Transfer (Aches.Rache.LRU) (Block_hash)) module Protocol_levels = struct include Map.Make (struct @@ -74,11 +74,10 @@ module Protocol_levels = struct author : string; message : string; test_chain_status : Test_chain_status.t; - predecessor_block_metadata_hash : Tezos_crypto.Block_metadata_hash.t option; - predecessor_ops_metadata_hash : - Tezos_crypto.Operation_metadata_list_list_hash.t option; - data_merkle_root : Tezos_crypto.Context_hash.t; - parents_contexts : Tezos_crypto.Context_hash.t list; + predecessor_block_metadata_hash : Block_metadata_hash.t option; + predecessor_ops_metadata_hash : Operation_metadata_list_list_hash.t option; + data_merkle_root : Context_hash.t; + parents_contexts : Context_hash.t list; } let commit_info_of_tuple @@ -140,18 +139,16 @@ module Protocol_levels = struct (req "author" string) (req "message" string) (req "test_chain_status" Test_chain_status.encoding) - (req "data_merkle_root" Tezos_crypto.Context_hash.encoding) - (opt - "predecessor_block_metadata_hash" - Tezos_crypto.Block_metadata_hash.encoding) + (req "data_merkle_root" Context_hash.encoding) + (opt "predecessor_block_metadata_hash" Block_metadata_hash.encoding) (opt "predecessor_ops_metadata_hash" - Tezos_crypto.Operation_metadata_list_list_hash.encoding) - (req "parents_contexts" (list Tezos_crypto.Context_hash.encoding))) + Operation_metadata_list_list_hash.encoding) + (req "parents_contexts" (list Context_hash.encoding))) type activation_block = { block : block_descriptor; - protocol : Tezos_crypto.Protocol_hash.t; + protocol : Protocol_hash.t; commit_info : commit_info option; } @@ -162,7 +159,7 @@ module Protocol_levels = struct (fun (block, protocol, commit_info) -> {block; protocol; commit_info}) (obj3 (req "block" block_descriptor_encoding) - (req "protocol" Tezos_crypto.Protocol_hash.encoding) + (req "protocol" Protocol_hash.encoding) (opt "commit_info" commit_info_encoding)) let encoding = diff --git a/src/lib_store/shared/store_types.mli b/src/lib_store/shared/store_types.mli index bb939becc251..c6743b7fa3be 100644 --- a/src/lib_store/shared/store_types.mli +++ b/src/lib_store/shared/store_types.mli @@ -26,7 +26,7 @@ (** {1 Global types used in the store library} *) (** The type used to describe a block pointer i.e. its hash and level. *) -type block_descriptor = Tezos_crypto.Block_hash.t * int32 +type block_descriptor = Block_hash.t * int32 (** Encoding for {!block_descriptor}. *) val block_descriptor_encoding : block_descriptor Data_encoding.t @@ -54,8 +54,7 @@ val invalid_block_encoding : invalid_block Data_encoding.t (** Module [Block_lru_cache] implements a lwt LRU cache map indexed by block hashes. *) -module Block_lru_cache : - Aches_lwt.Lache.MAP_OPTION with type key = Tezos_crypto.Block_hash.t +module Block_lru_cache : Aches_lwt.Lache.MAP_OPTION with type key = Block_hash.t (** Module [Protocol_levels] represents an associative map of protocol levels to corresponding blocks which supposedly activate new @@ -65,7 +64,7 @@ module Protocol_levels : sig include Map.S with type key = int (** The type representing a subset of the commit information. These - are used to easily check that a given [Tezos_crypto.Context_hash.t], with the + are used to easily check that a given [Context_hash.t], with the associated context not present on disk, is consistent. It is used to verify that an announced protocol is indeed the one that was committed on disk. Fields are: @@ -88,23 +87,22 @@ module Protocol_levels : sig author : string; message : string; test_chain_status : Test_chain_status.t; - predecessor_block_metadata_hash : Tezos_crypto.Block_metadata_hash.t option; - predecessor_ops_metadata_hash : - Tezos_crypto.Operation_metadata_list_list_hash.t option; - data_merkle_root : Tezos_crypto.Context_hash.t; - parents_contexts : Tezos_crypto.Context_hash.t list; + predecessor_block_metadata_hash : Block_metadata_hash.t option; + predecessor_ops_metadata_hash : Operation_metadata_list_list_hash.t option; + data_merkle_root : Context_hash.t; + parents_contexts : Context_hash.t list; } val commit_info_of_tuple : - Tezos_crypto.Protocol_hash.t + Protocol_hash.t * string * string * Time.Protocol.t * Test_chain_status.t - * Tezos_crypto.Context_hash.t - * Tezos_crypto.Block_metadata_hash.t option - * Tezos_crypto.Operation_metadata_list_list_hash.t option - * Tezos_crypto.Context_hash.t list -> + * Context_hash.t + * Block_metadata_hash.t option + * Operation_metadata_list_list_hash.t option + * Context_hash.t list -> commit_info (** Encoding for {!commit_info}. *) @@ -133,7 +131,7 @@ module Protocol_levels : sig allowing us to remove the option. *) type activation_block = { block : block_descriptor; - protocol : Tezos_crypto.Protocol_hash.t; + protocol : Protocol_hash.t; commit_info : commit_info option; } diff --git a/src/lib_store/store.mli b/src/lib_store/store.mli index ce41bba5d834..c3e3c7c5d3c7 100644 --- a/src/lib_store/store.mli +++ b/src/lib_store/store.mli @@ -184,7 +184,7 @@ type chain_store chains and instantiate testchain's sub chain stores, for all chains contained in the store. The chain store created is based on the [genesis] provided. Its chain identifier will be computed - using the {!Tezos_crypto.Tezos_crypto.Chain_id.of_block_hash} function. + using the {!Chain_id.of_block_hash} function. @param patch_context the handle called when initializing the context. It usually is passed when creating a sandboxed chain. @@ -213,9 +213,7 @@ val init : ?patch_context: (Tezos_protocol_environment.Context.t -> Tezos_protocol_environment.Context.t tzresult Lwt.t) -> - ?commit_genesis: - (chain_id:Tezos_crypto.Chain_id.t -> - Tezos_crypto.Context_hash.t tzresult Lwt.t) -> + ?commit_genesis:(chain_id:Chain_id.t -> Context_hash.t tzresult Lwt.t) -> ?history_mode:History_mode.t -> ?readonly:bool -> ?block_cache_limit:int -> @@ -267,13 +265,11 @@ val all_chain_stores : store -> chain_store list Lwt.t (** [get_chain_store global_store chain_id] returns the initialized chain store in [global_store] associated to [chain_id]. *) -val get_chain_store : - store -> Tezos_crypto.Chain_id.t -> chain_store tzresult Lwt.t +val get_chain_store : store -> Chain_id.t -> chain_store tzresult Lwt.t (** [get_chain_store_opt global_store chain_id] optional version of [get_chain_store]. *) -val get_chain_store_opt : - store -> Tezos_crypto.Chain_id.t -> chain_store option Lwt.t +val get_chain_store_opt : store -> Chain_id.t -> chain_store option Lwt.t val make_pp_store : store -> (Format.formatter -> unit -> unit) Lwt.t @@ -307,49 +303,41 @@ module Block : sig (** [is_known_valid chain_store bh] tests that the block [bh] is known and valid in [chain_store] (i.e. the block is present in the block store). *) - val is_known_valid : chain_store -> Tezos_crypto.Block_hash.t -> bool Lwt.t + val is_known_valid : chain_store -> Block_hash.t -> bool Lwt.t (** [is_known_invalid chain_store bh] tests that the block [bh] is invalid in [chain_store] (i.e. the block is present in the invalid blocks file). *) - val is_known_invalid : chain_store -> Tezos_crypto.Block_hash.t -> bool Lwt.t + val is_known_invalid : chain_store -> Block_hash.t -> bool Lwt.t (** [is_known_prechecked chain_store bh] tests that the block [bh] is prechecked in [chain_store] (i.e. the block is present in the prechecked block cache). *) - val is_known_prechecked : - chain_store -> Tezos_crypto.Block_hash.t -> bool Lwt.t + val is_known_prechecked : chain_store -> Block_hash.t -> bool Lwt.t (** [is_known chain_store bh] tests that the block [bh] is either known valid or known invalid in [chain_store]. *) - val is_known : chain_store -> Tezos_crypto.Block_hash.t -> bool Lwt.t + val is_known : chain_store -> Block_hash.t -> bool Lwt.t (** [is_genesis chain_store bh] tests that the block [bh] is the genesis initialized in [chain_store]. *) - val is_genesis : chain_store -> Tezos_crypto.Block_hash.t -> bool + val is_genesis : chain_store -> Block_hash.t -> bool (** [validity chain_store bh] computes the {!Block_locator.validity} ([Unknown], [Known_valid] or [Known_invalid]) for the block [bh] in [chain_store]. *) - val validity : - chain_store -> Tezos_crypto.Block_hash.t -> Block_locator.validity Lwt.t + val validity : chain_store -> Block_hash.t -> Block_locator.validity Lwt.t (** [read_block chain_store ?distance bh] tries to read in the [chain_store] the block [bh] or the predecessing block at the offset [distance] of [bh]. By default, [distance] is 0.*) val read_block : - chain_store -> - ?distance:int -> - Tezos_crypto.Block_hash.t -> - block tzresult Lwt.t + chain_store -> ?distance:int -> Block_hash.t -> block tzresult Lwt.t (** [read_block_opt chain_store ?distance bh] optional version of [read_block]. *) val read_block_opt : - chain_store -> - ?distance:int -> - Tezos_crypto.Block_hash.t -> - block option Lwt.t + chain_store -> ?distance:int -> Block_hash.t -> block option Lwt.t (** [read_block_by_level chain_store level] reads in the [chain_store] the block at [level]. The block retrieved will be @@ -371,16 +359,13 @@ module Block : sig val read_block_metadata : ?distance:int -> chain_store -> - Tezos_crypto.Block_hash.t -> + Block_hash.t -> metadata option tzresult Lwt.t (** [read_block_metadata_opt ?distance chain_store bh] same as [read_block_metadata] but returns [None] in case of errors. *) val read_block_metadata_opt : - ?distance:int -> - chain_store -> - Tezos_crypto.Block_hash.t -> - metadata option Lwt.t + ?distance:int -> chain_store -> Block_hash.t -> metadata option Lwt.t (** [get_block_metadata chain_store block] reads in the [chain_store] the metadata associated to the [block]. Returns @@ -405,7 +390,7 @@ module Block : sig (** [read_predecessor_of_hash chain_store bh] reads in [chain_store] the predecessor's block of [bh]. *) val read_predecessor_of_hash : - chain_store -> Tezos_crypto.Block_hash.t -> block tzresult Lwt.t + chain_store -> Block_hash.t -> block tzresult Lwt.t (** [read_ancestor_hash chain_store ~distance bh] retrieves in the [chain_store] the hash of the ancestor of the block [bh] at @@ -413,31 +398,28 @@ module Block : sig val read_ancestor_hash : chain_store -> distance:int -> - Tezos_crypto.Block_hash.t -> - Tezos_crypto.Block_hash.t option tzresult Lwt.t + Block_hash.t -> + Block_hash.t option tzresult Lwt.t (** [read_ancestor_hash_opt chain_store ~distance bh] same as [read_ancestor_hash] but returns [None] on errors. *) val read_ancestor_hash_opt : - chain_store -> - distance:int -> - Tezos_crypto.Block_hash.t -> - Tezos_crypto.Block_hash.t option Lwt.t + chain_store -> distance:int -> Block_hash.t -> Block_hash.t option Lwt.t (** [read_ancestor_opt chain_store block] optional version of [read_ancestor]. *) val read_predecessor_of_hash_opt : - chain_store -> Tezos_crypto.Block_hash.t -> block option Lwt.t + chain_store -> Block_hash.t -> block option Lwt.t (** [read_prechecked_block chain_store bh] tries to read in the [chain_store]'s prechecked block cache the block [bh].*) val read_prechecked_block : - chain_store -> Tezos_crypto.Block_hash.t -> block tzresult Lwt.t + chain_store -> Block_hash.t -> block tzresult Lwt.t (** [read_prechecked_block_opt chain_store bh] optional version of [read_prechecked_block].*) val read_prechecked_block_opt : - chain_store -> Tezos_crypto.Block_hash.t -> block option Lwt.t + chain_store -> Block_hash.t -> block option Lwt.t (** [store_block chain_store ~block_header ~operations validation_result] stores in [chain_store] the block with its @@ -463,7 +445,7 @@ module Block : sig its [block_header] and [operations]. *) val store_prechecked_block : chain_store -> - hash:Tezos_crypto.Block_hash.t -> + hash:Block_hash.t -> block_header:Block_header.t -> operations:Operation.t trace trace -> unit tzresult Lwt.t @@ -493,50 +475,46 @@ module Block : sig val testchain_status : chain_store -> block -> - (Test_chain_status.t * Tezos_crypto.Block_hash.t option) tzresult Lwt.t + (Test_chain_status.t * Block_hash.t option) tzresult Lwt.t (** [protocol_hash_exn chain_store block] reads the protocol associated to [block] in its associated context. Fails when the context is unknown. *) - val protocol_hash_exn : - chain_store -> block -> Tezos_crypto.Protocol_hash.t Lwt.t + val protocol_hash_exn : chain_store -> block -> Protocol_hash.t Lwt.t (** [protocol_hash chain_store block] error monad version of [protocol_hash_exn]. *) - val protocol_hash : - chain_store -> block -> Tezos_crypto.Protocol_hash.t tzresult Lwt.t + val protocol_hash : chain_store -> block -> Protocol_hash.t tzresult Lwt.t (** [read_invalid_block_opt chain_store bh] reads in the [chain_store] the invalid block [bh] if it exists. *) val read_invalid_block_opt : - chain_store -> Tezos_crypto.Block_hash.t -> invalid_block option Lwt.t + chain_store -> Block_hash.t -> invalid_block option Lwt.t (** [read_invalid_blocks chain_store] returns the map of all invalid blocks of [chain_store]. *) - val read_invalid_blocks : - chain_store -> invalid_block Tezos_crypto.Block_hash.Map.t Lwt.t + val read_invalid_blocks : chain_store -> invalid_block Block_hash.Map.t Lwt.t (** [mark_invalid chain_store bh ~level errors] stores the block [bh] at [level] with the given [errors]. Fails when trying to mark the genesis block as invalid. *) val mark_invalid : chain_store -> - Tezos_crypto.Block_hash.t -> + Block_hash.t -> level:int32 -> error list -> unit tzresult Lwt.t (** [unmark_invalid chain_store bh] unmarks invalid the block [bh] in the [chain_store]. *) - val unmark_invalid : - chain_store -> Tezos_crypto.Block_hash.t -> unit tzresult Lwt.t + val unmark_invalid : chain_store -> Block_hash.t -> unit tzresult Lwt.t (** [descriptor block] returns the pair (hash x level) of [block]. *) val descriptor : block -> block_descriptor (** {3 Block field accessors} *) - val hash : block -> Tezos_crypto.Block_hash.t + val hash : block -> Block_hash.t val header : block -> Block_header.t @@ -548,30 +526,30 @@ module Block : sig val proto_level : block -> int - val predecessor : block -> Tezos_crypto.Block_hash.t + val predecessor : block -> Block_hash.t val timestamp : block -> Time.Protocol.t val validation_passes : block -> int - val operations_hash : block -> Tezos_crypto.Operation_list_list_hash.t + val operations_hash : block -> Operation_list_list_hash.t val fitness : block -> Fitness.t - val context_hash : block -> Tezos_crypto.Context_hash.t + val context_hash : block -> Context_hash.t val protocol_data : block -> bytes - val block_metadata_hash : block -> Tezos_crypto.Block_metadata_hash.t option + val block_metadata_hash : block -> Block_metadata_hash.t option val operations_metadata_hashes : - block -> Tezos_crypto.Operation_metadata_hash.t list list option + block -> Operation_metadata_hash.t list list option val operations_metadata_hashes_path : - block -> int -> Tezos_crypto.Operation_metadata_hash.t list option + block -> int -> Operation_metadata_hash.t list option val all_operations_metadata_hash : - block -> Tezos_crypto.Operation_metadata_list_list_hash.t option + block -> Operation_metadata_list_list_hash.t option (** {3 Block metadata field accessors} *) @@ -589,21 +567,16 @@ module Block : sig (** [operations_path block nth] computes the [nth] operations list of [block] along with the hash of all operations. *) val operations_path : - block -> - int -> - Operation.t list * Tezos_crypto.Operation_list_list_hash.path + block -> int -> Operation.t list * Operation_list_list_hash.path (** [operations_hashes_path block nth] computes the [nth] operations hash list of [block] along with the hash of all operations. *) val operations_hashes_path : - block -> - int -> - Tezos_crypto.Operation_hash.t list - * Tezos_crypto.Operation_list_list_hash.path + block -> int -> Operation_hash.t list * Operation_list_list_hash.path (** [all_operation_hashes block] computes the hash of all operations in [block]. *) - val all_operation_hashes : block -> Tezos_crypto.Operation_hash.t list list + val all_operation_hashes : block -> Operation_hash.t list list end (** The module for handling chain-related operations such as setting @@ -627,7 +600,7 @@ module Chain : sig val global_store : chain_store -> store (** [chain_id chain_store] returns chain id of [chain_store]. *) - val chain_id : chain_store -> Tezos_crypto.Chain_id.t + val chain_id : chain_store -> Chain_id.t (** [chain_dir chain_store] returns the path of directory of [chain_store].*) @@ -741,16 +714,12 @@ module Chain : sig which might happen when a new head concurrently arrives just before this operation being called. *) val set_mempool : - chain_store -> - head:Tezos_crypto.Block_hash.t -> - Mempool.t -> - unit tzresult Lwt.t + chain_store -> head:Block_hash.t -> Mempool.t -> unit tzresult Lwt.t (** [live_blocks chain_store] returns the set of previously computed live blocks for the current_head's [chain_store]. *) val live_blocks : - chain_store -> - (Tezos_crypto.Block_hash.Set.t * Tezos_crypto.Operation_hash.Set.t) Lwt.t + chain_store -> (Block_hash.Set.t * Operation_hash.Set.t) Lwt.t (** [compute_live_blocks ~block chain_store] computes the set of live blocks and live operations relative to [block]. Does nothing @@ -762,8 +731,7 @@ module Chain : sig val compute_live_blocks : chain_store -> block:Block.t -> - (Tezos_crypto.Block_hash.Set.t * Tezos_crypto.Operation_hash.Set.t) tzresult - Lwt.t + (Block_hash.Set.t * Operation_hash.Set.t) tzresult Lwt.t (** [set_head chain_store block] promotes the [block] as head of the [chain_store] and triggers an asynchronous store merge if a @@ -864,7 +832,7 @@ module Chain : sig (** [testchain_forked_block testchain] returns the hash of the forked block of its parent chain.*) - val testchain_forked_block : testchain -> Tezos_crypto.Block_hash.t + val testchain_forked_block : testchain -> Block_hash.t (** [testchain_store testchain] returns the chain store associated to this [testchain]. *) @@ -878,11 +846,11 @@ module Chain : sig registers this chain in the set of activated testchains. *) val fork_testchain : chain_store -> - testchain_id:Tezos_crypto.Chain_id.t -> + testchain_id:Chain_id.t -> forked_block:Block.t -> - genesis_hash:Tezos_crypto.Block_hash.t -> + genesis_hash:Block_hash.t -> genesis_header:Block_header.t -> - test_protocol:Tezos_crypto.Protocol_hash.t -> + test_protocol:Protocol_hash.t -> expiration:Time.Protocol.t -> testchain tzresult Lwt.t @@ -903,9 +871,7 @@ module Chain : sig (** [find_protocol chain_store ~protocol_level] returns the protocol with the level [protocol_level]. *) val find_protocol : - chain_store -> - protocol_level:int -> - Tezos_crypto.Protocol_hash.t option Lwt.t + chain_store -> protocol_level:int -> Protocol_hash.t option Lwt.t (** [all_protocol_levels chain_store] returns all the protocols registered in [chain_store]. *) @@ -924,7 +890,7 @@ module Chain : sig chain_store -> ?pred:Block.block -> ?protocol_level:int -> - Block.block * Tezos_crypto.Protocol_hash.t -> + Block.block * Protocol_hash.t -> unit tzresult Lwt.t (** [may_update_ancestor_protocol_level chain_store ~head] tries to @@ -951,8 +917,8 @@ module Chain : sig in [chain_store]. *) val set_rpc_directory : chain_store -> - protocol_hash:Tezos_crypto.Protocol_hash.t -> - next_protocol_hash:Tezos_crypto.Protocol_hash.t -> + protocol_hash:Protocol_hash.t -> + next_protocol_hash:Protocol_hash.t -> (chain_store * Block.t) Tezos_rpc.Directory.t -> unit Lwt.t @@ -960,7 +926,7 @@ module Chain : sig [callback] that may be triggered during a block store merge in order to garbage-collect old contexts. *) val register_gc_callback : - chain_store -> (Tezos_crypto.Context_hash.t -> unit tzresult Lwt.t) -> unit + chain_store -> (Context_hash.t -> unit tzresult Lwt.t) -> unit end (** [global_block_watcher global_store] instantiates a new block @@ -972,15 +938,15 @@ val global_block_watcher : module Protocol : sig (** [mem global_store ph] checks the existence of the protocol [ph] in [global_store]. *) - val mem : store -> Tezos_crypto.Protocol_hash.t -> bool + val mem : store -> Protocol_hash.t -> bool (** [all global_store ph] returns the set of all stored protocol in [global_store]. *) - val all : store -> Tezos_crypto.Protocol_hash.Set.t + val all : store -> Protocol_hash.Set.t (** [read global_store ph] reads the protocol [ph] from the [global_store]. Returns [None] if it does not exist. *) - val read : store -> Tezos_crypto.Protocol_hash.t -> Protocol.t option Lwt.t + val read : store -> Protocol_hash.t -> Protocol.t option Lwt.t (** [store global_store ph protocol] stores the [protocol] under the hash [ph] in the [global_store]. Returns [None] if it already @@ -989,22 +955,16 @@ module Protocol : sig {b Warning} No hash check is made, the caller must be careful when storing protocols. *) val store : - store -> - Tezos_crypto.Protocol_hash.t -> - Protocol.t -> - Tezos_crypto.Protocol_hash.t option Lwt.t + store -> Protocol_hash.t -> Protocol.t -> Protocol_hash.t option Lwt.t (** [store_raw global_store ph bytes] raw version of [store]. *) val store_raw : - store -> - Tezos_crypto.Protocol_hash.t -> - bytes -> - Tezos_crypto.Protocol_hash.t option Lwt.t + store -> Protocol_hash.t -> bytes -> Protocol_hash.t option Lwt.t (** [protocol_watcher global_store] instantiates a new protocol watcher in [global_store]. *) val protocol_watcher : - store -> Tezos_crypto.Protocol_hash.t Lwt_stream.t * Lwt_watcher.stopper + store -> Protocol_hash.t Lwt_stream.t * Lwt_watcher.stopper end (** The utility module used to traverse the chain. *) diff --git a/src/lib_store/unix/block_repr_unix.ml b/src/lib_store/unix/block_repr_unix.ml index 92ed377f8c88..bf9fc0c565b3 100644 --- a/src/lib_store/unix/block_repr_unix.ml +++ b/src/lib_store/unix/block_repr_unix.ml @@ -107,7 +107,7 @@ let pread_block fd ~file_offset = let raw_pruned_block_length bytes = (* Hypothesis: (Int32.to_int total_len + 4 <= Bytes.length bytes) *) let offset = 4 in - let offset = offset + Tezos_crypto.Block_hash.size (* hash *) in + let offset = offset + Block_hash.size (* hash *) in let header_length = Bytes.get_int32_be bytes offset in let offset = offset + 4 + Int32.to_int header_length in let operations_length = Bytes.get_int32_be bytes offset in @@ -115,7 +115,7 @@ let raw_pruned_block_length bytes = let offset = (* block metadata hash *) if Bytes.get_uint8 bytes offset = 0xff then - offset + 1 + Tezos_crypto.Block_metadata_hash.size + offset + 1 + Block_metadata_hash.size else offset + 1 in let offset = @@ -137,22 +137,21 @@ let prune_raw_block_bytes bytes = let hash_offset = 4 -let header_length_offset = hash_offset + Tezos_crypto.Block_hash.size +let header_length_offset = hash_offset + Block_hash.size let level_offset = header_length_offset + 4 let predecessor_offset = level_offset + 4 (* level *) + 1 (* proto_level *) let raw_get_block_hash block_bytes = - Tezos_crypto.Block_hash.of_bytes_exn - (Bytes.sub block_bytes hash_offset Tezos_crypto.Block_hash.size) + Block_hash.of_bytes_exn (Bytes.sub block_bytes hash_offset Block_hash.size) let raw_get_block_level block_bytes = Bytes.get_int32_be block_bytes level_offset let raw_get_block_predecessor block_bytes = - Tezos_crypto.Block_hash.of_bytes_exn - (Bytes.sub block_bytes predecessor_offset Tezos_crypto.Block_hash.size) + Block_hash.of_bytes_exn + (Bytes.sub block_bytes predecessor_offset Block_hash.size) let raw_get_last_allowed_fork_level block_bytes total_block_length = let header_length = Bytes.get_int32_be block_bytes header_length_offset in @@ -168,7 +167,7 @@ let raw_get_last_allowed_fork_level block_bytes total_block_length = let operation_metadata_hashes_offset = (* block metadata hash *) if Bytes.get_uint8 block_bytes block_metadata_hash_offset = 0xff then - block_metadata_hash_offset + 1 + Tezos_crypto.Block_metadata_hash.size + block_metadata_hash_offset + 1 + Block_metadata_hash.size else block_metadata_hash_offset + 1 in let metadata_offset = diff --git a/src/lib_store/unix/block_repr_unix.mli b/src/lib_store/unix/block_repr_unix.mli index 1ee336e92f52..58826411ec3d 100644 --- a/src/lib_store/unix/block_repr_unix.mli +++ b/src/lib_store/unix/block_repr_unix.mli @@ -63,7 +63,7 @@ val prune_raw_block_bytes : bytes -> int (** [raw_get_block_hash block_buffer] introspects the hash of the block contained in [block_buffer]. *) -val raw_get_block_hash : bytes -> Tezos_crypto.Block_hash.t +val raw_get_block_hash : bytes -> Block_hash.t (** [raw_get_block_level block_buffer] introspects the level of the block contained in [block_buffer]. *) @@ -71,7 +71,7 @@ val raw_get_block_level : bytes -> int32 (** [raw_get_block_predecessor block_buffer] introspects the predecessor's hash of the block contained in [block_buffer]. *) -val raw_get_block_predecessor : bytes -> Tezos_crypto.Block_hash.t +val raw_get_block_predecessor : bytes -> Block_hash.t (** [raw_get_last_allowed_fork_level block_buffer] introspects the last allowed fork level of the block's metadata contained in diff --git a/src/lib_store/unix/block_store.ml b/src/lib_store/unix/block_store.ml index dde141ec7607..bb6f021353c5 100644 --- a/src/lib_store/unix/block_store.ml +++ b/src/lib_store/unix/block_store.ml @@ -44,8 +44,7 @@ type block_store = { savepoint : block_descriptor Stored_data.t; status_data : status Stored_data.t; block_cache : Block_repr.t Block_lru_cache.t; - mutable gc_callback : - (Tezos_crypto.Context_hash.t -> unit tzresult Lwt.t) option; + mutable gc_callback : (Context_hash.t -> unit tzresult Lwt.t) option; merge_mutex : Lwt_mutex.t; merge_scheduler : Lwt_idle_waiter.t; (* Target level x Merging thread *) @@ -54,7 +53,7 @@ type block_store = { type t = block_store -type key = Block of (Tezos_crypto.Block_hash.t * int) +type key = Block of (Block_hash.t * int) let status_encoding = let open Data_encoding in @@ -169,7 +168,7 @@ let compute_predecessors block_store block = | Some pred' -> loop (pred' :: predecessors_acc) pred' (dist + 1) in let predecessor = predecessor block in - if Tezos_crypto.Block_hash.equal block.hash predecessor then + if Block_hash.equal block.hash predecessor then (* genesis *) Lwt.return [block.hash] else @@ -226,9 +225,7 @@ let mem block_store key = match o with | None -> return_false | Some predecessor_hash - when Tezos_crypto.Block_hash.equal - block_store.genesis_block.hash - predecessor_hash -> + when Block_hash.equal block_store.genesis_block.hash predecessor_hash -> return_true | Some predecessor_hash -> let*! is_known_in_floating = @@ -251,11 +248,8 @@ let read_block ~read_metadata block_store key_kind = match o with | None -> return_none | Some adjusted_hash -> - if - Tezos_crypto.Block_hash.equal - block_store.genesis_block.hash - adjusted_hash - then return_some block_store.genesis_block + if Block_hash.equal block_store.genesis_block.hash adjusted_hash then + return_some block_store.genesis_block else let fetch_block adjusted_hash = (* First look in the floating stores *) @@ -297,11 +291,8 @@ let read_block_metadata block_store key_kind = match o with | None -> return_none | Some adjusted_hash -> ( - if - Tezos_crypto.Block_hash.equal - block_store.genesis_block.hash - adjusted_hash - then return (Block_repr.metadata block_store.genesis_block) + if Block_hash.equal block_store.genesis_block.hash adjusted_hash then + return (Block_repr.metadata block_store.genesis_block) else (* First look in the floating stores *) let*! o = @@ -876,9 +867,7 @@ let update_floating_stores block_store ~history_mode ~ro_store ~rw_store {stores}. cementing_highwatermark < b.lafl <= new_head_lafl HYPOTHESIS: all blocks at a given level have the same lafl. *) - let visited = - ref (Tezos_crypto.Block_hash.Set.singleton (Block_repr.hash lafl_block)) - in + let visited = ref (Block_hash.Set.singleton (Block_repr.hash lafl_block)) in let blocks_lafl = ref BlocksLAFL.empty in let*! () = Store_events.(emit start_retreiving_cycles) () in let* () = @@ -911,8 +900,8 @@ let update_floating_stores block_store ~history_mode ~ro_store ~rw_store Block_repr_unix.raw_get_block_predecessor block_bytes in let block_hash = Block_repr_unix.raw_get_block_hash block_bytes in - if Tezos_crypto.Block_hash.Set.mem block_predecessor !visited then ( - visited := Tezos_crypto.Block_hash.Set.add block_hash !visited ; + if Block_hash.Set.mem block_predecessor !visited then ( + visited := Block_hash.Set.add block_hash !visited ; let*! predecessors = let*! pred_opt = Floating_block_store.find_predecessors store block_hash @@ -1237,7 +1226,7 @@ let may_trigger_gc block_store history_mode ~previous_savepoint ~new_savepoint = let savepoint_hash = fst new_savepoint in if History_mode.(equal history_mode Archive) - || Tezos_crypto.Block_hash.(savepoint_hash = fst previous_savepoint) + || Block_hash.(savepoint_hash = fst previous_savepoint) then (* No GC required *) return_unit else match block_store.gc_callback with diff --git a/src/lib_store/unix/block_store.mli b/src/lib_store/unix/block_store.mli index c0ffdfe1eab5..c6d21cc09bed 100644 --- a/src/lib_store/unix/block_store.mli +++ b/src/lib_store/unix/block_store.mli @@ -159,7 +159,7 @@ type t = block_store A block key may represent an invalid block (wrong hash and/or offset) as it is not ensured to be valid by construction.*) -type key = Block of (Tezos_crypto.Block_hash.t * int) +type key = Block of (Block_hash.t * int) (** The status of the merging thread *) type merge_status = Not_running | Running | Merge_failed of tztrace @@ -206,8 +206,7 @@ val mem : block_store -> key -> bool tzresult Lwt.t (** [get_hash block_store key] retrieves the hash corresponding to the given [key] in [block_store]. Return [None] if the block is unknown. *) -val get_hash : - block_store -> key -> Tezos_crypto.Block_hash.t option tzresult Lwt.t +val get_hash : block_store -> key -> Block_hash.t option tzresult Lwt.t (** [read_block ~read_metadata block_store key] reads the block [key] in [block_store] if present. Return [None] if the block is @@ -340,7 +339,7 @@ val load : that may be triggered during a block store merge in order to garbage-collect old contexts. *) val register_gc_callback : - block_store -> (Tezos_crypto.Context_hash.t -> unit tzresult Lwt.t) -> unit + block_store -> (Context_hash.t -> unit tzresult Lwt.t) -> unit (** [close block_store] closes the [block_store] and every underlying opened stores. diff --git a/src/lib_store/unix/cemented_block_store.ml b/src/lib_store/unix/cemented_block_store.ml index 1fb654bcfe91..25d42bfb7cdb 100644 --- a/src/lib_store/unix/cemented_block_store.ml +++ b/src/lib_store/unix/cemented_block_store.ml @@ -61,8 +61,7 @@ type t = { type chunk_iterator = { chunk_length : int; - reading_sequence : - (Tezos_crypto.Block_hash.t * int * bytes) tzresult Lwt.t Seq.t; + reading_sequence : (Block_hash.t * int * bytes) tzresult Lwt.t Seq.t; } let make_chunk_iterator bl = diff --git a/src/lib_store/unix/cemented_block_store.mli b/src/lib_store/unix/cemented_block_store.mli index 9144bb59e7e8..ad6d6b70ed29 100644 --- a/src/lib_store/unix/cemented_block_store.mli +++ b/src/lib_store/unix/cemented_block_store.mli @@ -176,17 +176,17 @@ val find_block_file : t -> int32 -> cemented_blocks_file option (** [is_cemented cemented_store block_hash] checks if the [block_hash] is stored in the [cemented_store]. *) -val is_cemented : t -> Tezos_crypto.Block_hash.t -> bool +val is_cemented : t -> Block_hash.t -> bool (** [get_cemented_block_level cemented_store block_hash] returns the level of the [block_hash] if present in [cemented_store]. Returns [None] otherwise. *) -val get_cemented_block_level : t -> Tezos_crypto.Block_hash.t -> int32 option +val get_cemented_block_level : t -> Block_hash.t -> int32 option (** [get_cemented_block_hash cemented_store block_level] returns the hash of the block at [block_level] if present in [cemented_store]. Returns [None] otherwise. *) -val get_cemented_block_hash : t -> int32 -> Tezos_crypto.Block_hash.t option +val get_cemented_block_hash : t -> int32 -> Block_hash.t option (** [read_block_metadata cemented_store block_level] returns the metadata of the block at [block_level] if present in @@ -227,15 +227,14 @@ val get_cemented_block_by_level : val get_cemented_block_by_hash : read_metadata:bool -> t -> - Tezos_crypto.Block_hash.t -> + Block_hash.t -> Block_repr.block option tzresult Lwt.t (** The type used to describe reading sequences used to perform buffered block cementing. *) type chunk_iterator = { chunk_length : int; - reading_sequence : - (Tezos_crypto.Block_hash.t * int * bytes) tzresult Lwt.t Seq.t; + reading_sequence : (Block_hash.t * int * bytes) tzresult Lwt.t Seq.t; } (** [make_chunk_iterator bl] is an utility function that transforms a @@ -296,6 +295,6 @@ val raw_iter_cemented_file : snapshot imports. *) val check_indexes_consistency : ?post_step:(unit -> unit Lwt.t) -> - ?genesis_hash:Tezos_crypto.Block_hash.t -> + ?genesis_hash:Block_hash.t -> t -> unit tzresult Lwt.t diff --git a/src/lib_store/unix/consistency.ml b/src/lib_store/unix/consistency.ml index a201dc1b023e..64f697f8c919 100644 --- a/src/lib_store/unix/consistency.ml +++ b/src/lib_store/unix/consistency.ml @@ -166,9 +166,7 @@ let check_consistency chain_dir genesis = let*! genesis_block = Stored_data.get genesis_data in let* () = fail_unless - (Tezos_crypto.Block_hash.equal - (Block_repr.hash genesis_block) - genesis.Genesis.block) + (Block_hash.equal (Block_repr.hash genesis_block) genesis.Genesis.block) (Inconsistent_genesis {expected = genesis.block; got = Block_repr.hash genesis_block}) in @@ -316,7 +314,7 @@ let fix_head chain_dir block_store genesis_block = cemented block known. *) if cemented_block_files <> [] - && Tezos_crypto.Block_hash.equal + && Block_hash.equal (Block_repr.hash genesis_block) (Block_repr.hash floating_head) then @@ -740,7 +738,7 @@ let check_block_protocol_hash context_index ~expected block = protect @@ fun () -> let*! ctxt = Context.checkout_exn context_index (Block_repr.context block) in let*! got = Context.get_protocol ctxt in - return Tezos_crypto.Protocol_hash.(got = expected) + return Protocol_hash.(got = expected) (** Look into the cemented store for the lowest block with an associated proto level that is above the savepoint. *) @@ -1104,7 +1102,7 @@ let fix_chain_state chain_dir block_store ~head ~cementing_highwatermark let* () = Stored_data.write_file (Naming.invalid_blocks_file chain_dir) - Tezos_crypto.Block_hash.Map.empty + Block_hash.Map.empty in let* () = Stored_data.write_file (Naming.forked_chains_file chain_dir) forked_chains @@ -1145,12 +1143,12 @@ let infer_history_mode chain_dir block_store genesis caboose savepoint = in let history_mode = (* Caboose is not genesis: we sure are in rolling*) - if not (Tezos_crypto.Block_hash.equal (fst caboose) genesis.Genesis.block) - then History_mode.Rolling offset + if not (Block_hash.equal (fst caboose) genesis.Genesis.block) then + History_mode.Rolling offset else if (* Caboose is genesis and savepoint is not genesis: we can be in both rolling and full. We choose full as the less destructive. *) - not (Tezos_crypto.Block_hash.equal (fst savepoint) genesis.block) + not (Block_hash.equal (fst savepoint) genesis.block) then Full offset else if (* Caboose is genesis and savepoint is genesis and there are as @@ -1273,7 +1271,7 @@ let fix_consistency ?history_mode chain_dir context_index genesis = ~savepoint ~caboose ~alternate_heads:[] - ~forked_chains:Tezos_crypto.Chain_id.Map.empty + ~forked_chains:Chain_id.Map.empty ~protocol_levels ~chain_config ~genesis diff --git a/src/lib_store/unix/dune b/src/lib_store/unix/dune index e0313148e964..73ffdff66854 100644 --- a/src/lib_store/unix/dune +++ b/src/lib_store/unix/dune @@ -20,6 +20,7 @@ tezos-protocol-updater tezos-stdlib-unix tezos-stdlib + tezos-crypto lwt-watcher aches aches-lwt @@ -39,7 +40,8 @@ -open Tezos_validation -open Tezos_protocol_updater -open Tezos_stdlib_unix - -open Tezos_stdlib) + -open Tezos_stdlib + -open Tezos_crypto) (modules block_repr_unix block_store @@ -59,6 +61,7 @@ (libraries tezos-base tezos-stdlib-unix + tezos-crypto tezos-shell-services tezos-protocol-updater tezos-validation @@ -70,6 +73,7 @@ -open Tezos_base -open Tezos_base.TzPervasives -open Tezos_stdlib_unix + -open Tezos_crypto -open Tezos_shell_services -open Tezos_protocol_updater -open Tezos_validation @@ -85,6 +89,7 @@ (libraries tezos-base tezos-stdlib-unix + tezos-crypto tezos-shell-services tezos-context tezos-validation @@ -95,6 +100,7 @@ -open Tezos_base -open Tezos_base.TzPervasives -open Tezos_stdlib_unix + -open Tezos_crypto -open Tezos_shell_services -open Tezos_context -open Tezos_validation diff --git a/src/lib_store/unix/floating_block_index.ml b/src/lib_store/unix/floating_block_index.ml index b3c7113f4375..67735faa0e59 100644 --- a/src/lib_store/unix/floating_block_index.ml +++ b/src/lib_store/unix/floating_block_index.ml @@ -27,19 +27,19 @@ time: if a block has fewer than 12 predecessors then it is padded so its size remain constant. *) module Block_info = struct - type t = {offset : int; predecessors : Tezos_crypto.Block_hash.t list} + type t = {offset : int; predecessors : Block_hash.t list} let max_predecessors = 12 let encoded_list_size = - let r = max_predecessors * Tezos_crypto.Block_hash.size (* uint16 *) in + let r = max_predecessors * Block_hash.size (* uint16 *) in assert (r < 1 lsl 16) ; r let encoded_size = 8 + 1 + encoded_list_size (* Format: - (8) + (1) + (list_size * Tezos_crypto.Block_hash.size) *) + (8) + (1) + (list_size * Block_hash.size) *) let t = let open Repr in @@ -58,11 +58,11 @@ module Block_info = struct (fun i h -> (* Start reading after the (8) + (1) *) Bytes.blit - (Tezos_crypto.Block_hash.to_bytes h) + (Block_hash.to_bytes h) 0 bytes - (8 + 1 + (i * Tezos_crypto.Block_hash.size)) - Tezos_crypto.Block_hash.size) + (8 + 1 + (i * Block_hash.size)) + Block_hash.size) v.predecessors ; Bytes.unsafe_to_string bytes @@ -80,13 +80,13 @@ module Block_info = struct current_offset := !current_offset + 1 ; let predecessors = ref [] in let limit = !current_offset in - current_offset := limit + ((list_size - 1) * Tezos_crypto.Block_hash.size) ; + current_offset := limit + ((list_size - 1) * Block_hash.size) ; while !current_offset >= limit do predecessors := - (String.sub str !current_offset Tezos_crypto.Block_hash.size - |> Tezos_crypto.Block_hash.of_string_exn) + (String.sub str !current_offset Block_hash.size + |> Block_hash.of_string_exn) :: !predecessors ; - current_offset := !current_offset - Tezos_crypto.Block_hash.size + current_offset := !current_offset - Block_hash.size done ; {offset; predecessors = !predecessors} @@ -96,9 +96,7 @@ module Block_info = struct fmt "@[offset: %d, predecessors : [ @[%a @]]@]" v.offset - (pp_print_list - ~pp_sep:(fun fmt () -> fprintf fmt " ;@,") - Tezos_crypto.Block_hash.pp) + (pp_print_list ~pp_sep:(fun fmt () -> fprintf fmt " ;@,") Block_hash.pp) v.predecessors end diff --git a/src/lib_store/unix/floating_block_index.mli b/src/lib_store/unix/floating_block_index.mli index 3645bf29508f..17b9afa5f040 100644 --- a/src/lib_store/unix/floating_block_index.mli +++ b/src/lib_store/unix/floating_block_index.mli @@ -37,8 +37,7 @@ module Block_info : sig (** The type for storing the block's info. *) type t = { offset : int; (** offset in the file *) - predecessors : Tezos_crypto.Block_hash.t list; - (** predecessors of the block *) + predecessors : Block_hash.t list; (** predecessors of the block *) } (** Pretty-printer for {!t} *) @@ -46,8 +45,5 @@ module Block_info : sig end (** Key/value index associated to a floating block store where the key - is a {!Tezos_crypto.Block_hash.t} and the value is {!Block_info.t}. *) -include - Index.S - with type key = Tezos_crypto.Block_hash.t - and type value = Block_info.t + is a {!Block_hash.t} and the value is {!Block_info.t}. *) +include Index.S with type key = Block_hash.t and type value = Block_info.t diff --git a/src/lib_store/unix/floating_block_store.ml b/src/lib_store/unix/floating_block_store.ml index a648198d2c37..4917c7e4f3b4 100644 --- a/src/lib_store/unix/floating_block_store.ml +++ b/src/lib_store/unix/floating_block_store.ml @@ -128,7 +128,7 @@ let append_block ?(flush = true) ?(log_metrics = false) floating_store return_unit) let append_all floating_store - (blocks : (Tezos_crypto.Block_hash.t list * Block_repr.t) Seq.t) = + (blocks : (Block_hash.t list * Block_repr.t) Seq.t) = let open Lwt_result_syntax in Lwt_idle_waiter.force_idle floating_store.scheduler (fun () -> let*! eof_offset = Lwt_unix.lseek floating_store.fd 0 Unix.SEEK_END in diff --git a/src/lib_store/unix/floating_block_store.mli b/src/lib_store/unix/floating_block_store.mli index 0d4c04056763..4425e89b121c 100644 --- a/src/lib_store/unix/floating_block_store.mli +++ b/src/lib_store/unix/floating_block_store.mli @@ -27,7 +27,7 @@ The floating block store is an append-only store where blocks are stored arbitrarily. This structure possess an indexed map - {!Tezos_crypto.Block_hash.t} -> (offset × predecessors) which points to its + {!Block_hash.t} -> (offset × predecessors) which points to its offset in the associated file along with an exponential list of predecessors block hashes (as implemented in {!Block_store.compute_predecessors}). The structure @@ -77,26 +77,23 @@ val kind : t -> floating_kind (** [mem floating_store hash] tests whether [hash] is stored in [floating_store]. *) -val mem : t -> Tezos_crypto.Block_hash.t -> bool Lwt.t +val mem : t -> Block_hash.t -> bool Lwt.t (** [find_predecessors floating_store block_hash] reads from the index the list of [block_hash]'s predecessors if the block is stored in [floating_store], returns [None] otherwise. *) -val find_predecessors : - t -> Tezos_crypto.Block_hash.t -> Tezos_crypto.Block_hash.t list option Lwt.t +val find_predecessors : t -> Block_hash.t -> Block_hash.t list option Lwt.t (** [read_block floating_store hash] reads from the file the block of [hash] if the block is stored in [floating_store], returns [None] otherwise. *) -val read_block : t -> Tezos_crypto.Block_hash.t -> Block_repr.t option Lwt.t +val read_block : t -> Block_hash.t -> Block_repr.t option Lwt.t (** [read_block_and_predecessors floating_store hash] same as [read_block] but also returns the block's predecessors. Returns [None] if it fails to resolve the given [hash].*) val read_block_and_predecessors : - t -> - Tezos_crypto.Block_hash.t -> - (Block_repr.t * Tezos_crypto.Block_hash.t list) option Lwt.t + t -> Block_hash.t -> (Block_repr.t * Block_hash.t list) option Lwt.t (** [append_block floating_store ?flush preds block] stores the [block] in [floating_store] updating its index with the given @@ -107,7 +104,7 @@ val append_block : ?flush:bool -> ?log_metrics:bool -> t -> - Tezos_crypto.Block_hash.t trace -> + Block_hash.t trace -> Block_repr.block -> unit tzresult Lwt.t @@ -115,9 +112,7 @@ val append_block : (predecessors × blocks) in [floating_store] updating its index accordingly. *) val append_all : - t -> - (Tezos_crypto.Block_hash.t list * Block_repr.t) Seq.t -> - unit tzresult Lwt.t + t -> (Block_hash.t list * Block_repr.t) Seq.t -> unit tzresult Lwt.t (** [iter_s_raw_fd f fd] unsafe sequential iterator on a file descriptor [fd]. Applies [f] on every block encountered. @@ -139,7 +134,7 @@ val fold_left_s : element. The function [f] is given the last read block along with its predecessors. *) val fold_left_with_pred_s : - ('a -> Block_repr.block * Tezos_crypto.Block_hash.t list -> 'a tzresult Lwt.t) -> + ('a -> Block_repr.block * Block_hash.t list -> 'a tzresult Lwt.t) -> 'a -> t -> 'a tzresult Lwt.t @@ -152,7 +147,7 @@ val iter_s : (Block_repr.t -> unit tzresult Lwt.t) -> t -> unit tzresult Lwt.t [floating_store]. Applies [f] on every block with its predecessors read. *) val iter_with_pred_s : - (Block_repr.t * Tezos_crypto.Block_hash.t list -> unit tzresult Lwt.t) -> + (Block_repr.t * Block_hash.t list -> unit tzresult Lwt.t) -> t -> unit tzresult Lwt.t @@ -203,16 +198,14 @@ val fix_integrity : appends a block with its [hash] in [dst_store] contained in the [buffer]. *) val raw_append : - t -> - Tezos_crypto.Block_hash.t * bytes * int * Tezos_crypto.Block_hash.t list -> - unit tzresult Lwt.t + t -> Block_hash.t * bytes * int * Block_hash.t list -> unit tzresult Lwt.t (** [raw_copy_all src_stores block_hashes dst_store] retrieves [block_hashes] from [src_stores] and copy them (without decoding) to [dst_store] with a buffering mechanism. *) val raw_copy_all : src_floating_stores:t list -> - block_hashes:Tezos_crypto.Block_hash.t list -> + block_hashes:Block_hash.t list -> dst_floating_store:t -> unit tzresult Lwt.t @@ -229,8 +222,8 @@ val raw_copy_all : during the iteration. *) val raw_retrieve_blocks_seq : src_floating_stores:t list -> - block_hashes:Tezos_crypto.Block_hash.t list -> - (Tezos_crypto.Block_hash.t * int * bytes) tzresult Lwt.t Seq.t + block_hashes:Block_hash.t list -> + (Block_hash.t * int * bytes) tzresult Lwt.t Seq.t (** [raw_iterate f store] iterate over all blocks in a store and calling [f] providing it with a [buffer] and the diff --git a/src/lib_store/unix/protocol_store.ml b/src/lib_store/unix/protocol_store.ml index edd4706cf9be..a094ef4919db 100644 --- a/src/lib_store/unix/protocol_store.ml +++ b/src/lib_store/unix/protocol_store.ml @@ -25,11 +25,10 @@ type t = { protocol_store_dir : [`Protocol_dir] Naming.directory; - mutable protocols : Tezos_crypto.Protocol_hash.Set.t; + mutable protocols : Protocol_hash.Set.t; } -let mem t protocol_hash = - Tezos_crypto.Protocol_hash.Set.mem protocol_hash t.protocols +let mem t protocol_hash = Protocol_hash.Set.mem protocol_hash t.protocols let all {protocols; _} = protocols @@ -48,8 +47,7 @@ let raw_store store protocol_hash bytes = in let* () = Lwt_utils_unix.write_bytes fd bytes in let* _ = Lwt_utils_unix.safe_close fd in - store.protocols <- - Tezos_crypto.Protocol_hash.Set.add protocol_hash store.protocols ; + store.protocols <- Protocol_hash.Set.add protocol_hash store.protocols ; Lwt.return_some protocol_hash let store store protocol_hash protocol = @@ -80,12 +78,11 @@ let init store_dir = Lwt.try_bind (fun () -> Lwt_unix.readdir dir) (fun file -> - match Tezos_crypto.Protocol_hash.of_b58check_opt file with - | Some protocol_hash -> - loop (Tezos_crypto.Protocol_hash.Set.add protocol_hash set) + match Protocol_hash.of_b58check_opt file with + | Some protocol_hash -> loop (Protocol_hash.Set.add protocol_hash set) | None -> loop set) (function End_of_file -> Lwt.return set | _ -> loop set) in - let* protocols = loop Tezos_crypto.Protocol_hash.Set.empty in + let* protocols = loop Protocol_hash.Set.empty in let* () = Lwt_unix.closedir dir in Lwt.return {protocol_store_dir; protocols} diff --git a/src/lib_store/unix/protocol_store.mli b/src/lib_store/unix/protocol_store.mli index 505203ee6bac..c8d281e5e33e 100644 --- a/src/lib_store/unix/protocol_store.mli +++ b/src/lib_store/unix/protocol_store.mli @@ -30,33 +30,25 @@ type t (** [mem pstore proto_hash] tests the existence of the protocol indexed by [proto_hash] in the store. *) -val mem : t -> Tezos_crypto.Protocol_hash.t -> bool +val mem : t -> Protocol_hash.t -> bool (** [all pstore] returns the set of all stored protocols in [pstore]. *) -val all : t -> Tezos_crypto.Protocol_hash.Set.t +val all : t -> Protocol_hash.Set.t (** [raw_store pstore proto_hash proto_bytes] stores on disk the protocol [proto_bytes] (encoded bytes) indexed as [proto_hash]. Returns [None] if the protocol already exists. *) -val raw_store : - t -> - Tezos_crypto.Protocol_hash.t -> - bytes -> - Tezos_crypto.Protocol_hash.t option Lwt.t +val raw_store : t -> Protocol_hash.t -> bytes -> Protocol_hash.t option Lwt.t (** [store pstore proto_hash protocol] stores on disk the protocol [protocol] indexed as [proto_hash]. Returns [None] if the protocol already exists. *) -val store : - t -> - Tezos_crypto.Protocol_hash.t -> - Protocol.t -> - Tezos_crypto.Protocol_hash.t option Lwt.t +val store : t -> Protocol_hash.t -> Protocol.t -> Protocol_hash.t option Lwt.t (** [read pstore proto_hash] reads from [pstore] and returns the protocol indexed by [proto_hash]. Returns [None] if the protocol cannot be read. *) -val read : t -> Tezos_crypto.Protocol_hash.t -> Protocol.t option Lwt.t +val read : t -> Protocol_hash.t -> Protocol.t option Lwt.t (** [init store_dir] creates a store relatively to [store_dir] path or loads it if it already exists. *) diff --git a/src/lib_store/unix/reconstruction.ml b/src/lib_store/unix/reconstruction.ml index 8dae68df1add..e7419926917b 100644 --- a/src/lib_store/unix/reconstruction.ml +++ b/src/lib_store/unix/reconstruction.ml @@ -25,9 +25,8 @@ type failure_kind = | Nothing_to_reconstruct - | Context_hash_mismatch of - Block_header.t * Tezos_crypto.Context_hash.t * Tezos_crypto.Context_hash.t - | Cannot_read_block_hash of Tezos_crypto.Block_hash.t + | Context_hash_mismatch of Block_header.t * Context_hash.t * Context_hash.t + | Cannot_read_block_hash of Block_hash.t | Cannot_read_block_level of Int32.t let failure_kind_encoding = @@ -45,15 +44,15 @@ let failure_kind_encoding = ~title:"context_hash_mismatch" (obj3 (req "block_header" Block_header.encoding) - (req "expected" Tezos_crypto.Context_hash.encoding) - (req "got" Tezos_crypto.Context_hash.encoding)) + (req "expected" Context_hash.encoding) + (req "got" Context_hash.encoding)) (function | Context_hash_mismatch (h, e, g) -> Some (h, e, g) | _ -> None) (fun (h, e, g) -> Context_hash_mismatch (h, e, g)); case (Tag 2) ~title:"cannot_read_block_hash" - Tezos_crypto.Block_hash.encoding + Block_hash.encoding (function Cannot_read_block_hash h -> Some h | _ -> None) (fun h -> Cannot_read_block_hash h); case @@ -71,19 +70,15 @@ let failure_kind_pp ppf = function ppf "resulting context hash for block %a (level %ld) does not match. \ Context hash expected %a, got %a" - Tezos_crypto.Block_hash.pp + Block_hash.pp (Block_header.hash h) h.shell.level - Tezos_crypto.Context_hash.pp + Context_hash.pp e - Tezos_crypto.Context_hash.pp + Context_hash.pp g | Cannot_read_block_hash h -> - Format.fprintf - ppf - "Unexpected missing block in store: %a" - Tezos_crypto.Block_hash.pp - h + Format.fprintf ppf "Unexpected missing block in store: %a" Block_hash.pp h | Cannot_read_block_level l -> Format.fprintf ppf "Unexpected missing block in store at level %ld" l @@ -165,13 +160,13 @@ let check_context_hash_consistency block_validation_result block_header = let expected = block_header.Block_header.shell.context in let got = block_validation_result.Block_validation.context_hash in fail_unless - (Tezos_crypto.Context_hash.equal expected got) + (Context_hash.equal expected got) (Reconstruction_failure (Context_hash_mismatch (block_header, expected, got))) (* We assume that the given list is not empty. *) let compute_block_metadata_hash block_metadata = - Some (Tezos_crypto.Block_metadata_hash.hash_bytes [block_metadata]) + Some (Block_metadata_hash.hash_bytes [block_metadata]) let split_operations_metadata = function | Block_validation.No_metadata_hash metadata -> (metadata, None) @@ -191,8 +186,8 @@ let compute_all_operations_metadata_hash block = else Option.map (fun ll -> - Tezos_crypto.Operation_metadata_list_list_hash.compute - (List.map Tezos_crypto.Operation_metadata_list_hash.compute ll)) + Operation_metadata_list_list_hash.compute + (List.map Operation_metadata_list_hash.compute ll)) (Block_repr.operations_metadata_hashes block) let apply_context context_index chain_id ~user_activated_upgrades diff --git a/src/lib_store/unix/reconstruction.mli b/src/lib_store/unix/reconstruction.mli index e55f0cac4a09..05a3b296ca49 100644 --- a/src/lib_store/unix/reconstruction.mli +++ b/src/lib_store/unix/reconstruction.mli @@ -53,9 +53,8 @@ type failure_kind = | Nothing_to_reconstruct - | Context_hash_mismatch of - Block_header.t * Tezos_crypto.Context_hash.t * Tezos_crypto.Context_hash.t - | Cannot_read_block_hash of Tezos_crypto.Block_hash.t + | Context_hash_mismatch of Block_header.t * Context_hash.t * Context_hash.t + | Cannot_read_block_hash of Block_hash.t | Cannot_read_block_level of Int32.t type error += Reconstruction_failure of failure_kind diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index 8a9247f683a1..bc800b9c9110 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -32,7 +32,7 @@ type error += stored : History_mode.t; } | Invalid_export_block of { - block : Tezos_crypto.Block_hash.t option; + block : Block_hash.t option; reason : [ `Pruned | `Pruned_pred @@ -45,15 +45,15 @@ type error += | Invalid_export_path of string | Snapshot_file_not_found of string | Inconsistent_protocol_hash of { - expected : Tezos_crypto.Protocol_hash.t; - got : Tezos_crypto.Protocol_hash.t; + expected : Protocol_hash.t; + got : Protocol_hash.t; } | Inconsistent_context_hash of { - expected : Tezos_crypto.Context_hash.t; - got : Tezos_crypto.Context_hash.t; + expected : Context_hash.t; + got : Context_hash.t; } - | Inconsistent_context of Tezos_crypto.Context_hash.t - | Cannot_decode_protocol of Tezos_crypto.Protocol_hash.t + | Inconsistent_context of Context_hash.t + | Cannot_decode_protocol of Protocol_hash.t | Cannot_write_metadata of string | Cannot_read of { kind : @@ -74,7 +74,7 @@ type error += | Missing_cemented_file of string | Corrupted_floating_store | Invalid_protocol_file of string - | Target_block_validation_failed of Tezos_crypto.Block_hash.t * string + | Target_block_validation_failed of Block_hash.t * string | Directory_already_exists of string | Empty_floating_store | Cannot_remove_tmp_export_directory of string @@ -87,10 +87,9 @@ type error += requested : History_mode.t; stored : History_mode.t; } - | Inconsistent_imported_block of - Tezos_crypto.Block_hash.t * Tezos_crypto.Block_hash.t + | Inconsistent_imported_block of Block_hash.t * Block_hash.t | Wrong_snapshot_file of {filename : string} - | Invalid_chain_store_export of Tezos_crypto.Chain_id.t * string + | Invalid_chain_store_export of Chain_id.t * string let () = let open Data_encoding in @@ -128,7 +127,7 @@ let () = "The selected block %a is invalid: %s." (Format.pp_print_option ~none:(fun fmt () -> Format.fprintf fmt "(n/a)") - Tezos_crypto.Block_hash.pp) + Block_hash.pp) hash (match reason with | `Pruned -> "the block is too old and has been pruned" @@ -139,7 +138,7 @@ let () = | `Caboose -> "the caboose block is not a valid export point" | `Not_enough_pred -> "not enough of the block's predecessors are known")) (obj2 - (opt "block" Tezos_crypto.Block_hash.encoding) + (opt "block" Block_hash.encoding) (req "reason" (string_enum @@ -187,13 +186,13 @@ let () = Format.fprintf ppf "Inconsistent protocol_hash. Expected: %a, got %a." - Tezos_crypto.Protocol_hash.pp + Protocol_hash.pp oph - Tezos_crypto.Protocol_hash.pp + Protocol_hash.pp oph') (obj2 - (req "expected" Tezos_crypto.Protocol_hash.encoding) - (req "got" Tezos_crypto.Protocol_hash.encoding)) + (req "expected" Protocol_hash.encoding) + (req "got" Protocol_hash.encoding)) (function | Inconsistent_protocol_hash {expected; got} -> Some (expected, got) | _ -> None) @@ -207,13 +206,13 @@ let () = Format.fprintf ppf "Inconsistent context_hash. Expected: %a, got %a." - Tezos_crypto.Context_hash.pp + Context_hash.pp oph - Tezos_crypto.Context_hash.pp + Context_hash.pp oph') (obj2 - (req "expected" Tezos_crypto.Context_hash.encoding) - (req "got" Tezos_crypto.Context_hash.encoding)) + (req "expected" Context_hash.encoding) + (req "got" Context_hash.encoding)) (function | Inconsistent_context_hash {expected; got} -> Some (expected, got) | _ -> None) @@ -227,9 +226,9 @@ let () = Format.fprintf ppf "Failed to checkout context %a after restoring it." - Tezos_crypto.Context_hash.pp + Context_hash.pp h) - (obj1 (req "context_hash" Tezos_crypto.Context_hash.encoding)) + (obj1 (req "context_hash" Context_hash.encoding)) (function Inconsistent_context h -> Some h | _ -> None) (fun h -> Inconsistent_context h) ; register_error_kind @@ -241,9 +240,9 @@ let () = Format.fprintf ppf "Cannot decode the protocol in file: %a" - Tezos_crypto.Protocol_hash.pp + Protocol_hash.pp hash) - (obj1 (req "filename" Tezos_crypto.Protocol_hash.encoding)) + (obj1 (req "filename" Protocol_hash.encoding)) (function Cannot_decode_protocol hash -> Some hash | _ -> None) (fun hash -> Cannot_decode_protocol hash) ; register_error_kind @@ -408,13 +407,8 @@ let () = ~title:"target block validation failed" ~description:"Failed to validate the target block." ~pp:(fun ppf (h, errs) -> - Format.fprintf - ppf - "Failed to validate block %a: %s" - Tezos_crypto.Block_hash.pp - h - errs) - (obj2 (req "block" Tezos_crypto.Block_hash.encoding) (req "errors" string)) + Format.fprintf ppf "Failed to validate block %a: %s" Block_hash.pp h errs) + (obj2 (req "block" Block_hash.encoding) (req "errors" string)) (function | Target_block_validation_failed (h, errs) -> Some (h, errs) | _ -> None) (fun (h, errs) -> Target_block_validation_failed (h, errs)) ; @@ -538,13 +532,13 @@ let () = Format.fprintf ppf "The block contained in the file is %a instead of %a." - Tezos_crypto.Block_hash.pp + Block_hash.pp got - Tezos_crypto.Block_hash.pp + Block_hash.pp exp) (obj2 - (req "block_hash" Tezos_crypto.Block_hash.encoding) - (req "block_hash_expected" Tezos_crypto.Block_hash.encoding)) + (req "block_hash" Block_hash.encoding) + (req "block_hash_expected" Block_hash.encoding)) (function | Inconsistent_imported_block (got, exp) -> Some (got, exp) | _ -> None) (fun (got, exp) -> Inconsistent_imported_block (got, exp)) ; @@ -572,13 +566,11 @@ let () = ppf "Failed to export snapshot. Cannot find chain %a from store located at \ directory %s." - Tezos_crypto.Chain_id.pp_short + Chain_id.pp_short chain_id store_dir) Data_encoding.( - obj2 - (req "chain_id" Tezos_crypto.Chain_id.encoding) - (req "store_dir" string)) + obj2 (req "chain_id" Chain_id.encoding) (req "store_dir" string)) (function | Invalid_chain_store_export (chain_id, store_dir) -> Some (chain_id, store_dir) @@ -634,7 +626,7 @@ let default_index_log_size = 30_000_000 type metadata = { chain_name : Distributed_db_version.Name.t; history_mode : History_mode.t; - block_hash : Tezos_crypto.Block_hash.t; + block_hash : Block_hash.t; level : Int32.t; timestamp : Time.Protocol.t; context_elements : int; @@ -662,7 +654,7 @@ let metadata_encoding = (obj6 (req "chain_name" Distributed_db_version.Name.encoding) (req "mode" History_mode.encoding) - (req "block_hash" Tezos_crypto.Block_hash.encoding) + (req "block_hash" Block_hash.encoding) (req "level" int32) (req "timestamp" Time.Protocol.encoding) (req "context_elements" int31)) @@ -685,7 +677,7 @@ let pp_snapshot_header ppf version %d)" Distributed_db_version.Name.pp chain_name - Tezos_crypto.Block_hash.pp + Block_hash.pp block_hash level Time.Protocol.pp_hum @@ -713,9 +705,8 @@ type block_data = { block_header : Block_header.t; operations : Operation.t list list; predecessor_header : Block_header.t; - predecessor_block_metadata_hash : Tezos_crypto.Block_metadata_hash.t option; - predecessor_ops_metadata_hash : - Tezos_crypto.Operation_metadata_list_list_hash.t option; + predecessor_block_metadata_hash : Block_metadata_hash.t option; + predecessor_ops_metadata_hash : Operation_metadata_list_list_hash.t option; } let block_data_encoding = @@ -749,12 +740,10 @@ let block_data_encoding = (req "operations" (list (list (dynamic_size Operation.encoding)))) (req "block_header" (dynamic_size Block_header.encoding)) (req "predecessor_header" (dynamic_size Block_header.encoding)) - (opt - "predecessor_block_metadata_hash" - Tezos_crypto.Block_metadata_hash.encoding) + (opt "predecessor_block_metadata_hash" Block_metadata_hash.encoding) (opt "predecessor_ops_metadata_hash" - Tezos_crypto.Operation_metadata_list_list_hash.encoding)) + Operation_metadata_list_list_hash.encoding)) let default_snapshot_filename metadata = let {chain_name; block_hash; level; history_mode; _} = metadata in @@ -765,7 +754,7 @@ let default_snapshot_filename metadata = "%a-%a-%ld.%a" Distributed_db_version.Name.pp chain_name - Tezos_crypto.Block_hash.pp + Block_hash.pp block_hash level History_mode.pp_short @@ -1330,16 +1319,15 @@ module type EXPORTER = sig val write_block_data : t -> predecessor_header:Block_header.t -> - predecessor_block_metadata_hash:Tezos_crypto.Block_metadata_hash.t option -> - predecessor_ops_metadata_hash: - Tezos_crypto.Operation_metadata_list_list_hash.t option -> + predecessor_block_metadata_hash:Block_metadata_hash.t option -> + predecessor_ops_metadata_hash:Operation_metadata_list_list_hash.t option -> export_block:Store.Block.t -> unit Lwt.t val dump_context : t -> Context.index -> - Tezos_crypto.Context_hash.t -> + Context_hash.t -> on_disk:bool -> Animation.progress_display_mode -> int tzresult Lwt.t @@ -1362,8 +1350,7 @@ module type EXPORTER = sig val write_protocols_table : t -> f:(Lwt_unix.file_descr -> 'a Lwt.t) -> 'a Lwt.t - val copy_protocol : - t -> src:string -> dst_ph:Tezos_crypto.Protocol_hash.t -> unit Lwt.t + val copy_protocol : t -> src:string -> dst_ph:Protocol_hash.t -> unit Lwt.t val write_metadata : t -> metadata -> unit tzresult Lwt.t @@ -1803,7 +1790,7 @@ module Tar_exporter : EXPORTER = struct Filename.( concat (Naming.dir_path t.snapshot_protocol_dir) - (Tezos_crypto.Protocol_hash.to_b58check dst_ph)) + (Protocol_hash.to_b58check dst_ph)) in Onthefly.add_file_and_finalize t.tar ~file:src ~filename:dst @@ -1977,8 +1964,7 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct (* FIXME: we also write potential branches, it will eventually be GCed *) if Compare.Int32.(Block_repr.level block >= limit_level) then - if Tezos_crypto.Block_hash.equal limit_hash (Block_repr.hash block) - then raise Done + if Block_hash.equal limit_hash (Block_repr.hash block) then raise Done else return_unit else let block = (* Prune everything *) {block with metadata = None} in @@ -2049,7 +2035,7 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct || filename = Filename.parent_dir_name -> copy_protocols () | filename -> ( - match Tezos_crypto.Protocol_hash.of_b58check_opt filename with + match Protocol_hash.of_b58check_opt filename with | None -> return_ok_unit | Some ph -> let src_protocol_file = @@ -2057,10 +2043,7 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct in let* () = if - List.mem - ~equal:Tezos_crypto.Protocol_hash.equal - ph - proto_to_export + List.mem ~equal:Protocol_hash.equal ph proto_to_export then let* () = Exporter.copy_protocol @@ -2514,7 +2497,7 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct let export ?snapshot_path ?(rolling = false) ~block ~store_dir ~context_dir ~chain_name ~on_disk ~progress_display_mode genesis = let open Lwt_result_syntax in - let chain_id = Tezos_crypto.Chain_id.of_block_hash genesis.Genesis.block in + let chain_id = Chain_id.of_block_hash genesis.Genesis.block in let* () = ensure_valid_export_chain_dir store_dir chain_id in let* snapshot_exporter = init snapshot_path in (* Register a clean up callback to prevent export cancellation not @@ -2757,7 +2740,7 @@ module type IMPORTER = sig val init : snapshot_path:string -> dst_store_dir:[`Store_dir] Naming.directory -> - Tezos_crypto.Chain_id.t -> + Chain_id.t -> t Lwt.t val load_snapshot_header : t -> snapshot_header tzresult Lwt.t @@ -2767,7 +2750,7 @@ module type IMPORTER = sig val restore_context : t -> Context.index -> - expected_context_hash:Tezos_crypto.Context_hash.t -> + expected_context_hash:Context_hash.t -> nb_context_elements:int -> legacy:bool -> in_memory:bool -> @@ -2778,12 +2761,10 @@ module type IMPORTER = sig t -> Protocol_levels.activation_block Protocol_levels.t tzresult Lwt.t val load_and_validate_protocol_filenames : - t -> Tezos_crypto.Protocol_hash.t list tzresult Lwt.t + t -> Protocol_hash.t list tzresult Lwt.t val copy_and_validate_protocol : - t -> - protocol_hash:Tezos_crypto.Protocol_hash.t -> - (unit, error trace) result Lwt.t + t -> protocol_hash:Protocol_hash.t -> (unit, error trace) result Lwt.t val restore_cemented_indexes : t -> unit Lwt.t @@ -2793,7 +2774,7 @@ module type IMPORTER = sig val restore_floating_blocks : t -> - Tezos_crypto.Block_hash.t -> + Block_hash.t -> (unit tzresult Lwt.t * Block_repr.block Lwt_stream.t) tzresult Lwt.t val close : t -> unit Lwt.t @@ -2925,7 +2906,7 @@ module Raw_importer : IMPORTER = struct in List.map_es (fun file -> - match Tezos_crypto.Protocol_hash.of_b58check_opt file with + match Protocol_hash.of_b58check_opt file with | Some ph -> return ph | None -> tzfail (Invalid_protocol_file file)) protocol_files @@ -2935,12 +2916,12 @@ module Raw_importer : IMPORTER = struct let src = Filename.concat (Naming.dir_path t.snapshot_protocol_dir) - (Tezos_crypto.Protocol_hash.to_b58check protocol_hash) + (Protocol_hash.to_b58check protocol_hash) in let dst = Filename.concat (Naming.dir_path t.dst_protocol_dir) - (Tezos_crypto.Protocol_hash.to_b58check protocol_hash) + (Protocol_hash.to_b58check protocol_hash) in let*! () = Lwt_utils_unix.copy_file ~src ~dst in let*! protocol_sources = Lwt_utils_unix.read_file dst in @@ -2949,7 +2930,7 @@ module Raw_importer : IMPORTER = struct | Some p -> let hash = Protocol.hash p in fail_unless - (Tezos_crypto.Protocol_hash.equal protocol_hash hash) + (Protocol_hash.equal protocol_hash hash) (Inconsistent_protocol_hash {expected = protocol_hash; got = hash}) let restore_cemented_indexes t = @@ -3185,7 +3166,7 @@ module Tar_importer : IMPORTER = struct in List.map_es (fun file -> - match Tezos_crypto.Protocol_hash.of_b58check_opt file with + match Protocol_hash.of_b58check_opt file with | Some ph -> return ph | None -> tzfail (Invalid_protocol_file file)) protocol_files @@ -3196,7 +3177,7 @@ module Tar_importer : IMPORTER = struct Filename.( concat Naming.(protocol_store_dir t.snapshot_tar |> dir_path) - (Tezos_crypto.Protocol_hash.to_b58check protocol_hash)) + (Protocol_hash.to_b58check protocol_hash)) in let* file = let*! o = Onthefly.get_file t.tar ~filename:src in @@ -3208,7 +3189,7 @@ module Tar_importer : IMPORTER = struct Filename.( concat (Naming.dir_path t.dst_protocol_dir) - (Tezos_crypto.Protocol_hash.to_b58check protocol_hash)) + (Protocol_hash.to_b58check protocol_hash)) in let*! () = Onthefly.copy_to_file t.tar file ~dst in let*! protocol_sources = Lwt_utils_unix.read_file dst in @@ -3217,7 +3198,7 @@ module Tar_importer : IMPORTER = struct | Some p -> let hash = Protocol.hash p in fail_unless - (Tezos_crypto.Protocol_hash.equal protocol_hash hash) + (Protocol_hash.equal protocol_hash hash) (Inconsistent_protocol_hash {expected = protocol_hash; got = hash}) let restore_cemented_indexes t = @@ -3356,7 +3337,7 @@ module type Snapshot_importer = sig ?patch_context: (Tezos_protocol_environment.Context.t -> Tezos_protocol_environment.Context.t tzresult Lwt.t) -> - ?block:Tezos_crypto.Block_hash.t -> + ?block:Block_hash.t -> ?check_consistency:bool -> dst_store_dir:[`Store_dir] Naming.directory -> dst_context_dir:string -> @@ -3504,7 +3485,7 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct let check_context_hash_consistency validation_store block_header = fail_unless - (Tezos_crypto.Context_hash.equal + (Context_hash.equal validation_store.Tezos_validation.Block_validation.context_hash block_header.Block_header.shell.context) (Inconsistent_context_hash @@ -3542,7 +3523,7 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct match user_expected_block with | Some bh -> fail_unless - (Tezos_crypto.Block_hash.equal bh block_header_hash) + (Block_hash.equal bh block_header_hash) (Inconsistent_imported_block (block_header_hash, bh)) | None -> return_unit in @@ -3550,9 +3531,7 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct expected one *) let* () = fail_unless - (Tezos_crypto.Block_hash.equal - snapshot_metadata.block_hash - block_header_hash) + (Block_hash.equal snapshot_metadata.block_hash block_header_hash) (Inconsistent_imported_block (block_header_hash, snapshot_metadata.block_hash)) in @@ -3626,7 +3605,7 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct ~user_activated_protocol_overrides ~operation_metadata_size_limit ~in_memory ~progress_display_mode (genesis : Genesis.t) = let open Lwt_result_syntax in - let chain_id = Tezos_crypto.Chain_id.of_block_hash genesis.Genesis.block in + let chain_id = Chain_id.of_block_hash genesis.Genesis.block in let*! snapshot_importer = init ~snapshot_path ~dst_store_dir chain_id in let dst_store_dir = Naming.dir_path dst_store_dir in let* () = @@ -3636,7 +3615,7 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct in let dst_store_dir = Naming.store_dir ~dir_path:dst_store_dir in let dst_protocol_dir = Naming.protocol_store_dir dst_store_dir in - let chain_id = Tezos_crypto.Chain_id.of_block_hash genesis.block in + let chain_id = Chain_id.of_block_hash genesis.block in let dst_chain_dir = Naming.chain_dir dst_store_dir chain_id in let dst_cemented_dir = Naming.cemented_blocks_dir dst_chain_dir in (* Create directories *) diff --git a/src/lib_store/unix/snapshots.mli b/src/lib_store/unix/snapshots.mli index 61c6e46c37b5..aa5a9c403715 100644 --- a/src/lib_store/unix/snapshots.mli +++ b/src/lib_store/unix/snapshots.mli @@ -86,7 +86,7 @@ type error += stored : History_mode.t; } | Invalid_export_block of { - block : Tezos_crypto.Block_hash.t option; + block : Block_hash.t option; reason : [ `Pruned | `Pruned_pred @@ -99,15 +99,15 @@ type error += | Invalid_export_path of string | Snapshot_file_not_found of string | Inconsistent_protocol_hash of { - expected : Tezos_crypto.Protocol_hash.t; - got : Tezos_crypto.Protocol_hash.t; + expected : Protocol_hash.t; + got : Protocol_hash.t; } | Inconsistent_context_hash of { - expected : Tezos_crypto.Context_hash.t; - got : Tezos_crypto.Context_hash.t; + expected : Context_hash.t; + got : Context_hash.t; } - | Inconsistent_context of Tezos_crypto.Context_hash.t - | Cannot_decode_protocol of Tezos_crypto.Protocol_hash.t + | Inconsistent_context of Context_hash.t + | Cannot_decode_protocol of Protocol_hash.t | Cannot_write_metadata of string | Cannot_read of { kind : @@ -128,7 +128,7 @@ type error += | Missing_cemented_file of string | Corrupted_floating_store | Invalid_protocol_file of string - | Target_block_validation_failed of Tezos_crypto.Block_hash.t * string + | Target_block_validation_failed of Block_hash.t * string | Directory_already_exists of string | Empty_floating_store | Cannot_remove_tmp_export_directory of string @@ -136,8 +136,7 @@ type error += expected : Distributed_db_version.Name.t; got : Distributed_db_version.Name.t; } - | Inconsistent_imported_block of - Tezos_crypto.Block_hash.t * Tezos_crypto.Block_hash.t + | Inconsistent_imported_block of Block_hash.t * Block_hash.t | Wrong_snapshot_file of {filename : string} (** Current version of snapshots *) @@ -206,7 +205,7 @@ val import : ?patch_context: (Tezos_protocol_environment.Context.t -> Tezos_protocol_environment.Context.t tzresult Lwt.t) -> - ?block:Tezos_crypto.Block_hash.t -> + ?block:Block_hash.t -> ?check_consistency:bool -> dst_store_dir:string -> dst_context_dir:string -> diff --git a/src/lib_store/unix/snapshots_events.ml b/src/lib_store/unix/snapshots_events.ml index 69f21844964a..994ebc65c574 100644 --- a/src/lib_store/unix/snapshots_events.ml +++ b/src/lib_store/unix/snapshots_events.ml @@ -138,6 +138,6 @@ module Event = struct ~level:Notice ~name:"validate_protocol_sources" ~msg:"validating protocol {hash} against sources" - ~pp1:Tezos_crypto.Protocol_hash.pp - ("hash", Tezos_crypto.Protocol_hash.encoding) + ~pp1:Protocol_hash.pp + ("hash", Protocol_hash.encoding) end diff --git a/src/lib_store/unix/store.ml b/src/lib_store/unix/store.ml index cb66ca387479..3ea97afccde7 100644 --- a/src/lib_store/unix/store.ml +++ b/src/lib_store/unix/store.ml @@ -63,13 +63,13 @@ type store = { context_index : Context_ops.index; protocol_store : Protocol_store.t; allow_testchains : bool; - protocol_watcher : Tezos_crypto.Protocol_hash.t Lwt_watcher.input; + protocol_watcher : Protocol_hash.t Lwt_watcher.input; global_block_watcher : (chain_store * block) Lwt_watcher.input; } and chain_store = { global_store : store; - chain_id : Tezos_crypto.Chain_id.t; + chain_id : Chain_id.t; chain_dir : [`Chain_dir] Naming.directory; chain_config : chain_config; block_store : Block_store.t; @@ -78,8 +78,8 @@ and chain_store = { genesis_block_data : block Stored_data.t; block_watcher : block Lwt_watcher.input; block_rpc_directories : - (chain_store * block) Tezos_rpc.Directory.t Tezos_crypto.Protocol_hash.Map.t - Tezos_crypto.Protocol_hash.Table.t; + (chain_store * block) Tezos_rpc.Directory.t Protocol_hash.Map.t + Protocol_hash.Table.t; lockfile : Lwt_unix.file_descr; } @@ -94,26 +94,20 @@ and chain_state = { (* Following fields are safe to update directly *) protocol_levels_data : Protocol_levels.activation_block Protocol_levels.t Stored_data.t; - invalid_blocks_data : - invalid_block Tezos_crypto.Block_hash.Map.t Stored_data.t; - forked_chains_data : - Tezos_crypto.Block_hash.t Tezos_crypto.Chain_id.Map.t Stored_data.t; + invalid_blocks_data : invalid_block Block_hash.Map.t Stored_data.t; + forked_chains_data : Block_hash.t Chain_id.Map.t Stored_data.t; (* In memory-only: *) current_head : Block_repr.t; active_testchain : testchain option; mempool : Mempool.t; - live_blocks : Tezos_crypto.Block_hash.Set.t; - live_operations : Tezos_crypto.Operation_hash.Set.t; + live_blocks : Block_hash.Set.t; + live_operations : Operation_hash.Set.t; mutable live_data_cache : - (Tezos_crypto.Block_hash.t * Tezos_crypto.Operation_hash.Set.t) Ringo.Ring.t - option; + (Block_hash.t * Operation_hash.Set.t) Ringo.Ring.t option; prechecked_blocks : Block_repr.t Block_lru_cache.t; } -and testchain = { - forked_block : Tezos_crypto.Block_hash.t; - testchain_store : chain_store; -} +and testchain = {forked_block : Block_hash.t; testchain_store : chain_store} and block = Block_repr.t @@ -165,7 +159,7 @@ let locked_is_acceptable_block chain_state (hash, level) = | None -> Lwt.return_true | Some (target_hash, target_level) -> if Compare.Int32.(level = target_level) then - Lwt.return @@ Tezos_crypto.Block_hash.equal hash target_hash + Lwt.return @@ Block_hash.equal hash target_hash else Lwt.return_true let create_lockfile chain_dir = @@ -208,8 +202,7 @@ module Block = struct operations_metadata : Block_validation.operation_metadata list list; } - let equal b b' = - Tezos_crypto.Block_hash.equal (Block_repr.hash b) (Block_repr.hash b') + let equal b b' = Block_hash.equal (Block_repr.hash b) (Block_repr.hash b') let descriptor blk = Block_repr.descriptor blk @@ -227,7 +220,7 @@ module Block = struct let locked_is_known_invalid chain_state hash = let open Lwt_syntax in let* invalid_blocks = Stored_data.get chain_state.invalid_blocks_data in - Lwt.return (Tezos_crypto.Block_hash.Map.mem hash invalid_blocks) + Lwt.return (Block_hash.Map.mem hash invalid_blocks) let is_known_invalid {chain_state; _} hash = Shared.use chain_state (fun chain_state -> @@ -258,7 +251,7 @@ module Block = struct let is_genesis chain_store hash = let genesis = genesis chain_store in - Tezos_crypto.Block_hash.equal hash genesis.Genesis.block + Block_hash.equal hash genesis.Genesis.block let read_block {block_store; _} ?(distance = 0) hash = let open Lwt_result_syntax in @@ -456,7 +449,7 @@ module Block = struct in let*! genesis_block = Stored_data.get chain_store.genesis_block_data in let is_main_chain = - Tezos_crypto.Chain_id.equal + Chain_id.equal chain_store.chain_id (WithExceptions.Option.get ~loc:__LOC__ @@ -509,9 +502,7 @@ module Block = struct in let* () = fail_unless - (Tezos_crypto.Context_hash.equal - block_header.shell.context - context_hash) + (Context_hash.equal block_header.shell.context context_hash) (Validation_errors.Inconsistent_hash (context_hash, block_header.shell.context)) in @@ -634,7 +625,7 @@ module Block = struct in let testchain_id = Context.compute_testchain_chain_id genesis in let forked_hash_opt = - Tezos_crypto.Chain_id.Map.find testchain_id forked_chains + Chain_id.Map.find testchain_id forked_chains in return (status, forked_hash_opt)) | Forking _ -> return (status, Some (Block_repr.hash block)) @@ -663,7 +654,7 @@ module Block = struct let open Lwt_syntax in Shared.use chain_state (fun chain_state -> let* invalid_blocks = Stored_data.get chain_state.invalid_blocks_data in - Lwt.return (Tezos_crypto.Block_hash.Map.find hash invalid_blocks)) + Lwt.return (Block_hash.Map.find hash invalid_blocks)) let read_invalid_blocks {chain_state; _} = Shared.use chain_state (fun chain_state -> @@ -680,10 +671,7 @@ module Block = struct (fun invalid_blocks -> Prometheus.Gauge.inc_one Store_metrics.metrics.invalid_blocks ; Lwt.return - (Tezos_crypto.Block_hash.Map.add - hash - {level; errors} - invalid_blocks))) + (Block_hash.Map.add hash {level; errors} invalid_blocks))) in return_unit @@ -693,7 +681,7 @@ module Block = struct chain_state.invalid_blocks_data (fun invalid_blocks -> Prometheus.Gauge.dec_one Store_metrics.metrics.invalid_blocks ; - Lwt.return (Tezos_crypto.Block_hash.Map.remove hash invalid_blocks))) + Lwt.return (Block_hash.Map.remove hash invalid_blocks))) (** Accessors *) @@ -740,8 +728,8 @@ module Block = struct else Option.map (fun ll -> - Tezos_crypto.Operation_metadata_list_list_hash.compute - (List.map Tezos_crypto.Operation_metadata_list_hash.compute ll)) + Operation_metadata_list_list_hash.compute + (List.map Operation_metadata_list_hash.compute ll)) (Block_repr.operations_metadata_hashes blk) (** Metadata accessors *) @@ -758,10 +746,8 @@ module Block = struct let operations_metadata metadata = Block_repr.operations_metadata metadata let compute_operation_path hashes = - let list_hashes = - List.map Tezos_crypto.Operation_list_hash.compute hashes - in - Tezos_crypto.Operation_list_list_hash.compute_path list_hashes + let list_hashes = List.map Operation_list_hash.compute hashes in + Operation_list_list_hash.compute_path list_hashes let operations_path block i = if i < 0 || validation_passes block <= i then invalid_arg "operations_path" ; @@ -837,27 +823,22 @@ module Chain_traversal = struct let live_blocks chain_store block n = let fold (bacc, oacc) (head_hash, op_hashes) = - let bacc = Tezos_crypto.Block_hash.Set.add head_hash bacc in + let bacc = Block_hash.Set.add head_hash bacc in let oacc = List.fold_left - (List.fold_left (fun oacc op -> - Tezos_crypto.Operation_hash.Set.add op oacc)) + (List.fold_left (fun oacc op -> Operation_hash.Set.add op oacc)) oacc op_hashes in (bacc, oacc) in - let init = - (Tezos_crypto.Block_hash.Set.empty, Tezos_crypto.Operation_hash.Set.empty) - in + let init = (Block_hash.Set.empty, Operation_hash.Set.empty) in folder chain_store block n fold init let live_blocks_with_ring chain_store block n ring = let open Lwt_syntax in let fold acc (head_hash, op_hashes) = - let op_hash_set = - Tezos_crypto.Operation_hash.Set.(of_list (List.flatten op_hashes)) - in + let op_hash_set = Operation_hash.Set.(of_list (List.flatten op_hashes)) in (head_hash, op_hash_set) :: acc in let* l = folder chain_store block n fold [] in @@ -968,7 +949,7 @@ module Chain = struct let*! current_head_descr = Stored_data.get chain_state.current_head_data in - if Tezos_crypto.Block_hash.equal head (fst current_head_descr) then + if Block_hash.equal head (fst current_head_descr) then return (Some {chain_state with mempool}, ()) else return (None, ())) @@ -990,22 +971,20 @@ module Chain = struct match live_data_cache with | Some live_data_cache when update_cache - && Tezos_crypto.Block_hash.equal + && Block_hash.equal (Block.predecessor block) (Block.hash current_head) && Ringo.Ring.capacity live_data_cache = expected_capacity -> ( let most_recent_block = Block.hash block in let most_recent_ops = Block.all_operation_hashes block - |> List.flatten |> Tezos_crypto.Operation_hash.Set.of_list + |> List.flatten |> Operation_hash.Set.of_list in let new_live_blocks = - Tezos_crypto.Block_hash.Set.add most_recent_block live_blocks + Block_hash.Set.add most_recent_block live_blocks in let new_live_operations = - Tezos_crypto.Operation_hash.Set.union - most_recent_ops - live_operations + Operation_hash.Set.union most_recent_ops live_operations in match Ringo.Ring.add_and_return_erased @@ -1015,12 +994,10 @@ module Chain = struct | None -> Lwt.return (new_live_blocks, new_live_operations) | Some (last_block, last_ops) -> let diffed_new_live_blocks = - Tezos_crypto.Block_hash.Set.remove last_block new_live_blocks + Block_hash.Set.remove last_block new_live_blocks in let diffed_new_live_operations = - Tezos_crypto.Operation_hash.Set.diff - new_live_operations - last_ops + Operation_hash.Set.diff new_live_operations last_ops in Lwt.return (diffed_new_live_blocks, diffed_new_live_operations)) | _ when update_cache -> @@ -1036,12 +1013,9 @@ module Chain = struct let live_blocks, live_ops = Ringo.Ring.fold new_cache - ~init: - ( Tezos_crypto.Block_hash.Set.empty, - Tezos_crypto.Operation_hash.Set.empty ) + ~init:(Block_hash.Set.empty, Operation_hash.Set.empty) ~f:(fun (bhs, opss) (bh, ops) -> - ( Tezos_crypto.Block_hash.Set.add bh bhs, - Tezos_crypto.Operation_hash.Set.union ops opss )) + (Block_hash.Set.add bh bhs, Operation_hash.Set.union ops opss)) in Lwt.return (live_blocks, live_ops) | _ -> Chain_traversal.live_blocks chain_store block expected_capacity @@ -1064,7 +1038,7 @@ module Chain = struct let open Lwt_syntax in if Compare.Int32.(lvl' > lvl) then Lwt.return_false else if Compare.Int32.(lvl = lvl') then - Lwt.return (Tezos_crypto.Block_hash.equal hash hash') + Lwt.return (Block_hash.equal hash hash') else let* o = Block.read_ancestor_hash_opt @@ -1074,8 +1048,7 @@ module Chain = struct in match o with | None -> Lwt.return_false - | Some hash_found -> - Lwt.return (Tezos_crypto.Block_hash.equal hash' hash_found) + | Some hash_found -> Lwt.return (Block_hash.equal hash' hash_found) let is_in_chain chain_store (hash, level) = let open Lwt_syntax in @@ -1559,7 +1532,7 @@ module Chain = struct chain_state.invalid_blocks_data (fun invalid_blocks -> Lwt.return - (Tezos_crypto.Block_hash.Map.filter + (Block_hash.Map.filter (fun _k {level; _} -> if level > snd new_checkpoint then ( Prometheus.Gauge.inc_one @@ -1647,8 +1620,7 @@ module Chain = struct in match o with | None -> tzfail Missing_last_allowed_fork_level_block - | Some lafl_hash -> - return (Tezos_crypto.Block_hash.equal lafl_hash ancestor))) + | Some lafl_hash -> return (Block_hash.equal lafl_hash ancestor))) let is_valid_for_checkpoint chain_store given_checkpoint = let open Lwt_syntax in @@ -1784,10 +1756,7 @@ module Chain = struct ( best_head, List.filter (fun (hash, _) -> - not - (Tezos_crypto.Block_hash.equal - (Block.hash best_head) - hash)) + not (Block_hash.equal (Block.hash best_head) hash)) all_heads ) in (* Case 1 *) @@ -1914,20 +1883,18 @@ module Chain = struct let* invalid_blocks_data = Stored_data.init (Naming.invalid_blocks_file chain_dir) - ~initial_data:Tezos_crypto.Block_hash.Map.empty + ~initial_data:Block_hash.Map.empty in let* forked_chains_data = Stored_data.init (Naming.forked_chains_file chain_dir) - ~initial_data:Tezos_crypto.Chain_id.Map.empty + ~initial_data:Chain_id.Map.empty in let current_head = genesis_block in let active_testchain = None in let mempool = Mempool.empty in - let live_blocks = - Tezos_crypto.Block_hash.Set.singleton genesis_block.hash - in - let live_operations = Tezos_crypto.Operation_hash.Set.empty in + let live_blocks = Block_hash.Set.singleton genesis_block.hash in + let live_operations = Operation_hash.Set.empty in let live_data_cache = None in let prechecked_blocks = Block_lru_cache.create 10 in return @@ -2026,8 +1993,8 @@ module Chain = struct | Some current_head -> let active_testchain = None in let mempool = Mempool.empty in - let live_blocks = Tezos_crypto.Block_hash.Set.empty in - let live_operations = Tezos_crypto.Operation_hash.Set.empty in + let live_blocks = Block_hash.Set.empty in + let live_operations = Operation_hash.Set.empty in let live_data_cache = None in let prechecked_blocks = Block_lru_cache.create 10 in return @@ -2105,7 +2072,7 @@ module Chain = struct in let chain_state = Shared.create chain_state in let block_watcher = Lwt_watcher.create_input () in - let block_rpc_directories = Tezos_crypto.Protocol_hash.Table.create 7 in + let block_rpc_directories = Protocol_hash.Table.create 7 in let* lockfile = create_lockfile chain_dir in let chain_store : chain_store = { @@ -2140,7 +2107,7 @@ module Chain = struct let* chain_state = load_chain_state chain_dir block_store in let chain_state = Shared.create chain_state in let block_watcher = Lwt_watcher.create_input () in - let block_rpc_directories = Tezos_crypto.Protocol_hash.Table.create 7 in + let block_rpc_directories = Protocol_hash.Table.create 7 in let* lockfile = create_lockfile chain_dir in let chain_store = { @@ -2212,16 +2179,14 @@ module Chain = struct let {forked_chains_data; active_testchain; _} = chain_state in match active_testchain with | Some testchain - when Tezos_crypto.Chain_id.equal - chain_id - testchain.testchain_store.chain_id -> + when Chain_id.equal chain_id testchain.testchain_store.chain_id -> return_some testchain | _ -> ( let chain_dir = chain_store.chain_dir in let testchains_dir = Naming.testchains_dir chain_dir in let testchain_dir = Naming.chain_dir testchains_dir chain_id in let*! forked_chains = Stored_data.get forked_chains_data in - match Tezos_crypto.Chain_id.Map.find chain_id forked_chains with + match Chain_id.Map.find chain_id forked_chains with | None -> return_none | Some forked_block -> let* testchain_store = @@ -2239,7 +2204,7 @@ module Chain = struct let open Lwt_result_syntax in let forked_block_hash = Block.hash forked_block in let genesis_hash' = Context.compute_testchain_genesis forked_block_hash in - assert (Tezos_crypto.Block_hash.equal genesis_hash genesis_hash') ; + assert (Block_hash.equal genesis_hash genesis_hash') ; let* () = fail_unless chain_store.global_store.allow_testchains @@ -2251,10 +2216,8 @@ module Chain = struct match active_testchain with | Some ({testchain_store; forked_block} as testchain) -> (* Already forked and active *) - if Tezos_crypto.Chain_id.equal testchain_store.chain_id testchain_id - then ( - assert ( - Tezos_crypto.Block_hash.equal forked_block forked_block_hash) ; + if Chain_id.equal testchain_store.chain_id testchain_id then ( + assert (Block_hash.equal forked_block forked_block_hash) ; return (None, testchain)) else tzfail (Cannot_fork_testchain testchain_id) | None -> @@ -2308,7 +2271,7 @@ module Chain = struct chain_state.forked_chains_data (fun forked_chains -> Lwt.return - (Tezos_crypto.Chain_id.Map.add + (Chain_id.Map.add testchain_id forked_block_hash forked_chains)) @@ -2414,7 +2377,7 @@ module Chain = struct let*! o = find_activation_block chain_store ~protocol_level in match o with | Some {block = bh, _; _} -> - if Tezos_crypto.Block_hash.(bh <> Block.hash block) then + if Block_hash.(bh <> Block.hash block) then set_protocol_level chain_store ~protocol_level (block, protocol_hash) else return_unit | None -> @@ -2493,8 +2456,7 @@ module Chain = struct let* o = Block.read_predecessor_opt chain_store block in match o with | None -> Lwt.return_none (* genesis *) - | Some pred - when Tezos_crypto.Block_hash.equal (Block.hash pred) (Block.hash block) -> + | Some pred when Block_hash.equal (Block.hash pred) (Block.hash block) -> Lwt.return_none (* genesis *) | Some pred -> ( let* _, save_point_level = savepoint chain_store in @@ -2511,27 +2473,25 @@ module Chain = struct else Block.protocol_hash_exn chain_store pred in match - Tezos_crypto.Protocol_hash.Table.find - chain_store.block_rpc_directories - protocol + Protocol_hash.Table.find chain_store.block_rpc_directories protocol with | None -> Lwt.return_none | Some map -> let* next_protocol = Block.protocol_hash_exn chain_store block in - Lwt.return (Tezos_crypto.Protocol_hash.Map.find next_protocol map)) + Lwt.return (Protocol_hash.Map.find next_protocol map)) let set_rpc_directory chain_store ~protocol_hash ~next_protocol_hash dir = let map = Option.value - ~default:Tezos_crypto.Protocol_hash.Map.empty - (Tezos_crypto.Protocol_hash.Table.find + ~default:Protocol_hash.Map.empty + (Protocol_hash.Table.find chain_store.block_rpc_directories protocol_hash) in - Tezos_crypto.Protocol_hash.Table.replace + Protocol_hash.Table.replace chain_store.block_rpc_directories protocol_hash - (Tezos_crypto.Protocol_hash.Map.add next_protocol_hash dir map) ; + (Protocol_hash.Map.add next_protocol_hash dir map) ; Lwt.return_unit let register_gc_callback chain_store callback = @@ -2665,7 +2625,7 @@ let load_store ?history_mode ?block_cache_limit store_dir ~context_index let stored_genesis = Chain.genesis main_chain_store in let* () = fail_unless - (Tezos_crypto.Block_hash.equal genesis.Genesis.block stored_genesis.block) + (Block_hash.equal genesis.Genesis.block stored_genesis.block) (Inconsistent_genesis {expected = stored_genesis.block; got = genesis.block}) in @@ -2701,7 +2661,7 @@ let init ?patch_context ?commit_genesis ?history_mode ?(readonly = false) patch_context in let store_dir = Naming.store_dir ~dir_path:store_dir in - let chain_id = Tezos_crypto.Chain_id.of_block_hash genesis.Genesis.block in + let chain_id = Chain_id.of_block_hash genesis.Genesis.block in let*! context_index, commit_genesis = match commit_genesis with | Some commit_genesis -> @@ -2775,7 +2735,7 @@ let close_store global_store = let may_switch_history_mode ~store_dir ~context_dir genesis ~new_history_mode = let open Lwt_result_syntax in let store_dir = Naming.store_dir ~dir_path:store_dir in - let chain_id = Tezos_crypto.Chain_id.of_block_hash genesis.Genesis.block in + let chain_id = Chain_id.of_block_hash genesis.Genesis.block in let chain_dir = Naming.chain_dir store_dir chain_id in let chain_dir_path = Naming.dir_path chain_dir in if not (Sys.file_exists chain_dir_path && Sys.is_directory chain_dir_path) @@ -2846,7 +2806,7 @@ let get_chain_store store chain_id = let chain_store = main_chain_store store in let rec loop chain_store = let open Lwt_result_syntax in - if Tezos_crypto.Chain_id.equal (Chain.chain_id chain_store) chain_id then + if Chain_id.equal (Chain.chain_id chain_store) chain_id then return chain_store else Shared.use chain_store.chain_state (fun {active_testchain; _} -> @@ -3003,7 +2963,7 @@ let rec make_pp_chain_store (chain_store : chain_store) = proto_level pp_block_descriptor block - Tezos_crypto.Protocol_hash.pp + Protocol_hash.pp protocol (option_pp ~default:"n/a" (fun fmt _ -> Format.fprintf fmt "available")) commit_info @@ -3025,7 +2985,7 @@ let rec make_pp_chain_store (chain_store : chain_store) = %a@ target: %a@ @[protocol levels:@ %a@]@ @[invalid \ blocks:@ %a@]@ @[forked chains:@ %a@]@ @[active testchain: \ %a@]@]" - Tezos_crypto.Chain_id.pp + Chain_id.pp chain_id (Naming.dir_path chain_dir) Data_encoding.Json.pp @@ -3069,22 +3029,19 @@ let rec make_pp_chain_store (chain_store : chain_store) = target (Format.pp_print_list ~pp_sep:Format.pp_print_cut pp_protocol_level) (Protocol_levels.bindings protocol_levels_data) - (Format.pp_print_list - ~pp_sep:Format.pp_print_cut - Tezos_crypto.Block_hash.pp) - (Tezos_crypto.Block_hash.Map.bindings invalid_blocks_data - |> List.map fst) + (Format.pp_print_list ~pp_sep:Format.pp_print_cut Block_hash.pp) + (Block_hash.Map.bindings invalid_blocks_data |> List.map fst) (Format.pp_print_list ~pp_sep:Format.pp_print_cut (fun fmt (chain_id, block_hash) -> Format.fprintf fmt "testchain's chain id: %a, forked block: %a" - Tezos_crypto.Chain_id.pp + Chain_id.pp chain_id - Tezos_crypto.Block_hash.pp + Block_hash.pp block_hash)) - (Tezos_crypto.Chain_id.Map.bindings forked_chains_data) + (Chain_id.Map.bindings forked_chains_data) pp_testchain_opt ()) @@ -3150,7 +3107,7 @@ module Unsafe = struct ~(locked_f : chain_store -> 'a tzresult Lwt.t) = let open Lwt_result_syntax in let store_dir = Naming.store_dir ~dir_path:store_dir in - let chain_id = Tezos_crypto.Chain_id.of_block_hash genesis.Genesis.block in + let chain_id = Chain_id.of_block_hash genesis.Genesis.block in let chain_dir = Naming.chain_dir store_dir chain_id in let* lockfile = create_lockfile chain_dir in let*! () = lock_for_read lockfile in @@ -3179,7 +3136,7 @@ module Unsafe = struct ~genesis ~genesis_context_hash ~floating_blocks_stream ~new_head_with_metadata ~protocol_levels ~history_mode = let open Lwt_result_syntax in - let chain_id = Tezos_crypto.Chain_id.of_block_hash genesis.Genesis.block in + let chain_id = Chain_id.of_block_hash genesis.Genesis.block in let chain_dir = Naming.chain_dir store_dir chain_id in let genesis_block = Block_repr.create_genesis_block ~genesis genesis_context_hash @@ -3251,12 +3208,12 @@ module Unsafe = struct let* () = Stored_data.write_file (Naming.invalid_blocks_file chain_dir) - Tezos_crypto.Block_hash.Map.empty + Block_hash.Map.empty in let* () = Stored_data.write_file (Naming.forked_chains_file chain_dir) - Tezos_crypto.Chain_id.Map.empty + Chain_id.Map.empty in let* () = Stored_data.write_file (Naming.genesis_block_file chain_dir) genesis_block @@ -3299,7 +3256,7 @@ module Unsafe = struct return_unit | _ -> fail_unless - (Tezos_crypto.Block_hash.equal real_genesis_hash bh) + (Block_hash.equal real_genesis_hash bh) (Missing_activation_block (bh, protocol, history_mode))) | Some _block, None -> return_unit | Some block, Some commit_info -> diff --git a/src/lib_store/unix/store.mli b/src/lib_store/unix/store.mli index e05a3df52243..a0d780408158 100644 --- a/src/lib_store/unix/store.mli +++ b/src/lib_store/unix/store.mli @@ -184,7 +184,7 @@ type chain_store chains and instantiate testchain's sub chain stores, for all chains contained in the store. The chain store created is based on the [genesis] provided. Its chain identifier will be computed - using the {!Tezos_crypto.Chain_id.of_block_hash} function. + using the {!Chain_id.of_block_hash} function. @param patch_context the handle called when initializing the context. It usually is passed when creating a sandboxed chain. @@ -213,9 +213,7 @@ val init : ?patch_context: (Tezos_protocol_environment.Context.t -> Tezos_protocol_environment.Context.t tzresult Lwt.t) -> - ?commit_genesis: - (chain_id:Tezos_crypto.Chain_id.t -> - Tezos_crypto.Context_hash.t tzresult Lwt.t) -> + ?commit_genesis:(chain_id:Chain_id.t -> Context_hash.t tzresult Lwt.t) -> ?history_mode:History_mode.t -> ?readonly:bool -> ?block_cache_limit:int -> @@ -267,13 +265,11 @@ val all_chain_stores : store -> chain_store list Lwt.t (** [get_chain_store global_store chain_id] returns the initialized chain store in [global_store] associated to [chain_id]. *) -val get_chain_store : - store -> Tezos_crypto.Chain_id.t -> chain_store tzresult Lwt.t +val get_chain_store : store -> Chain_id.t -> chain_store tzresult Lwt.t (** [get_chain_store_opt global_store chain_id] optional version of [get_chain_store]. *) -val get_chain_store_opt : - store -> Tezos_crypto.Chain_id.t -> chain_store option Lwt.t +val get_chain_store_opt : store -> Chain_id.t -> chain_store option Lwt.t val make_pp_store : store -> (Format.formatter -> unit -> unit) Lwt.t @@ -307,49 +303,41 @@ module Block : sig (** [is_known_valid chain_store bh] tests that the block [bh] is known and valid in [chain_store] (i.e. the block is present in the block store). *) - val is_known_valid : chain_store -> Tezos_crypto.Block_hash.t -> bool Lwt.t + val is_known_valid : chain_store -> Block_hash.t -> bool Lwt.t (** [is_known_invalid chain_store bh] tests that the block [bh] is invalid in [chain_store] (i.e. the block is present in the invalid blocks file). *) - val is_known_invalid : chain_store -> Tezos_crypto.Block_hash.t -> bool Lwt.t + val is_known_invalid : chain_store -> Block_hash.t -> bool Lwt.t (** [is_known_prechecked chain_store bh] tests that the block [bh] is prechecked in [chain_store] (i.e. the block is present in the prechecked block cache). *) - val is_known_prechecked : - chain_store -> Tezos_crypto.Block_hash.t -> bool Lwt.t + val is_known_prechecked : chain_store -> Block_hash.t -> bool Lwt.t (** [is_known chain_store bh] tests that the block [bh] is either known valid or known invalid in [chain_store]. *) - val is_known : chain_store -> Tezos_crypto.Block_hash.t -> bool Lwt.t + val is_known : chain_store -> Block_hash.t -> bool Lwt.t (** [is_genesis chain_store bh] tests that the block [bh] is the genesis initialized in [chain_store]. *) - val is_genesis : chain_store -> Tezos_crypto.Block_hash.t -> bool + val is_genesis : chain_store -> Block_hash.t -> bool (** [validity chain_store bh] computes the {!Block_locator.validity} ([Unknown], [Known_valid] or [Known_invalid]) for the block [bh] in [chain_store]. *) - val validity : - chain_store -> Tezos_crypto.Block_hash.t -> Block_locator.validity Lwt.t + val validity : chain_store -> Block_hash.t -> Block_locator.validity Lwt.t (** [read_block chain_store ?distance bh] tries to read in the [chain_store] the block [bh] or the predecessing block at the offset [distance] of [bh]. By default, [distance] is 0.*) val read_block : - chain_store -> - ?distance:int -> - Tezos_crypto.Block_hash.t -> - block tzresult Lwt.t + chain_store -> ?distance:int -> Block_hash.t -> block tzresult Lwt.t (** [read_block_opt chain_store ?distance bh] optional version of [read_block]. *) val read_block_opt : - chain_store -> - ?distance:int -> - Tezos_crypto.Block_hash.t -> - block option Lwt.t + chain_store -> ?distance:int -> Block_hash.t -> block option Lwt.t (** [read_block_by_level chain_store level] reads in the [chain_store] the block at [level]. The block retrieved will be @@ -371,16 +359,13 @@ module Block : sig val read_block_metadata : ?distance:int -> chain_store -> - Tezos_crypto.Block_hash.t -> + Block_hash.t -> metadata option tzresult Lwt.t (** [read_block_metadata_opt ?distance chain_store bh] same as [read_block_metadata] but returns [None] in case of errors. *) val read_block_metadata_opt : - ?distance:int -> - chain_store -> - Tezos_crypto.Block_hash.t -> - metadata option Lwt.t + ?distance:int -> chain_store -> Block_hash.t -> metadata option Lwt.t (** [get_block_metadata chain_store block] reads in the [chain_store] the metadata associated to the [block]. Returns @@ -405,7 +390,7 @@ module Block : sig (** [read_predecessor_of_hash chain_store bh] reads in [chain_store] the predecessor's block of [bh]. *) val read_predecessor_of_hash : - chain_store -> Tezos_crypto.Block_hash.t -> block tzresult Lwt.t + chain_store -> Block_hash.t -> block tzresult Lwt.t (** [read_ancestor_hash chain_store ~distance bh] retrieves in the [chain_store] the hash of the ancestor of the block [bh] at @@ -413,31 +398,28 @@ module Block : sig val read_ancestor_hash : chain_store -> distance:int -> - Tezos_crypto.Block_hash.t -> - Tezos_crypto.Block_hash.t option tzresult Lwt.t + Block_hash.t -> + Block_hash.t option tzresult Lwt.t (** [read_ancestor_hash_opt chain_store ~distance bh] same as [read_ancestor_hash] but returns [None] on errors. *) val read_ancestor_hash_opt : - chain_store -> - distance:int -> - Tezos_crypto.Block_hash.t -> - Tezos_crypto.Block_hash.t option Lwt.t + chain_store -> distance:int -> Block_hash.t -> Block_hash.t option Lwt.t (** [read_ancestor_opt chain_store block] optional version of [read_ancestor]. *) val read_predecessor_of_hash_opt : - chain_store -> Tezos_crypto.Block_hash.t -> block option Lwt.t + chain_store -> Block_hash.t -> block option Lwt.t (** [read_prechecked_block chain_store bh] tries to read in the [chain_store]'s prechecked block cache the block [bh].*) val read_prechecked_block : - chain_store -> Tezos_crypto.Block_hash.t -> block tzresult Lwt.t + chain_store -> Block_hash.t -> block tzresult Lwt.t (** [read_prechecked_block_opt chain_store bh] optional version of [read_prechecked_block].*) val read_prechecked_block_opt : - chain_store -> Tezos_crypto.Block_hash.t -> block option Lwt.t + chain_store -> Block_hash.t -> block option Lwt.t (** [store_block chain_store ~block_header ~operations validation_result] stores in [chain_store] the block with its @@ -463,7 +445,7 @@ module Block : sig its [block_header] and [operations]. *) val store_prechecked_block : chain_store -> - hash:Tezos_crypto.Block_hash.t -> + hash:Block_hash.t -> block_header:Block_header.t -> operations:Operation.t trace trace -> unit tzresult Lwt.t @@ -493,50 +475,46 @@ module Block : sig val testchain_status : chain_store -> block -> - (Test_chain_status.t * Tezos_crypto.Block_hash.t option) tzresult Lwt.t + (Test_chain_status.t * Block_hash.t option) tzresult Lwt.t (** [protocol_hash_exn chain_store block] reads the protocol associated to [block] in its associated context. Fails when the context is unknown. *) - val protocol_hash_exn : - chain_store -> block -> Tezos_crypto.Protocol_hash.t Lwt.t + val protocol_hash_exn : chain_store -> block -> Protocol_hash.t Lwt.t (** [protocol_hash chain_store block] error monad version of [protocol_hash_exn]. *) - val protocol_hash : - chain_store -> block -> Tezos_crypto.Protocol_hash.t tzresult Lwt.t + val protocol_hash : chain_store -> block -> Protocol_hash.t tzresult Lwt.t (** [read_invalid_block_opt chain_store bh] reads in the [chain_store] the invalid block [bh] if it exists. *) val read_invalid_block_opt : - chain_store -> Tezos_crypto.Block_hash.t -> invalid_block option Lwt.t + chain_store -> Block_hash.t -> invalid_block option Lwt.t (** [read_invalid_blocks chain_store] returns the map of all invalid blocks of [chain_store]. *) - val read_invalid_blocks : - chain_store -> invalid_block Tezos_crypto.Block_hash.Map.t Lwt.t + val read_invalid_blocks : chain_store -> invalid_block Block_hash.Map.t Lwt.t (** [mark_invalid chain_store bh ~level errors] stores the block [bh] at [level] with the given [errors]. Fails when trying to mark the genesis block as invalid. *) val mark_invalid : chain_store -> - Tezos_crypto.Block_hash.t -> + Block_hash.t -> level:int32 -> error list -> unit tzresult Lwt.t (** [unmark_invalid chain_store bh] unmarks invalid the block [bh] in the [chain_store]. *) - val unmark_invalid : - chain_store -> Tezos_crypto.Block_hash.t -> unit tzresult Lwt.t + val unmark_invalid : chain_store -> Block_hash.t -> unit tzresult Lwt.t (** [descriptor block] returns the pair (hash x level) of [block]. *) val descriptor : block -> block_descriptor (** {3 Block field accessors} *) - val hash : block -> Tezos_crypto.Block_hash.t + val hash : block -> Block_hash.t val header : block -> Block_header.t @@ -548,30 +526,30 @@ module Block : sig val proto_level : block -> int - val predecessor : block -> Tezos_crypto.Block_hash.t + val predecessor : block -> Block_hash.t val timestamp : block -> Time.Protocol.t val validation_passes : block -> int - val operations_hash : block -> Tezos_crypto.Operation_list_list_hash.t + val operations_hash : block -> Operation_list_list_hash.t val fitness : block -> Fitness.t - val context_hash : block -> Tezos_crypto.Context_hash.t + val context_hash : block -> Context_hash.t val protocol_data : block -> bytes - val block_metadata_hash : block -> Tezos_crypto.Block_metadata_hash.t option + val block_metadata_hash : block -> Block_metadata_hash.t option val operations_metadata_hashes : - block -> Tezos_crypto.Operation_metadata_hash.t list list option + block -> Operation_metadata_hash.t list list option val operations_metadata_hashes_path : - block -> int -> Tezos_crypto.Operation_metadata_hash.t list option + block -> int -> Operation_metadata_hash.t list option val all_operations_metadata_hash : - block -> Tezos_crypto.Operation_metadata_list_list_hash.t option + block -> Operation_metadata_list_list_hash.t option (** {3 Block metadata field accessors} *) @@ -589,21 +567,16 @@ module Block : sig (** [operations_path block nth] computes the [nth] operations list of [block] along with the hash of all operations. *) val operations_path : - block -> - int -> - Operation.t list * Tezos_crypto.Operation_list_list_hash.path + block -> int -> Operation.t list * Operation_list_list_hash.path (** [operations_hashes_path block nth] computes the [nth] operations hash list of [block] along with the hash of all operations. *) val operations_hashes_path : - block -> - int -> - Tezos_crypto.Operation_hash.t list - * Tezos_crypto.Operation_list_list_hash.path + block -> int -> Operation_hash.t list * Operation_list_list_hash.path (** [all_operation_hashes block] computes the hash of all operations in [block]. *) - val all_operation_hashes : block -> Tezos_crypto.Operation_hash.t list list + val all_operation_hashes : block -> Operation_hash.t list list end (** The module for handling chain-related operations such as setting @@ -627,7 +600,7 @@ module Chain : sig val global_store : chain_store -> store (** [chain_id chain_store] returns chain id of [chain_store]. *) - val chain_id : chain_store -> Tezos_crypto.Chain_id.t + val chain_id : chain_store -> Chain_id.t (** [chain_dir chain_store] returns the path of directory of [chain_store].*) @@ -741,16 +714,12 @@ module Chain : sig which might happen when a new head concurrently arrives just before this operation being called. *) val set_mempool : - chain_store -> - head:Tezos_crypto.Block_hash.t -> - Mempool.t -> - unit tzresult Lwt.t + chain_store -> head:Block_hash.t -> Mempool.t -> unit tzresult Lwt.t (** [live_blocks chain_store] returns the set of previously computed live blocks for the current_head's [chain_store]. *) val live_blocks : - chain_store -> - (Tezos_crypto.Block_hash.Set.t * Tezos_crypto.Operation_hash.Set.t) Lwt.t + chain_store -> (Block_hash.Set.t * Operation_hash.Set.t) Lwt.t (** [compute_live_blocks ~block chain_store] computes the set of live blocks and live operations relative to [block]. Does nothing @@ -762,8 +731,7 @@ module Chain : sig val compute_live_blocks : chain_store -> block:Block.t -> - (Tezos_crypto.Block_hash.Set.t * Tezos_crypto.Operation_hash.Set.t) tzresult - Lwt.t + (Block_hash.Set.t * Operation_hash.Set.t) tzresult Lwt.t (** [set_head ?trigger_gc_callback chain_store block] promotes the [block] as head of the [chain_store] and triggers an @@ -868,7 +836,7 @@ module Chain : sig (** [testchain_forked_block testchain] returns the hash of the forked block of its parent chain.*) - val testchain_forked_block : testchain -> Tezos_crypto.Block_hash.t + val testchain_forked_block : testchain -> Block_hash.t (** [testchain_store testchain] returns the chain store associated to this [testchain]. *) @@ -882,11 +850,11 @@ module Chain : sig registers this chain in the set of activated testchains. *) val fork_testchain : chain_store -> - testchain_id:Tezos_crypto.Chain_id.t -> + testchain_id:Chain_id.t -> forked_block:Block.t -> - genesis_hash:Tezos_crypto.Block_hash.t -> + genesis_hash:Block_hash.t -> genesis_header:Block_header.t -> - test_protocol:Tezos_crypto.Protocol_hash.t -> + test_protocol:Protocol_hash.t -> expiration:Time.Protocol.t -> testchain tzresult Lwt.t @@ -907,9 +875,7 @@ module Chain : sig (** [find_protocol chain_store ~protocol_level] returns the protocol with the level [protocol_level]. *) val find_protocol : - chain_store -> - protocol_level:int -> - Tezos_crypto.Protocol_hash.t option Lwt.t + chain_store -> protocol_level:int -> Protocol_hash.t option Lwt.t (** [all_protocol_levels chain_store] returns all the protocols registered in [chain_store]. *) @@ -928,7 +894,7 @@ module Chain : sig chain_store -> ?pred:Block.block -> ?protocol_level:int -> - Block.block * Tezos_crypto.Protocol_hash.t -> + Block.block * Protocol_hash.t -> unit tzresult Lwt.t (** [may_update_ancestor_protocol_level chain_store ~head] tries to @@ -955,8 +921,8 @@ module Chain : sig in [chain_store]. *) val set_rpc_directory : chain_store -> - protocol_hash:Tezos_crypto.Protocol_hash.t -> - next_protocol_hash:Tezos_crypto.Protocol_hash.t -> + protocol_hash:Protocol_hash.t -> + next_protocol_hash:Protocol_hash.t -> (chain_store * Block.t) Tezos_rpc.Directory.t -> unit Lwt.t @@ -964,7 +930,7 @@ module Chain : sig [callback] that may be triggered during a block store merge in order to garbage-collect old contexts. *) val register_gc_callback : - chain_store -> (Tezos_crypto.Context_hash.t -> unit tzresult Lwt.t) -> unit + chain_store -> (Context_hash.t -> unit tzresult Lwt.t) -> unit end (** [global_block_watcher global_store] instantiates a new block @@ -976,15 +942,15 @@ val global_block_watcher : module Protocol : sig (** [mem global_store ph] checks the existence of the protocol [ph] in [global_store]. *) - val mem : store -> Tezos_crypto.Protocol_hash.t -> bool + val mem : store -> Protocol_hash.t -> bool (** [all global_store ph] returns the set of all stored protocol in [global_store]. *) - val all : store -> Tezos_crypto.Protocol_hash.Set.t + val all : store -> Protocol_hash.Set.t (** [read global_store ph] reads the protocol [ph] from the [global_store]. Returns [None] if it does not exist. *) - val read : store -> Tezos_crypto.Protocol_hash.t -> Protocol.t option Lwt.t + val read : store -> Protocol_hash.t -> Protocol.t option Lwt.t (** [store global_store ph protocol] stores the [protocol] under the hash [ph] in the [global_store]. Returns [None] if it already @@ -993,22 +959,16 @@ module Protocol : sig {b Warning} No hash check is made, the caller must be careful when storing protocols. *) val store : - store -> - Tezos_crypto.Protocol_hash.t -> - Protocol.t -> - Tezos_crypto.Protocol_hash.t option Lwt.t + store -> Protocol_hash.t -> Protocol.t -> Protocol_hash.t option Lwt.t (** [store_raw global_store ph bytes] raw version of [store]. *) val store_raw : - store -> - Tezos_crypto.Protocol_hash.t -> - bytes -> - Tezos_crypto.Protocol_hash.t option Lwt.t + store -> Protocol_hash.t -> bytes -> Protocol_hash.t option Lwt.t (** [protocol_watcher global_store] instantiates a new protocol watcher in [global_store]. *) val protocol_watcher : - store -> Tezos_crypto.Protocol_hash.t Lwt_stream.t * Lwt_watcher.stopper + store -> Protocol_hash.t Lwt_stream.t * Lwt_watcher.stopper end (** The utility module used to traverse the chain. *) @@ -1057,9 +1017,7 @@ module Unsafe : sig val get_block_store : chain_store -> Block_store.block_store val load_testchain : - chain_store -> - chain_id:Tezos_crypto.Chain_id.t -> - Chain.testchain option tzresult Lwt.t + chain_store -> chain_id:Chain_id.t -> Chain.testchain option tzresult Lwt.t (** [set_head chain_store block] sets the block as the current head of [chain_store] without checks. *) @@ -1093,7 +1051,7 @@ module Unsafe : sig val set_protocol_level : chain_store -> protocol_level:int -> - Block.block * Tezos_crypto.Protocol_hash.t -> + Block.block * Protocol_hash.t -> unit tzresult Lwt.t (** Snapshots utility functions *) @@ -1124,7 +1082,7 @@ module Unsafe : sig ?notify:(unit -> unit Lwt.t) -> [`Store_dir] Naming.directory -> genesis:Genesis.t -> - genesis_context_hash:Tezos_crypto.Context_hash.t -> + genesis_context_hash:Context_hash.t -> floating_blocks_stream:Block_repr.block Lwt_stream.t -> new_head_with_metadata:Block_repr.block -> protocol_levels:Protocol_levels.activation_block Protocol_levels.t -> diff --git a/src/lib_store/unix/test/alpha_utils.ml b/src/lib_store/unix/test/alpha_utils.ml index f8e8e3239404..f058b708b63c 100644 --- a/src/lib_store/unix/test/alpha_utils.ml +++ b/src/lib_store/unix/test/alpha_utils.ml @@ -59,58 +59,49 @@ end module Account = struct type t = { - pkh : Tezos_crypto.Signature.Public_key_hash.t; - pk : Tezos_crypto.Signature.Public_key.t; - sk : Tezos_crypto.Signature.Secret_key.t; + pkh : Signature.Public_key_hash.t; + pk : Signature.Public_key.t; + sk : Signature.Secret_key.t; } let pp fmt {pkh; pk; sk} = Format.fprintf fmt "pkh: %a@ pk: %a@ nsk: %a" - Tezos_crypto.Signature.Public_key_hash.pp + Signature.Public_key_hash.pp pkh - Tezos_crypto.Signature.Public_key.pp + Signature.Public_key.pp pk - Tezos_crypto.Signature.Secret_key.pp + Signature.Secret_key.pp sk type account = t - let known_accounts = Tezos_crypto.Signature.Public_key_hash.Table.create 17 + let known_accounts = Signature.Public_key_hash.Table.create 17 let new_account ?seed () = - let pkh, pk, sk = Tezos_crypto.Signature.generate_key ?seed () in + let pkh, pk, sk = Signature.generate_key ?seed () in let account = {pkh; pk; sk} in - Tezos_crypto.Signature.Public_key_hash.Table.add known_accounts pkh account ; + Signature.Public_key_hash.Table.add known_accounts pkh account ; account let add_account ({pkh; _} as account) = - Tezos_crypto.Signature.Public_key_hash.Table.replace - known_accounts - pkh - account + Signature.Public_key_hash.Table.replace known_accounts pkh account let activator_account = new_account () let find pkh = let open Lwt_result_syntax in - match - Tezos_crypto.Signature.Public_key_hash.Table.find known_accounts pkh - with + match Signature.Public_key_hash.Table.find known_accounts pkh with | Some v -> return v - | None -> - failwith - "Missing account: %a" - Tezos_crypto.Signature.Public_key_hash.pp - pkh + | None -> failwith "Missing account: %a" Signature.Public_key_hash.pp pkh let find_alternate pkh = let exception Found of t in try - Tezos_crypto.Signature.Public_key_hash.Table.iter + Signature.Public_key_hash.Table.iter (fun pkh' account -> - if not (Tezos_crypto.Signature.Public_key_hash.equal pkh pkh') then + if not (Signature.Public_key_hash.equal pkh pkh') then raise (Found account)) known_accounts ; raise Not_found @@ -136,9 +127,7 @@ module Account = struct let new_commitment ?seed () = let open Lwt_result_syntax in - let pkh, pk, sk = - Tezos_crypto.Signature.generate_key ?seed ~algo:Ed25519 () - in + let pkh, pk, sk = Signature.generate_key ?seed ~algo:Ed25519 () in let unactivated_account = {pkh; pk; sk} in let open Commitment in let pkh = match pkh with Ed25519 pkh -> pkh | _ -> assert false in @@ -243,11 +232,7 @@ let get_next_baker_excluding rpc_ctxt excludes block = let {Plugin.RPC.Baking_rights.delegate = pkh; timestamp; round; _} = List.find (fun {Plugin.RPC.Baking_rights.delegate; _} -> - not - (List.mem - ~equal:Tezos_crypto.Signature.Public_key_hash.equal - delegate - excludes)) + not (List.mem ~equal:Signature.Public_key_hash.equal delegate excludes)) bakers |> WithExceptions.Option.get ~loc:__LOC__ in @@ -299,7 +284,7 @@ module Forge = struct operations_hash; proto_level; validation_passes = List.length Main.validation_passes; - context = Tezos_crypto.Context_hash.zero (* to update later *); + context = Context_hash.zero (* to update later *); } let set_seed_nonce_hash seed_nonce_hash {baker; shell; contents} = @@ -316,7 +301,7 @@ module Forge = struct (shell, contents) in let signature = - Tezos_crypto.Signature.sign + Signature.sign ~watermark:Block_header.(to_watermark (Block_header chain_id)) delegate.sk unsigned_bytes @@ -351,9 +336,9 @@ module Forge = struct | {expected_commitment = false; _} -> None in let operations_hash = - Tezos_crypto.Operation_list_list_hash.compute + Operation_list_list_hash.compute (List.map - Tezos_crypto.Operation_list_hash.compute + Operation_list_hash.compute (List.map (List.map Operation.hash_packed) operations)) in let shell = @@ -426,9 +411,9 @@ let default_accounts = let open Account in let to_account (pkh, pk, sk) = { - pkh = Tezos_crypto.Signature.Public_key_hash.of_b58check_exn pkh; - pk = Tezos_crypto.Signature.Public_key.of_b58check_exn pk; - sk = Tezos_crypto.Signature.Secret_key.of_b58check_exn sk; + pkh = Signature.Public_key_hash.of_b58check_exn pkh; + pk = Signature.Public_key.of_b58check_exn pk; + sk = Signature.Secret_key.of_b58check_exn sk; } in let accounts = List.map to_account initial_accounts in @@ -455,9 +440,9 @@ let patch_context ctxt ~json = predecessor = Test_utils.genesis.block; timestamp = Test_utils.genesis.time; validation_passes = 0; - operations_hash = Tezos_crypto.Operation_list_list_hash.empty; + operations_hash = Operation_list_list_hash.empty; fitness = []; - context = Tezos_crypto.Context_hash.zero; + context = Context_hash.zero; } in let proto_params = @@ -465,7 +450,7 @@ let patch_context ctxt ~json = in let* ctxt = Context_ops.add ctxt ["version"] (Bytes.of_string "genesis") in let* ctxt = Context_ops.add ctxt protocol_param_key proto_params in - let* r = Main.init Tezos_crypto.Chain_id.zero ctxt shell in + let* r = Main.init Chain_id.zero ctxt shell in match r with | Error e -> failwith "%a" Environment.Error_monad.pp_trace e | Ok {context; _} -> return_ok context @@ -531,9 +516,7 @@ let apply ctxt chain_id ~policy ?(operations = empty_operations) pred = let* {shell; contents; baker} = Forge.forge_header rpc_ctxt ?policy ~operations pred in - let protocol_data = - {Block_header.contents; signature = Tezos_crypto.Signature.zero} - in + let protocol_data = {Block_header.contents; signature = Signature.zero} in let*! context = match Store.Block.block_metadata_hash pred with | None -> Lwt.return ctxt @@ -624,7 +607,7 @@ let apply ctxt chain_id ~policy ?(operations = empty_operations) pred = header.protocol_data in let block_hash_metadata = - Some (Tezos_crypto.Block_metadata_hash.hash_bytes [block_header_metadata]) + Some (Block_metadata_hash.hash_bytes [block_header_metadata]) in let block_header = {Tezos_base.Block_header.shell = header.shell; protocol_data} @@ -632,8 +615,7 @@ let apply ctxt chain_id ~policy ?(operations = empty_operations) pred = let operations_metadata_hashes = Some (List.map - (List.map (fun r -> - Tezos_crypto.Operation_metadata_hash.hash_bytes [r])) + (List.map (fun r -> Operation_metadata_hash.hash_bytes [r])) (WithExceptions.List.init ~loc:__LOC__ 4 (fun _ -> []))) in return @@ -716,7 +698,7 @@ let apply_and_store chain_store ?(synchronous_merge = true) ?policy return b | None -> let h = Tezos_base.Block_header.hash block_header in - Format.eprintf "block %a already stored@." Tezos_crypto.Block_hash.pp h ; + Format.eprintf "block %a already stored@." Block_hash.pp h ; Store.Block.read_block chain_store h let bake chain_store ?synchronous_merge ?policy ?operation ?operations pred = diff --git a/src/lib_store/unix/test/dune b/src/lib_store/unix/test/dune index adaf703aeea3..649d9ff36bc9 100644 --- a/src/lib_store/unix/test/dune +++ b/src/lib_store/unix/test/dune @@ -13,6 +13,7 @@ test_testchain) (libraries tezos-base + tezos-crypto tezos-context-ops tezos-store.shared tezos-store.unix @@ -32,6 +33,7 @@ (flags (:standard) -open Tezos_base.TzPervasives + -open Tezos_crypto -open Tezos_context_ops -open Tezos_store_shared -open Tezos_store_unix diff --git a/src/lib_store/unix/test/test_block_store.ml b/src/lib_store/unix/test/test_block_store.ml index a1d2611a3b19..6326002094c5 100644 --- a/src/lib_store/unix/test/test_block_store.ml +++ b/src/lib_store/unix/test/test_block_store.ml @@ -158,11 +158,11 @@ let test_storing_and_access_predecessors block_store = | None -> Alcotest.fail "expected predecessor but none found" | Some h -> Alcotest.check - (module Tezos_crypto.Block_hash) + (module Block_hash) (Format.asprintf "get %d-th predecessor of %a (%ld)" distance - Tezos_crypto.Block_hash.pp + Block_hash.pp hash level) h @@ -201,8 +201,7 @@ let make_n_consecutive_cycles pred ~cycle_length ~nb_cycles = Lwt.return (List.rev acc, head) | n -> let cycle_length = - if Tezos_crypto.Block_hash.equal (fst pred) genesis_hash then - cycle_length - 1 + if Block_hash.equal (fst pred) genesis_hash then cycle_length - 1 else cycle_length in let lafl = max 0l (snd pred) in @@ -662,10 +661,10 @@ let wrap_test ?(keep_dir = false) (name, g) = let genesis_block = Block_repr.create_genesis_block ~genesis:Test_utils.genesis - Tezos_crypto.Context_hash.zero + Context_hash.zero in let store_dir = Naming.store_dir ~dir_path in - let chain_dir = Naming.chain_dir store_dir Tezos_crypto.Chain_id.zero in + let chain_dir = Naming.chain_dir store_dir Chain_id.zero in let*! () = Lwt_utils_unix.create_dir (Naming.dir_path chain_dir) in let*! r = Block_store.create chain_dir ~genesis_block in match r with diff --git a/src/lib_store/unix/test/test_cemented_store.ml b/src/lib_store/unix/test/test_cemented_store.ml index b0aaac1b8b60..c7b0daf17bc8 100644 --- a/src/lib_store/unix/test/test_cemented_store.ml +++ b/src/lib_store/unix/test/test_cemented_store.ml @@ -107,9 +107,7 @@ let wrap_cemented_store_test (name, f) = let*! r = run (fun base_dir -> let store_dir = Naming.store_dir ~dir_path:base_dir in - let chain_dir = - Naming.chain_dir store_dir Tezos_crypto.Chain_id.zero - in + let chain_dir = Naming.chain_dir store_dir Chain_id.zero in let*! () = Lwt_unix.mkdir (Naming.dir_path chain_dir) 0o700 in let* cemented_store = Cemented_block_store.init ~readonly:false chain_dir diff --git a/src/lib_store/unix/test/test_consistency.ml b/src/lib_store/unix/test/test_consistency.ml index bab78db4ce6e..11134830e4de 100644 --- a/src/lib_store/unix/test/test_consistency.ml +++ b/src/lib_store/unix/test/test_consistency.ml @@ -176,7 +176,7 @@ let test_protocol_level_consistency_drop_one history_mode nth in let*! ctxt = Store.Block.context_exn chain_store b in let*! proto_block = Context_ops.get_protocol ctxt in - assert (Tezos_crypto.Protocol_hash.(proto = proto_block)) ; + assert (Protocol_hash.(proto = proto_block)) ; return_unit) (Int32.to_int savepoint_level -- Int32.to_int (Store.Block.level current_head)) @@ -257,7 +257,7 @@ let test_protocol_level_consistency_remove_file history_mode in let*! ctxt = Store.Block.context_exn chain_store b in let*! proto_block = Context_ops.get_protocol ctxt in - assert (Tezos_crypto.Protocol_hash.(proto = proto_block)) ; + assert (Protocol_hash.(proto = proto_block)) ; return_unit) (Int32.to_int savepoint_level -- Int32.to_int (Store.Block.level current_head)) diff --git a/src/lib_store/unix/test/test_locator.ml b/src/lib_store/unix/test/test_locator.ml index ed5ddb181d4f..3ca1629aa95b 100644 --- a/src/lib_store/unix/test/test_locator.ml +++ b/src/lib_store/unix/test/test_locator.ml @@ -37,11 +37,11 @@ module Assert = Lib_test.Assert (** Basic blocks *) let genesis_hash = - Tezos_crypto.Block_hash.of_b58check_exn + Block_hash.of_b58check_exn "BLockGenesisGenesisGenesisGenesisGenesisGeneskvg68z" let genesis_protocol = - Tezos_crypto.Protocol_hash.of_b58check_exn + Protocol_hash.of_b58check_exn "ProtoDemoNoopsDemoNoopsDemoNoopsDemoNoopsDemo6XBoYp" let proto = @@ -89,7 +89,7 @@ let init_chain ?(history_mode = History_mode.Archive) base_dir = let zero = Bytes.create 0 (* adds n blocks on top of an initialized chain *) -let make_empty_chain chain_store n : Tezos_crypto.Block_hash.t Lwt.t = +let make_empty_chain chain_store n : Block_hash.t Lwt.t = let open Lwt_result_syntax in let*! genesis = Store.Block.read_block_opt chain_store genesis_hash in let genesis = WithExceptions.Option.get ~loc:__LOC__ genesis in @@ -191,8 +191,7 @@ let make_multiple_protocol_chain (chain_store : Store.Chain.t) in let block_metadata = zero in let block_metadata_hash = - Option.some - @@ Tezos_crypto.Block_metadata_hash.hash_bytes [block_metadata] + Option.some @@ Block_metadata_hash.hash_bytes [block_metadata] in (* make some cycles *) let last_allowed_fork_level = @@ -268,8 +267,8 @@ let rec repeat f n = (* ----------------------------------------------------- *) (* returns the predecessor at distance one, reading the header *) -let linear_predecessor chain_store (bh : Tezos_crypto.Block_hash.t) : - Tezos_crypto.Block_hash.t option Lwt.t = +let linear_predecessor chain_store (bh : Block_hash.t) : + Block_hash.t option Lwt.t = let open Lwt_syntax in let* b = Store.Block.read_block_opt chain_store bh in let b = WithExceptions.Option.get ~loc:__LOC__ b in @@ -277,8 +276,8 @@ let linear_predecessor chain_store (bh : Tezos_crypto.Block_hash.t) : match o with None -> None | Some pred -> Some (Store.Block.hash pred) (* returns the predecessors at distance n, traversing all n intermediate blocks *) -let linear_predecessor_n chain_store (bh : Tezos_crypto.Block_hash.t) - (distance : int) : Tezos_crypto.Block_hash.t option Lwt.t = +let linear_predecessor_n chain_store (bh : Block_hash.t) (distance : int) : + Block_hash.t option Lwt.t = let open Lwt_syntax in (* let _ = Printf.printf "LP: %4i " distance; print_block_h chain bh in *) if distance < 1 then invalid_arg "distance<1" @@ -418,7 +417,7 @@ let bench_locator base_dir = @@ List.iter2 ~when_different_lengths:(TzTrace.make @@ Exn (Failure __LOC__)) (fun hn ho -> - if not (Tezos_crypto.Block_hash.equal hn ho) then + if not (Block_hash.equal hn ho) then Assert.fail_msg "Invalid locator %i" max_size) l_exp l_lin @@ -592,13 +591,13 @@ let test_protocol_locator base_dir = in Assert.assert_true "last block in locator is the checkpoint" - (Tezos_crypto.Block_hash.equal last_hash caboose_hash) ; + (Block_hash.equal last_hash caboose_hash) ; let first_hash = (WithExceptions.Option.get ~loc:__LOC__ @@ List.last_opt steps).block in Assert.assert_true "first block in locator is the head" - (Tezos_crypto.Block_hash.equal first_hash head_hash) ; + (Block_hash.equal first_hash head_hash) ; return_unit let wrap n f = diff --git a/src/lib_store/unix/test/test_snapshots.ml b/src/lib_store/unix/test/test_snapshots.ml index f689a42df204..2a8cabe62efd 100644 --- a/src/lib_store/unix/test/test_snapshots.ml +++ b/src/lib_store/unix/test/test_snapshots.ml @@ -160,7 +160,7 @@ let export_import ~test_descr ~previously_baked_blocks ?exported_block_hash | Some hash -> Assert.equal ~msg:("export with given hash: " ^ test_descr) - ~eq:Tezos_crypto.Block_hash.equal + ~eq:Block_hash.equal (Store.Block.hash head') hash ; Lwt.return head' @@ -168,7 +168,7 @@ let export_import ~test_descr ~previously_baked_blocks ?exported_block_hash let*! checkpoint = Store.Chain.checkpoint chain_store in Assert.equal ~msg:("export checkpoint: " ^ test_descr) - ~eq:Tezos_crypto.Block_hash.equal + ~eq:Block_hash.equal (Store.Block.hash head') (fst checkpoint) ; Lwt.return head' @@ -255,7 +255,7 @@ let check_baking_continuity ~test_descr ~exported_chain_store Assert.equal ~msg:("checkpoint equality: " ^ test_descr) ~pp:(fun ppf (hash, level) -> - Format.fprintf ppf "%a (%ld)" Tezos_crypto.Block_hash.pp hash level) + Format.fprintf ppf "%a (%ld)" Block_hash.pp hash level) checkpoint checkpoint' ; return_unit diff --git a/src/lib_store/unix/test/test_store.ml b/src/lib_store/unix/test/test_store.ml index e82a43c444f9..cacd0db7a28e 100644 --- a/src/lib_store/unix/test/test_store.ml +++ b/src/lib_store/unix/test/test_store.ml @@ -60,9 +60,7 @@ let pp_print_list fmt l = Format.fprintf fmt "@[%a@]" - (Format.pp_print_list - ~pp_sep:Format.pp_print_space - Tezos_crypto.Block_hash.pp_short) + (Format.pp_print_list ~pp_sep:Format.pp_print_space Block_hash.pp_short) (List.map Store.Block.hash l) let test_path chain_store tbl = @@ -109,9 +107,7 @@ let test_ancestor chain_store tbl = | Some a -> if not - (Tezos_crypto.Block_hash.equal - (Store.Block.hash a) - (Store.Block.hash expected)) + (Block_hash.equal (Store.Block.hash a) (Store.Block.hash expected)) then Assert.fail_msg "bad ancestor %s %s" h1 h2 ; Lwt.return_unit in @@ -167,12 +163,8 @@ let test_locator chain_store tbl = (List.length expected) ; iter2_exn (fun h h2 -> - if - not - (Tezos_crypto.Block_hash.equal - h - (Store.Block.hash @@ vblock tbl h2)) - then Assert.fail_msg "Invalid locator %s (expected: %s)" h1 h2) + if not (Block_hash.equal h (Store.Block.hash @@ vblock tbl h2)) then + Assert.fail_msg "Invalid locator %s (expected: %s)" h1 h2) history expected ; Lwt.return_unit @@ -201,10 +193,8 @@ let compare s name heads l = List.iter (fun bname -> let hash = Store.Block.hash (vblock s bname) in - if - not - (List.exists (fun bh -> Tezos_crypto.Block_hash.equal hash bh) heads) - then Assert.fail_msg "missing block in known_heads (%s: %s)" name bname) + if not (List.exists (fun bh -> Block_hash.equal hash bh) heads) then + Assert.fail_msg "missing block in known_heads (%s: %s)" name bname) l let test_known_heads chain_store tbl = @@ -324,7 +314,7 @@ let test_new_blocks chain_store tbl = in if not - (Tezos_crypto.Block_hash.equal + (Block_hash.equal (Store.Block.hash ancestor) (Store.Block.hash @@ vblock tbl expected_ancestor)) then @@ -343,7 +333,7 @@ let test_new_blocks chain_store tbl = (fun h1 h2 -> if not - (Tezos_crypto.Block_hash.equal + (Block_hash.equal (Store.Block.hash h1) (Store.Block.hash @@ vblock tbl h2)) then @@ -393,7 +383,7 @@ let test_basic_checkpoint chain_store table = | None -> return_unit in if - (not (Tezos_crypto.Block_hash.equal c_block (Store.Block.hash block))) + (not (Block_hash.equal c_block (Store.Block.hash block))) && Int32.equal c_level (Store.Block.level block) then Assert.fail_msg "unexpected checkpoint" else return_unit @@ -543,7 +533,7 @@ let test_reach_target chain_store table = then if Int32.equal header.shell.level checkpoint_level - && not (Tezos_crypto.Block_hash.equal checkpoint_hash c_hash) + && not (Block_hash.equal checkpoint_hash c_hash) then Assert.fail_msg "checkpoint error" else let* _ = Store.Chain.set_head chain_store (vblock table "A2") in @@ -604,9 +594,8 @@ let test_not_may_update_target chain_store table = let testable_hash = Alcotest.testable - (fun fmt h -> - Format.fprintf fmt "%s" (Tezos_crypto.Block_hash.to_b58check h)) - Tezos_crypto.Block_hash.equal + (fun fmt h -> Format.fprintf fmt "%s" (Block_hash.to_b58check h)) + Block_hash.equal let init_block_of_identifier_test chain_store table = let open Lwt_result_syntax in @@ -634,7 +623,7 @@ let assert_failing_block_of_identifier ?(init = init_block_of_identifier_test) match r with | Ok b -> Assert.fail_msg - ~given:(Store.Block.hash b |> Tezos_crypto.Block_hash.to_b58check) + ~given:(Store.Block.hash b |> Block_hash.to_b58check) "retrieving the block did not failed as expected" | _ -> return_unit diff --git a/src/lib_store/unix/test/test_testchain.ml b/src/lib_store/unix/test/test_testchain.ml index bc620ba4b490..68fa63ce809b 100644 --- a/src/lib_store/unix/test/test_testchain.ml +++ b/src/lib_store/unix/test/test_testchain.ml @@ -29,10 +29,9 @@ let fork_testchain chain_store (blocks, forked_block) = let open Lwt_result_syntax in let forked_block_hash = Store.Block.hash forked_block in let genesis_hash = - Tezos_crypto.Block_hash.hash_bytes - [Tezos_crypto.Block_hash.to_bytes forked_block_hash] + Block_hash.hash_bytes [Block_hash.to_bytes forked_block_hash] in - let testchain_id = Tezos_crypto.Chain_id.of_block_hash genesis_hash in + let testchain_id = Chain_id.of_block_hash genesis_hash in let head_header = Store.Block.header forked_block in let test_protocol = Tezos_protocol_alpha.Protocol.hash in let expiration = Time.Protocol.epoch in @@ -146,10 +145,7 @@ let test_shutdown store = | Some testchain'' -> let testchain_store'' = Store.Chain.testchain_store testchain'' in let testchain_id'' = Store.Chain.chain_id testchain_store'' in - Assert.equal - ~eq:Tezos_crypto.Chain_id.equal - testchain_id - testchain_id'' ; + Assert.equal ~eq:Chain_id.equal testchain_id testchain_id'' ; assert_presence_in_store testchain_store'' blocks)) let tests = diff --git a/src/lib_store/unix/test/test_utils.ml b/src/lib_store/unix/test/test_utils.ml index 88ea48e22084..b3b08da6d56a 100644 --- a/src/lib_store/unix/test/test_utils.ml +++ b/src/lib_store/unix/test/test_utils.ml @@ -66,7 +66,7 @@ let equal_metadata ?msg m1 m2 = Assert.equal ?msg ~pp ~eq m1 m2 let genesis_hash = - Tezos_crypto.Block_hash.of_b58check_exn + Block_hash.of_b58check_exn "BLockGenesisGenesisGenesisGenesisGenesisf79b5d1CoW2" let genesis = @@ -166,7 +166,7 @@ let dummy_patch_context ctxt = let*! ctxt = Context_ops.add ctxt ["protocol_parameters"] proto_params in let*! res = Protocol.Main.init - Tezos_crypto.Chain_id.zero + Chain_id.zero ctxt { level = 0l; @@ -174,9 +174,9 @@ let dummy_patch_context ctxt = predecessor = genesis.block; timestamp = genesis.time; validation_passes = 0; - operations_hash = Tezos_crypto.Operation_list_list_hash.empty; + operations_hash = Operation_list_list_hash.empty; fitness = []; - context = Tezos_crypto.Context_hash.zero; + context = Context_hash.zero; } in let*? {context; _} = Environment.wrap_tzresult res in @@ -338,8 +338,7 @@ let wrap_simple_store_init_test ?history_mode ?(speed = `Quick) ?patch_context (wrap_simple_store_init ?history_mode ?patch_context ?keep_dir ?with_gc f) let make_raw_block ?min_lafl ?(max_operations_ttl = default_max_operations_ttl) - ?(constants = default_protocol_constants) - ?(context = Tezos_crypto.Context_hash.zero) + ?(constants = default_protocol_constants) ?(context = Context_hash.zero) (pred_block_hash, pred_block_level) = let level = Int32.succ pred_block_level in let header = @@ -352,7 +351,7 @@ let make_raw_block ?min_lafl ?(max_operations_ttl = default_max_operations_ttl) timestamp = Time.Protocol.(add epoch (Int64.of_int32 level)); validation_passes = List.length Tezos_protocol_alpha.Protocol.Main.validation_passes; - operations_hash = Tezos_crypto.Operation_list_list_hash.zero; + operations_hash = Operation_list_list_hash.zero; fitness = [Bytes.create 8]; context; }; @@ -415,13 +414,12 @@ let make_raw_block ?min_lafl ?(max_operations_ttl = default_max_operations_ttl) header; operations; block_metadata_hash = - (if Random.bool () then Some Tezos_crypto.Block_metadata_hash.zero - else None); + (if Random.bool () then Some Block_metadata_hash.zero else None); operations_metadata_hashes = (if Random.bool () then Some (List.map - (List.map (fun _ -> Tezos_crypto.Operation_metadata_hash.zero)) + (List.map (fun _ -> Operation_metadata_hash.zero)) operations) else None); }; @@ -434,13 +432,13 @@ let prune_block block = block.Block_repr.metadata <- None let pp_block fmt b = let h, lvl = Store.Block.descriptor b in - Format.fprintf fmt "%a (%ld)" Tezos_crypto.Block_hash.pp h lvl + Format.fprintf fmt "%a (%ld)" Block_hash.pp h lvl let raw_descriptor b = (Block_repr.hash b, Block_repr.level b) let pp_raw_block fmt b = let h, lvl = raw_descriptor b in - Format.fprintf fmt "%a (%ld)" Tezos_crypto.Block_hash.pp h lvl + Format.fprintf fmt "%a (%ld)" Block_hash.pp h lvl let store_raw_block chain_store (raw_block : Block_repr.t) = let open Lwt_syntax in @@ -489,7 +487,7 @@ let store_raw_block chain_store (raw_block : Block_repr.t) = | Ok None -> Alcotest.failf "store_raw_block: could not store block %a (%ld)" - Tezos_crypto.Block_hash.pp + Block_hash.pp (Block_repr.hash raw_block) (Block_repr.level raw_block) | Error _ as err -> Lwt.return err @@ -791,8 +789,7 @@ module Example_tree = struct (fun k b o -> match o with | None -> - if Tezos_crypto.Block_hash.equal block_hash (Store.Block.hash b) - then Some k + if Block_hash.equal block_hash (Store.Block.hash b) then Some k else None | x -> x) tbl -- GitLab From dda6999aceca0234d0e5ede3a265d1034a9670e4 Mon Sep 17 00:00:00 2001 From: vbot Date: Thu, 10 Nov 2022 09:44:59 +0100 Subject: [PATCH 04/19] Store: add resulting context hash semantics --- src/bin_node/node_replay_command.ml | 10 +- src/lib_shell_services/store_errors.ml | 38 ++- src/lib_store/mocked/block_store.ml | 24 +- src/lib_store/mocked/block_store.mli | 17 +- src/lib_store/mocked/store.ml | 179 +++++++++-- src/lib_store/shared/naming.ml | 6 + src/lib_store/shared/naming.mli | 10 +- src/lib_store/shared/store_events.ml | 17 + src/lib_store/shared/store_types.ml | 59 +++- src/lib_store/shared/store_types.mli | 32 +- src/lib_store/store.mli | 58 +++- src/lib_store/unix/block_repr_unix.ml | 12 + src/lib_store/unix/block_repr_unix.mli | 5 + src/lib_store/unix/block_store.ml | 184 ++++++++++- src/lib_store/unix/block_store.mli | 38 ++- src/lib_store/unix/consistency.ml | 111 +++++-- src/lib_store/unix/floating_block_index.ml | 121 ++++++- src/lib_store/unix/floating_block_index.mli | 18 ++ src/lib_store/unix/floating_block_store.ml | 148 ++++++--- src/lib_store/unix/floating_block_store.mli | 128 +++++--- src/lib_store/unix/reconstruction.ml | 110 +++---- src/lib_store/unix/reconstruction.mli | 1 - src/lib_store/unix/snapshots.ml | 174 +++++++--- src/lib_store/unix/store.ml | 339 ++++++++++++++++---- src/lib_store/unix/store.mli | 82 +++-- src/lib_store/unix/test/alpha_utils.ml | 1 + src/lib_store/unix/test/test_block_store.ml | 48 ++- src/lib_store/unix/test/test_consistency.ml | 5 +- src/lib_store/unix/test/test_locator.ml | 3 + src/lib_store/unix/test/test_utils.ml | 1 + 30 files changed, 1551 insertions(+), 428 deletions(-) diff --git a/src/bin_node/node_replay_command.ml b/src/bin_node/node_replay_command.ml index 5fbe77ee766e..49fc80837817 100644 --- a/src/bin_node/node_replay_command.ml +++ b/src/bin_node/node_replay_command.ml @@ -227,7 +227,13 @@ let replay_one_block strict main_chain_store validator_process block = if Store.Block.level block <= savepoint_level then tzfail Cannot_replay_below_savepoint else - let expected_context_hash = Store.Block.context_hash block in + let* protocol = Store.Block.protocol_hash main_chain_store block in + let* (module Proto) = Registered_protocol.get_result protocol in + let* expected_context_hash = + if Proto.expected_context_hash = `Current then + return (Store.Block.context_hash block) + else Store.Block.resulting_context_hash main_chain_store block + in let* metadata = Store.Block.get_block_metadata main_chain_store block in let expected_block_receipt_bytes = Store.Block.block_metadata metadata in let expected_operation_receipts = @@ -271,8 +277,6 @@ let replay_one_block strict main_chain_store validator_process block = when_ (not (Bytes.equal expected_block_receipt_bytes block_metadata_bytes)) (fun () -> - let* protocol = Store.Block.protocol_hash main_chain_store block in - let* (module Proto) = Registered_protocol.get_result protocol in let to_json block = Data_encoding.Json.construct Proto.block_header_metadata_encoding @@ Data_encoding.Binary.of_bytes_exn diff --git a/src/lib_shell_services/store_errors.ml b/src/lib_shell_services/store_errors.ml index 2888fc2d415b..666d419bff37 100644 --- a/src/lib_shell_services/store_errors.ml +++ b/src/lib_shell_services/store_errors.ml @@ -25,8 +25,13 @@ type error += | Block_not_found of {hash : Tezos_crypto.Block_hash.t; distance : int} + | Resulting_context_hash_not_found of { + hash : Tezos_crypto.Block_hash.t; + level : int32; + } | Bad_level of {head_level : Int32.t; given_level : Int32.t} | Block_metadata_not_found of Tezos_crypto.Block_hash.t + | Protocol_not_found of {protocol_level : int} | Cannot_switch_history_mode of { previous_mode : History_mode.t; next_mode : History_mode.t; @@ -44,7 +49,7 @@ type error += let () = register_error_kind `Permanent - ~id:"store.not_found" + ~id:"store.block_not_found" ~title:"Block not found" ~description:"Block not found" ~pp:(fun ppf (block_hash, distance) -> @@ -55,10 +60,28 @@ let () = Tezos_crypto.Block_hash.pp block_hash) Data_encoding.( - obj1 (req "block_not_found" @@ tup2 Tezos_crypto.Block_hash.encoding int8)) + obj1 (req "block_not_found" @@ tup2 Tezos_crypto.Block_hash.encoding int31)) (function | Block_not_found {hash; distance} -> Some (hash, distance) | _ -> None) (fun (hash, distance) -> Block_not_found {hash; distance}) ; + register_error_kind + `Permanent + ~id:"store.resulting_context_hash_not_found" + ~title:"Resulting context hash not found" + ~description:"Resulting context hash not found" + ~pp:(fun ppf (block_hash, level) -> + Format.fprintf + ppf + "Cannot find the resulting context hash for the block %a (level: %ld)." + Tezos_crypto.Block_hash.pp + block_hash + level) + Data_encoding.( + obj1 (req "block_not_found" @@ tup2 Tezos_crypto.Block_hash.encoding int32)) + (function + | Resulting_context_hash_not_found {hash; level} -> Some (hash, level) + | _ -> None) + (fun (hash, level) -> Resulting_context_hash_not_found {hash; level}) ; register_error_kind `Permanent ~id:"store.bad_level" @@ -91,6 +114,17 @@ let () = (function | Block_metadata_not_found block_hash -> Some block_hash | _ -> None) (fun block_hash -> Block_metadata_not_found block_hash) ; + register_error_kind + `Permanent + ~id:"store.protocol_not_found" + ~title:"Protocol not found" + ~description:"Protocol not found" + ~pp:(fun ppf protocol_level -> + Format.fprintf ppf "Unable to find protocol %d." protocol_level) + Data_encoding.(obj1 (req "protocol_level" int31)) + (function + | Protocol_not_found {protocol_level} -> Some protocol_level | _ -> None) + (fun protocol_level -> Protocol_not_found {protocol_level}) ; register_error_kind `Permanent ~id:"config_file.cannot_switch_history_mode" diff --git a/src/lib_store/mocked/block_store.ml b/src/lib_store/mocked/block_store.ml index aa77977052ce..299940d9ab7c 100644 --- a/src/lib_store/mocked/block_store.ml +++ b/src/lib_store/mocked/block_store.ml @@ -39,8 +39,8 @@ end) type block_store = { genesis_block : Block_repr.t; - block_table : Block_repr.t Block_table.t; - level_table : Block_repr.t Level_table.t; + block_table : (Block_repr.t * Context_hash.t) Block_table.t; + level_table : (Block_repr.t * Context_hash.t) Level_table.t; mutable caboose : Store_types.block_descriptor; } @@ -51,7 +51,7 @@ let genesis_block {genesis_block; _} = genesis_block let get_block_by_key (Block (hash, off)) {block_table; level_table; _} = match Block_table.find_opt block_table hash with | None -> None - | Some block as res -> + | Some (block, _) as res -> let off = Int32.of_int off in if off = 0l then res else @@ -66,12 +66,11 @@ let mem block_store key = let read_block ~(read_metadata : bool) block_store key = ignore read_metadata ; - Lwt_result_syntax.return @@ get_block_by_key key block_store + Lwt_result.return (Option.map fst (get_block_by_key key block_store)) let get_hash block_store key = - let open Lwt_result_syntax in let block_opt = get_block_by_key key block_store in - return (Option.map Block_repr.hash block_opt) + Lwt_result.return (Option.map (fun (b, _) -> Block_repr.hash b) block_opt) let read_block_metadata block_store key = let open Lwt_result_syntax in @@ -80,11 +79,14 @@ let read_block_metadata block_store key = | None -> return None | Some block -> return (Block_repr.metadata block) -let store_block block_store block = +let resulting_context_hash block_store ~expect_predecessor_context:_ key = + Lwt_result.return (Option.map snd (get_block_by_key key block_store)) + +let store_block block_store block resulting_context_hash = let level = Block_repr.level block in let hash = Block_repr.hash block in - Block_table.add block_store.block_table hash block ; - Level_table.add block_store.level_table level block ; + Block_table.add block_store.block_table hash (block, resulting_context_hash) ; + Level_table.add block_store.level_table level (block, resulting_context_hash) ; Lwt_result_syntax.return_unit let caboose {caboose; _} = Lwt.return caboose @@ -99,7 +101,9 @@ let create ?block_cache_limit:_ _chain_dir ~genesis_block = caboose = (Block_repr.hash genesis_block, Block_repr.level genesis_block); } in - let* () = store_block block_store genesis_block in + let* () = + store_block block_store genesis_block (Block_repr.context genesis_block) + in return block_store diff --git a/src/lib_store/mocked/block_store.mli b/src/lib_store/mocked/block_store.mli index 346f432f48e2..637058c779d2 100644 --- a/src/lib_store/mocked/block_store.mli +++ b/src/lib_store/mocked/block_store.mli @@ -183,6 +183,15 @@ val mem : block_store -> key -> bool tzresult Lwt.t unknown. *) val get_hash : block_store -> key -> Block_hash.t option tzresult Lwt.t +(** [resulting_context_hash block_store ~expect_predecessor_context key] + retrieves the resulting context hash from the block application, + corresponding to the given [key]. *) +val resulting_context_hash : + block_store -> + expect_predecessor_context:bool -> + key -> + Context_hash.t option tzresult Lwt.t + (** [read_block ~read_metadata block_store key] reads the block [key] in [block_store] if present. Return [None] if the block is unknown. If [read_metadata] is set to [true] it tries to retreive @@ -196,9 +205,11 @@ val read_block : val read_block_metadata : block_store -> key -> Block_repr.metadata option tzresult Lwt.t -(** [store_block block_store block] stores the [block] in the current - [RW] floating store. *) -val store_block : block_store -> Block_repr.t -> unit tzresult Lwt.t +(** [store_block block_store block resulting_context_hash] stores the + [block] in the current [RW] floating store with its associated + [resulting_context_hash]. *) +val store_block : + block_store -> Block_repr.t -> Context_hash.t -> unit tzresult Lwt.t (** In this faked implementation, the caboose is always equal to the genesis block. *) val caboose : block_store -> Store_types.block_descriptor Lwt.t diff --git a/src/lib_store/mocked/store.ml b/src/lib_store/mocked/store.ml index 17726f82ba43..8f1ede5f34b8 100644 --- a/src/lib_store/mocked/store.ml +++ b/src/lib_store/mocked/store.ml @@ -89,7 +89,7 @@ and chain_state = { checkpoint_data : block_descriptor Stored_data.t; (* Following fields are safe to update directly *) protocol_levels_data : - Protocol_levels.activation_block Protocol_levels.t Stored_data.t; + Protocol_levels.protocol_info Protocol_levels.t Stored_data.t; invalid_blocks_data : invalid_block Block_hash.Map.t Stored_data.t; forked_chains_data : Block_hash.t Chain_id.Map.t Stored_data.t; (* In memory-only: *) @@ -152,6 +152,33 @@ let locked_is_acceptable_block chain_state (hash, level) = Lwt.return @@ Block_hash.equal hash target_hash else Lwt.return_true +let find_protocol_info chain_store ~protocol_level = + let open Lwt_syntax in + Shared.use chain_store.chain_state (fun {protocol_levels_data; _} -> + let* protocol_levels = Stored_data.get protocol_levels_data in + return (Protocol_levels.find protocol_level protocol_levels)) + +let expect_predecessor_context_hash_exn chain_store protocol_level = + let open Lwt_syntax in + let* protocol_info = find_protocol_info chain_store ~protocol_level in + match protocol_info with + | Some {expect_predecessor_context; _} -> return expect_predecessor_context + | None -> + Format.ksprintf + Stdlib.failwith + "cannot find protocol info for level: %d" + protocol_level + +let expect_predecessor_context_hash chain_store ~protocol_level = + let open Lwt_result_syntax in + Lwt.catch + (fun () -> + let*! b = + expect_predecessor_context_hash_exn chain_store protocol_level + in + return b) + (fun _ -> tzfail (Protocol_not_found {protocol_level})) + module Block = struct type nonrec block = block @@ -375,6 +402,7 @@ module Block = struct }; block_metadata; ops_metadata; + shell_header_hash = _; } = validation_result in @@ -463,12 +491,6 @@ module Block = struct known_invalid Store_errors.(Cannot_store_block (hash, Invalid_block)) in - let* () = - fail_unless - (Context_hash.equal block_header.shell.context context_hash) - (Validation_errors.Inconsistent_hash - (context_hash, block_header.shell.context)) - in let contents = { Block_repr.header = block_header; @@ -496,7 +518,9 @@ module Block = struct } in let block = {Block_repr.hash; contents; metadata} in - let* () = Block_store.store_block chain_store.block_store block in + let* () = + Block_store.store_block chain_store.block_store block context_hash + in let*! () = Store_events.(emit store_block) (hash, block_header.shell.level) in @@ -546,6 +570,27 @@ module Block = struct in return_unit + let resulting_context_hash chain_store block = + let open Lwt_result_syntax in + let* expect_predecessor_context = + expect_predecessor_context_hash + chain_store + ~protocol_level:(Block_repr.proto_level block) + in + let hash = Block_repr.hash block in + let* resulting_context_hash_opt = + Block_store.resulting_context_hash + ~expect_predecessor_context + chain_store.block_store + (Block (hash, 0)) + in + match resulting_context_hash_opt with + | None -> + tzfail + (Resulting_context_hash_not_found + {hash; level = Block_repr.level block}) + | Some resulting_context_hash -> return resulting_context_hash + let context_exn chain_store block = let context_index = chain_store.global_store.context_index in Context_ops.checkout_exn context_index (Block_repr.context block) @@ -1065,7 +1110,7 @@ module Chain = struct in match Protocol_levels.find proto_level protocol_levels with | None -> Lwt.return_none - | Some {block; _} -> ( + | Some {activation_block = {block; _}; _} -> ( let block_activation_level = snd block in (* proto level's lower bound found, now retrieving the upper bound *) let head_proto_level = @@ -1082,7 +1127,7 @@ module Chain = struct Protocol_levels.find (succ proto_level) protocol_levels with | None -> Lwt.return_none - | Some {block; _} -> ( + | Some {activation_block = {block; _}; _} -> ( let next_activation_level = snd block in let last_level_in_protocol = Int32.(pred next_activation_level) @@ -1581,6 +1626,12 @@ module Chain = struct ~some:(fun metadata -> Block.last_allowed_fork_level metadata) (Block_repr.metadata genesis_block) in + let expect_predecessor_context = + (* This depends on the genesis protocol. It should be false but + we cannot retrieved registered protocols due to unix + dependencies. *) + false + in let* protocol_levels_data = Stored_data.init (Naming.protocol_levels_file chain_dir) @@ -1589,9 +1640,10 @@ module Chain = struct add genesis_proto_level { - block = genesis_descr; protocol = genesis_protocol; - commit_info = genesis_commit_info; + activation_block = + {block = genesis_descr; commit_info = genesis_commit_info}; + expect_predecessor_context; } empty) in @@ -1751,6 +1803,9 @@ module Chain = struct (* Protocols *) + let expect_predecessor_context_hash chain_store ~protocol_level = + expect_predecessor_context_hash chain_store ~protocol_level + let compute_commit_info chain_store block = let open Lwt_result_syntax in let index = chain_store.global_store.context_index in @@ -1760,7 +1815,8 @@ module Chain = struct let* commit_info = get_commit_info index block in return_some commit_info) - let set_protocol_level chain_store ~protocol_level (block, protocol_hash) = + let set_protocol_level chain_store ~protocol_level + (block, protocol_hash, expect_predecessor_context) = let open Lwt_result_syntax in Shared.locked_use chain_store.chain_state (fun {protocol_levels_data; _} -> let* commit_info_opt = @@ -1773,9 +1829,13 @@ module Chain = struct add protocol_level { - block = Block.descriptor block; protocol = protocol_hash; - commit_info = commit_info_opt; + activation_block = + { + block = Block.descriptor block; + commit_info = commit_info_opt; + }; + expect_predecessor_context; } protocol_levels)) in @@ -1790,21 +1850,25 @@ module Chain = struct in return_unit) + let find_protocol_info chain_store ~protocol_level = + find_protocol_info chain_store ~protocol_level + let find_activation_block chain_store ~protocol_level = let open Lwt_syntax in - Shared.use chain_store.chain_state (fun {protocol_levels_data; _} -> - let* protocol_levels = Stored_data.get protocol_levels_data in - Lwt.return (Protocol_levels.find protocol_level protocol_levels)) + let* protocol_info = find_protocol_info chain_store ~protocol_level in + match protocol_info with + | Some {activation_block; _} -> return_some activation_block + | None -> return_none let find_protocol chain_store ~protocol_level = let open Lwt_syntax in - let* o = find_activation_block chain_store ~protocol_level in - match o with - | None -> Lwt.return_none - | Some {Protocol_levels.protocol; _} -> Lwt.return_some protocol + let* protocol_info = find_protocol_info chain_store ~protocol_level in + match protocol_info with + | Some {protocol; _} -> return_some protocol + | None -> return_none let may_update_protocol_level chain_store ?pred ?protocol_level - (block, protocol_hash) = + ~expect_predecessor_context (block, protocol_hash) = let open Lwt_result_syntax in let* pred = match pred with @@ -1820,21 +1884,60 @@ module Chain = struct match o with | Some {block = bh, _; _} -> if Block_hash.(bh <> Block.hash block) then - set_protocol_level chain_store ~protocol_level (block, protocol_hash) + set_protocol_level + chain_store + ~protocol_level + (block, protocol_hash, expect_predecessor_context) else return_unit | None -> - set_protocol_level chain_store ~protocol_level (block, protocol_hash) + set_protocol_level + chain_store + ~protocol_level + (block, protocol_hash, expect_predecessor_context) else return_unit let may_update_ancestor_protocol_level chain_store ~head = let open Lwt_result_syntax in let head_proto_level = Block.proto_level head in - let*! o = - find_activation_block chain_store ~protocol_level:head_proto_level - in + let*! o = find_protocol_info chain_store ~protocol_level:head_proto_level in match o with - | None -> return_unit - | Some {block; protocol; _} -> ( + | None -> + (* If no activation block is registered for a given protocol + level, we search for the activation block. This activation + block is expected to be found above the savepoint. If not, + the savepoint will be considered as the activation + block. *) + let*! _, savepoint_level = savepoint chain_store in + let rec find_activation_block lower_bound block = + let*! pred = Block.read_predecessor_opt chain_store block in + match pred with + | None -> return block + | Some pred -> + let pred_proto_level = Block.proto_level pred in + if Compare.Int.(pred_proto_level <= Int.pred head_proto_level) + then return block + else if Compare.Int32.(Block.level pred <= lower_bound) then + return pred + else find_activation_block lower_bound pred + in + let* activation_block = find_activation_block savepoint_level head in + let protocol_level = Block.proto_level head in + (* The head must have an associated context stored. *) + let* context = Block.context chain_store head in + let*! activated_protocol = Context_ops.get_protocol context in + (* This depends on the protocol but registered protocols + cannot be retrieved due to unix dependencies. *) + let expected_predecessor_context = true in + set_protocol_level + chain_store + ~protocol_level + (activation_block, activated_protocol, expected_predecessor_context) + | Some + { + Protocol_levels.protocol; + activation_block = {block; _}; + expect_predecessor_context; + } -> ( let*! _, savepoint_level = savepoint chain_store in if Compare.Int32.(savepoint_level > snd block) then (* the block is too far in the past *) @@ -1858,7 +1961,10 @@ module Chain = struct match o with | None -> return_unit | Some ancestor -> - may_update_protocol_level chain_store (ancestor, protocol))) + may_update_protocol_level + chain_store + ~expect_predecessor_context + (ancestor, protocol))) let all_protocol_levels chain_store = Shared.use chain_store.chain_state (fun {protocol_levels_data; _} -> @@ -1878,7 +1984,7 @@ module Chain = struct let* protocol = if Compare.Int32.(Block.level pred < save_point_level) then let* o = - find_activation_block + find_protocol_info chain_store ~protocol_level:(Block.proto_level pred) in @@ -2161,8 +2267,9 @@ let rec make_pp_chain_store (chain_store : chain_store) = forked_chains, active_testchain )) in - let pp_protocol_level fmt - (proto_level, {Protocol_levels.block; protocol; commit_info}) = + let pp_proto_info fmt + (proto_level, {Protocol_levels.protocol; activation_block; _}) = + let Protocol_levels.{block; commit_info} = activation_block in Format.fprintf fmt "proto level: %d, transition block: %a, protocol: %a, commit info: %a" @@ -2221,7 +2328,7 @@ let rec make_pp_chain_store (chain_store : chain_store) = caboose (option_pp ~default:"n/a" pp_block_descriptor) target - (Format.pp_print_list ~pp_sep:Format.pp_print_cut pp_protocol_level) + (Format.pp_print_list ~pp_sep:Format.pp_print_cut pp_proto_info) (Protocol_levels.bindings protocol_levels_data) (Format.pp_print_list ~pp_sep:Format.pp_print_cut Block_hash.pp) (Block_hash.Map.bindings invalid_blocks_data |> List.map fst) @@ -2261,3 +2368,5 @@ module Unsafe = struct let block_of_repr = Fun.id end + +let v_3_0_upgrade ~store_dir:_ _genesis = Lwt_result_syntax.return_unit diff --git a/src/lib_store/shared/naming.ml b/src/lib_store/shared/naming.ml index 50822947d8a5..679741463da6 100644 --- a/src/lib_store/shared/naming.ml +++ b/src/lib_store/shared/naming.ml @@ -77,6 +77,12 @@ let testchains_dir dir = mk_dir dir "testchains" let protocol_levels_file dir = make_encoded_file dir ~filename:"protocol_levels" Protocol_levels.encoding +let legacy_protocol_levels_file dir = + make_encoded_file + dir + ~filename:"protocol_levels" + Protocol_levels.Legacy.encoding + let genesis_block_file dir = make_encoded_file dir ~filename:"genesis" Block_repr.encoding diff --git a/src/lib_store/shared/naming.mli b/src/lib_store/shared/naming.mli index 6c974e22ec31..100971a490b5 100644 --- a/src/lib_store/shared/naming.mli +++ b/src/lib_store/shared/naming.mli @@ -86,7 +86,13 @@ val chain_config_file : val protocol_levels_file : [< `Chain_dir] directory -> ( [`Protocol_levels], - Protocol_levels.activation_block Protocol_levels.t ) + Protocol_levels.protocol_info Protocol_levels.t ) + encoded_file + +val legacy_protocol_levels_file : + [< `Chain_dir] directory -> + ( [`Protocol_levels], + Protocol_levels.Legacy.activation_block Protocol_levels.Legacy.t ) encoded_file val genesis_block_file : @@ -212,7 +218,7 @@ val snapshot_version_file : val snapshot_protocol_levels_file : [< `Snapshot_dir | `Snapshot_tmp_dir | `Tar_archive] directory -> ( [`Snapshot_protocol_levels], - Protocol_levels.activation_block Protocol_levels.t ) + Protocol_levels.protocol_info Protocol_levels.t ) encoded_file val snapshot_tar_root : [`Tar_archive] directory diff --git a/src/lib_store/shared/store_events.ml b/src/lib_store/shared/store_events.ml index 553cff1d4a24..f69dba37bfda 100644 --- a/src/lib_store/shared/store_events.ml +++ b/src/lib_store/shared/store_events.ml @@ -465,3 +465,20 @@ let notify_merge_error = {errs}" ~pp1:(fun ppf -> Format.fprintf ppf "%a" Error_monad.pp_print_trace) ("errs", Error_monad.trace_encoding) + +let upgrade_store_failed = + declare_1 + ~section + ~level:Internal_event.Error + ~name:"upgrade_store_failed" + ~msg:"store upgrade failed, cleaning up temporary files: {errs}" + ~pp1:(fun ppf -> Format.fprintf ppf "%a" Error_monad.pp_print_trace) + ("errs", Error_monad.trace_encoding) + +let upgrade_store_started = + declare_0 + ~section + ~level:Internal_event.Notice + ~name:"upgrade_store_started" + ~msg:"upgrading the store" + () diff --git a/src/lib_store/shared/store_types.ml b/src/lib_store/shared/store_types.ml index 941a219fa745..4aa865d01f22 100644 --- a/src/lib_store/shared/store_types.ml +++ b/src/lib_store/shared/store_types.ml @@ -148,24 +148,71 @@ module Protocol_levels = struct type activation_block = { block : block_descriptor; - protocol : Protocol_hash.t; commit_info : commit_info option; } let activation_block_encoding = let open Data_encoding in conv - (fun {block; protocol; commit_info} -> (block, protocol, commit_info)) - (fun (block, protocol, commit_info) -> {block; protocol; commit_info}) - (obj3 + (fun {block; commit_info} -> (block, commit_info)) + (fun (block, commit_info) -> {block; commit_info}) + (obj2 (req "block" block_descriptor_encoding) - (req "protocol" Protocol_hash.encoding) (opt "commit_info" commit_info_encoding)) + type protocol_info = { + protocol : Protocol_hash.t; + activation_block : activation_block; + expect_predecessor_context : bool; + } + + let protocol_info_encoding = + let open Data_encoding in + conv + (fun {protocol; activation_block; expect_predecessor_context} -> + (protocol, activation_block, expect_predecessor_context)) + (fun (protocol, activation_block, expect_predecessor_context) -> + {protocol; activation_block; expect_predecessor_context}) + (obj3 + (req "protocol" Protocol_hash.encoding) + (req "activation_block" activation_block_encoding) + (req "expect_predecessor_context" bool)) + let encoding = Data_encoding.conv (fun map -> bindings map) (fun bindings -> List.fold_left (fun map (k, v) -> add k v map) empty bindings) - Data_encoding.(list (tup2 uint8 activation_block_encoding)) + Data_encoding.(list (tup2 uint8 protocol_info_encoding)) + + module Legacy = struct + type activation_block = { + block : block_descriptor; + protocol : Protocol_hash.t; + commit_info : commit_info option; + } + + include Map.Make (struct + type t = int + + let compare = Compare.Int.compare + end) + + let legacy_activation_block_encoding = + let open Data_encoding in + conv + (fun {block; protocol; commit_info} -> (block, protocol, commit_info)) + (fun (block, protocol, commit_info) -> {block; protocol; commit_info}) + (obj3 + (req "block" block_descriptor_encoding) + (req "protocol" Protocol_hash.encoding) + (opt "commit_info" commit_info_encoding)) + + let encoding = + Data_encoding.conv + (fun map -> bindings map) + (fun bindings -> + List.fold_left (fun map (k, v) -> add k v map) empty bindings) + Data_encoding.(list (tup2 uint8 legacy_activation_block_encoding)) + end end diff --git a/src/lib_store/shared/store_types.mli b/src/lib_store/shared/store_types.mli index c6743b7fa3be..f3725b7e8baa 100644 --- a/src/lib_store/shared/store_types.mli +++ b/src/lib_store/shared/store_types.mli @@ -131,10 +131,38 @@ module Protocol_levels : sig allowing us to remove the option. *) type activation_block = { block : block_descriptor; - protocol : Protocol_hash.t; commit_info : commit_info option; } + (** The type for protocol info. + + [expect_predecessor_context] is a flag which reflects what is + referenced by the context hash of a block, depending on its + protocol. If the flag is true, the block contains the + predecessors context (context before the application of the + block). Otherwise, it contains the resulting context hash of the + application of the block. *) + type protocol_info = { + protocol : Protocol_hash.t; + activation_block : activation_block; + expect_predecessor_context : bool; + } + + val activation_block_encoding : activation_block Data_encoding.t + (** Encoding for the protocol level's association map. *) - val encoding : activation_block t Data_encoding.t + val encoding : protocol_info t Data_encoding.t + + module Legacy : sig + type activation_block = { + block : block_descriptor; + protocol : Protocol_hash.t; + commit_info : commit_info option; + } + + include Map.S with type key = int + + (** Encoding for the protocol level's association map. *) + val encoding : activation_block t Data_encoding.t + end end diff --git a/src/lib_store/store.mli b/src/lib_store/store.mli index c3e3c7c5d3c7..b681ec755464 100644 --- a/src/lib_store/store.mli +++ b/src/lib_store/store.mli @@ -450,8 +450,17 @@ module Block : sig operations:Operation.t trace trace -> unit tzresult Lwt.t - (** [context_exn chain_store block] checkouts the context of the - [block]. *) + (** [resulting_context_hash chain_store block] returns the resulting + context hash of the [block]. This context depends on the + [block]'s protocol associated semantics, i.e., it can either be + the one contained in its block header or the stored result of + its application. *) + val resulting_context_hash : + chain_store -> block -> Context_hash.t tzresult Lwt.t + + (** [context_exn chain_store block] checkouts the {b resulting} + context of the [block] which may differ from its block header's + one depending on the block's associated protocol semantics. *) val context_exn : chain_store -> block -> Tezos_protocol_environment.Context.t Lwt.t @@ -861,6 +870,11 @@ module Chain : sig (** {2 Chain's protocols} *) + (** [find_protocol_info chain_store ~protocol_level] returns the + protocol info associated to the given [protocol_level]. *) + val find_protocol_info : + t -> protocol_level:int -> Protocol_levels.protocol_info option Lwt.t + (** [find_activation_block chain_store ~protocol_level] returns the block that activated the protocol of level [protocol_level]. *) val find_activation_block : @@ -873,31 +887,43 @@ module Chain : sig val find_protocol : chain_store -> protocol_level:int -> Protocol_hash.t option Lwt.t + (** [expects_predecessor_context_hash chain_store proto_level] + returns whether or not a protocol requires the context hash of a + block to target resulting context of it's predecessor. This + depends on the environment of each protocol.*) + val expect_predecessor_context_hash : + chain_store -> protocol_level:int -> bool tzresult Lwt.t + (** [all_protocol_levels chain_store] returns all the protocols registered in [chain_store]. *) val all_protocol_levels : - chain_store -> Protocol_levels.activation_block Protocol_levels.t Lwt.t + chain_store -> Protocol_levels.protocol_info Protocol_levels.t Lwt.t (** [may_update_protocol_level chain_store ?pred ?protocol_level - (block, ph)] updates the protocol level for the protocol [ph] in - [chain_store] with the activation [block]. If [pred] is not - provided, it reads the [block]'s predecessor and check that the - [block]'s protocol level is increasing compared to its - predecessor. If [protocol_level] is provided, we use this value - instead of the protocol level found in [block]. If a previous - entry is found, it overwrites it. *) + ~expect_predecessor_context (block, ph)] updates the protocol + level for the protocol [ph] in [chain_store] with the activation + [block]. If [pred] is not provided, it reads the [block]'s + predecessor and check that the [block]'s protocol level is + increasing compared to its predecessor. If [protocol_level] is + provided, we use this value instead of the protocol level found + in [block]. If a previous entry is found, it overwrites it. The + [expect_predecessor_context] argument specifies which context + hash semantics should be used. *) val may_update_protocol_level : chain_store -> ?pred:Block.block -> ?protocol_level:int -> + expect_predecessor_context:bool -> Block.block * Protocol_hash.t -> unit tzresult Lwt.t - (** [may_update_ancestor_protocol_level chain_store ~head] tries to - find the activation block of the [head]'s protocol, checks that - its an ancestor and tries to update it if that's not the case. If - the registered activation block is not reachable (already - pruned), this function does nothing. *) + (** [may_update_ancestor_protocol_level chain_store ~head + ~expect_predecessor_context] tries to find the activation block + of the [head]'s protocol, checks that its an ancestor and tries + to update it if that's not the case. If the registered + activation block is not reachable (already pruned), this + function does nothing. The [expect_predecessor_context] argument + specifies which context hash semantics should be used. *) val may_update_ancestor_protocol_level : chain_store -> head:Block.block -> unit tzresult Lwt.t @@ -1000,6 +1026,8 @@ module Chain_traversal : sig (Block.t * Block.t list) Lwt.t end +val v_3_0_upgrade : store_dir:string -> Genesis.t -> unit tzresult Lwt.t + (**/**) (** Unsafe set of functions intended for internal store manipulation diff --git a/src/lib_store/unix/block_repr_unix.ml b/src/lib_store/unix/block_repr_unix.ml index bf9fc0c565b3..01386463709b 100644 --- a/src/lib_store/unix/block_repr_unix.ml +++ b/src/lib_store/unix/block_repr_unix.ml @@ -195,3 +195,15 @@ let raw_get_last_allowed_fork_level block_bytes total_block_length = else metadata_offset + 1 in Some (Bytes.get_int32_be block_bytes lafl_offset) + +let fitness_length_offset = + predecessor_offset + Block_hash.size + 8 (* timestamp *) + 1 + (* validation pass *) + 32 (* operation list list hash *) + +let raw_get_context block_bytes = + let fitness_length = Bytes.get_int32_be block_bytes fitness_length_offset in + let context_offset = + fitness_length_offset + 4 + Int32.to_int fitness_length + in + Context_hash.of_bytes_exn + (Bytes.sub block_bytes context_offset Context_hash.size) diff --git a/src/lib_store/unix/block_repr_unix.mli b/src/lib_store/unix/block_repr_unix.mli index 58826411ec3d..b1d44ffbde1d 100644 --- a/src/lib_store/unix/block_repr_unix.mli +++ b/src/lib_store/unix/block_repr_unix.mli @@ -78,3 +78,8 @@ val raw_get_block_predecessor : bytes -> Block_hash.t [block_buffer] if there is any. Returns [None] if no metadata is present. *) val raw_get_last_allowed_fork_level : bytes -> int -> int32 option + +(** [raw_get_context block_buffer] introspects the + context of the block contained in + [block_buffer]. *) +val raw_get_context : bytes -> Context_hash.t diff --git a/src/lib_store/unix/block_store.ml b/src/lib_store/unix/block_store.ml index bb6f021353c5..e6be3d6c5519 100644 --- a/src/lib_store/unix/block_store.ml +++ b/src/lib_store/unix/block_store.ml @@ -240,7 +240,7 @@ let mem block_store key = block_store.cemented_store predecessor_hash)) -let read_block ~read_metadata block_store key_kind = +let read_block block_store ~read_metadata key_kind = let open Lwt_result_syntax in Lwt_idle_waiter.task block_store.merge_scheduler (fun () -> (* Resolve the hash *) @@ -317,7 +317,67 @@ let read_block_metadata block_store key_kind = block_store.cemented_store level))) -let store_block block_store block = +let resulting_context_hash block_store ~fetch_expect_predecessor_context key = + (* Hypothesis: there is an intersection of at least 1 block with the + end of the cementing store and the beginning of the floating + store. + Indeed, there are [max_op_ttl] blocks below the checkpoint + kept in the floating store, so that this window of blocks + overlaps with the content of the cemented store. Thus, looking + at the successor of the last cemented block should never occur, + as this case would be tackled by the floating store's looking. *) + let open Lwt_result_syntax in + let ( let*? ) t k = + let* v_opt = t in + match v_opt with None -> return_none | Some v -> k v + in + Lwt_idle_waiter.task block_store.merge_scheduler (fun () -> + (* Resolve the hash *) + let*? adjusted_hash = get_hash block_store key in + if Block_hash.equal block_store.genesis_block.hash adjusted_hash then + return_some (Block_repr.context block_store.genesis_block) + else + (* First look in the floating stores *) + let*! resulting_context_opt = + List.find_map_s + (fun store -> + Floating_block_store.find_resulting_context_hash + store + adjusted_hash) + (block_store.rw_floating_block_store + :: block_store.ro_floating_block_stores) + in + match resulting_context_opt with + | Some resulting_context_hash -> return_some resulting_context_hash + | None -> + (* If not found, look at the context of the direct + successor of the looked up block in the cemented store. *) + let cemented_store = block_store.cemented_store in + let* expect_predecessor = fetch_expect_predecessor_context () in + if expect_predecessor then + let*? block_level = + return + (Cemented_block_store.get_cemented_block_level + cemented_store + adjusted_hash) + in + let*? succ_block = + Cemented_block_store.get_cemented_block_by_level + cemented_store + ~read_metadata:false + (Int32.succ block_level) + in + return_some (Block_repr.context succ_block) + else + let*? block = + Cemented_block_store.get_cemented_block_by_hash + cemented_store + ~read_metadata:false + adjusted_hash + in + return_some (Block_repr.context block)) + +let store_block block_store block resulting_context_hash = let open Lwt_result_syntax in let* () = fail_when block_store.readonly Cannot_write_in_readonly in Lwt_idle_waiter.task block_store.merge_scheduler (fun () -> @@ -330,7 +390,7 @@ let store_block block_store block = Floating_block_store.append_block ~log_metrics:true block_store.rw_floating_block_store - predecessors + {predecessors; resulting_context_hash} block)) let cement_blocks ?(check_consistency = true) ~write_metadata block_store @@ -902,15 +962,19 @@ let update_floating_stores block_store ~history_mode ~ro_store ~rw_store let block_hash = Block_repr_unix.raw_get_block_hash block_bytes in if Block_hash.Set.mem block_predecessor !visited then ( visited := Block_hash.Set.add block_hash !visited ; - let*! predecessors = + let*! {predecessors; resulting_context_hash} = let*! pred_opt = - Floating_block_store.find_predecessors store block_hash + Floating_block_store.find_info store block_hash in Lwt.return (WithExceptions.Option.get ~loc:__LOC__ pred_opt) in Floating_block_store.raw_append new_store - (block_hash, block_bytes, total_block_length, predecessors)) + ( block_hash, + block_bytes, + total_block_length, + predecessors, + resulting_context_hash )) else return_unit) store) [ro_store; rw_store] @@ -1534,7 +1598,12 @@ let create ?block_cache_limit chain_dir ~genesis_block = let* block_store = load chain_dir ?block_cache_limit ~genesis_block ~readonly:false in - let* () = store_block block_store genesis_block in + let* () = + store_block + block_store + genesis_block + genesis_block.contents.header.shell.context + in return block_store let register_gc_callback block_store gc_callback = @@ -1574,3 +1643,104 @@ let close block_store = List.iter_s Floating_block_store.close (block_store.rw_floating_block_store :: block_store.ro_floating_block_stores) + +(***************** Upgrade to V3 *****************) + +let v_3_0_upgrade chain_dir ~cleanups ~finalizers = + let open Lwt_result_syntax in + let get_floating_paths kind = + let legacy_floating_blocks_dir = + Naming.floating_blocks_dir chain_dir kind + in + let legacy_floating_index_dir = + Naming.dir_path + (Naming.floating_blocks_index_dir legacy_floating_blocks_dir) + in + let legacy_floating_blocks_file = + Naming.floating_blocks_file legacy_floating_blocks_dir + in + let new_floating_index_dir = + Naming.dir_path + (Naming.floating_blocks_index_dir legacy_floating_blocks_dir) + ^ ".new" + in + ( Naming.dir_path legacy_floating_blocks_dir, + legacy_floating_index_dir, + legacy_floating_blocks_file, + new_floating_index_dir ) + in + let all_kinds = Naming.[RO; RW; RW_TMP; RO_TMP] in + let upgrade_floating_index kind = + let ( legacy_floating_blocks_dir, + legacy_floating_index_dir, + legacy_floating_blocks_file, + new_floating_index_dir ) = + get_floating_paths kind + in + let*! should_upgrade = Lwt_unix.file_exists legacy_floating_blocks_dir in + if not should_upgrade then return_unit + else + let clean_failed_upgrade () = + let*! exists = Lwt_unix.file_exists new_floating_index_dir in + if exists then Lwt_utils_unix.remove_dir new_floating_index_dir + else Lwt.return_unit + in + let finalize () = + let*! exists = Lwt_unix.file_exists new_floating_index_dir in + if exists then + let*! () = Lwt_utils_unix.remove_dir legacy_floating_index_dir in + Lwt_unix.rename new_floating_index_dir legacy_floating_index_dir + else Lwt.return_unit + in + finalizers := finalize :: !finalizers ; + cleanups := clean_failed_upgrade :: !cleanups ; + let legacy_index = + Floating_block_index.Legacy.v + ~log_size:Floating_block_store.default_floating_blocks_log_size + ~readonly:true + legacy_floating_index_dir + in + let new_index = + Floating_block_index.v + ~log_size:Floating_block_store.default_floating_blocks_log_size + ~readonly:false + new_floating_index_dir + in + let*! fd = + Lwt_unix.openfile + (Naming.file_path legacy_floating_blocks_file) + [Unix.O_CLOEXEC; Unix.O_RDONLY] + 0o444 + in + Lwt.finalize + (fun () -> + (* Iterate over the existing stores and retrieve their context hash. *) + let* () = + Floating_block_store.raw_iterate_fd + (fun (block_b, _len) -> + let block_hash = Block_repr_unix.raw_get_block_hash block_b in + let block_context = Block_repr_unix.raw_get_context block_b in + let { + Floating_block_index.Legacy.Legacy_block_info.offset; + predecessors; + } = + Floating_block_index.Legacy.find legacy_index block_hash + in + let resulting_context_hash = block_context in + let new_value = + Floating_block_index.Block_info. + {offset; predecessors; resulting_context_hash} + in + Floating_block_index.replace new_index block_hash new_value ; + return_unit) + fd + in + return_unit) + (fun () -> + Floating_block_index.flush new_index ; + Floating_block_index.close new_index ; + Floating_block_index.Legacy.close legacy_index ; + let*! () = Lwt_unix.close fd in + Lwt.return_unit) + in + protect (fun () -> List.iter_es upgrade_floating_index all_kinds) diff --git a/src/lib_store/unix/block_store.mli b/src/lib_store/unix/block_store.mli index c6d21cc09bed..9b2c0f670485 100644 --- a/src/lib_store/unix/block_store.mli +++ b/src/lib_store/unix/block_store.mli @@ -208,12 +208,21 @@ val mem : block_store -> key -> bool tzresult Lwt.t unknown. *) val get_hash : block_store -> key -> Block_hash.t option tzresult Lwt.t -(** [read_block ~read_metadata block_store key] reads the block [key] - in [block_store] if present. Return [None] if the block is - unknown. If [read_metadata] is set to [true] it tries to retreive - the metadata but do not fail if it is not available. *) +(** [resulting_context_hash block_store ~fetch_expect_predecessor_context key] + retrieves the resulting context hash from the block application, + corresponding to the given [key]. *) +val resulting_context_hash : + block_store -> + fetch_expect_predecessor_context:(unit -> bool tzresult Lwt.t) -> + key -> + Context_hash.t option tzresult Lwt.t + +(** [read_block block_store ~read_metadata key] reads the block [key] + in [block_store] if present. Return [None] if the block is + unknown. If [read_metadata] is set to [true] it tries to retreive + the metadata but do not fail if it is not available. *) val read_block : - read_metadata:bool -> block_store -> key -> Block_repr.t option tzresult Lwt.t + block_store -> read_metadata:bool -> key -> Block_repr.t option tzresult Lwt.t (** [read_block_metadata block_store key] reads the metadata for the block [key] in [block_store] if present. Return [None] if the block @@ -221,9 +230,11 @@ val read_block : val read_block_metadata : block_store -> key -> Block_repr.metadata option tzresult Lwt.t -(** [store_block block_store block] stores the [block] in the current - [RW] floating store. *) -val store_block : block_store -> Block_repr.t -> unit tzresult Lwt.t +(** [store_block block_store block resulting_context_hash] stores the + [block] in the current [RW] floating store with its associated + [resulting_context_hash]. *) +val store_block : + block_store -> Block_repr.t -> Context_hash.t -> unit tzresult Lwt.t (** [cement_blocks ?check_consistency ~write_metadata block_store chunk_iterator] @@ -351,3 +362,14 @@ val close : block_store -> unit Lwt.t (** [may_recover_merge block_store] recovers, if needed, from a [block_store] where the merge procedure was interrupted. *) val may_recover_merge : block_store -> unit tzresult Lwt.t + +(** Upgrade a v_2 to v_3 block store by retrieving + [resulting_context_hash] of all blocks present in the floating + stores and updating their index. + + {b Warning} Not backward-compatible. *) +val v_3_0_upgrade : + [`Chain_dir] Naming.directory -> + cleanups:(unit -> unit Lwt.t) list ref -> + finalizers:(unit -> unit Lwt.t) list ref -> + unit tzresult Lwt.t diff --git a/src/lib_store/unix/consistency.ml b/src/lib_store/unix/consistency.ml index 64f697f8c919..d82083716e14 100644 --- a/src/lib_store/unix/consistency.ml +++ b/src/lib_store/unix/consistency.ml @@ -742,7 +742,7 @@ let check_block_protocol_hash context_index ~expected block = (** Look into the cemented store for the lowest block with an associated proto level that is above the savepoint. *) -let find_activation_block_in_cemented block_store ~savepoint_level ~proto_level +let find_activation_blocks_in_cemented block_store ~savepoint_level ~proto_level = let open Lwt_result_syntax in let cemented_store = Block_store.cemented_block_store block_store in @@ -841,23 +841,27 @@ let find_activation_block_in_cemented block_store ~savepoint_level ~proto_level in failwith "find_activation_block_in_cemented: cannot read cemented cycle") (function - | Found block -> return_some block + | Found block -> + let* next_block = + read_cemented_block_by_level (Int32.succ (Block_repr.level block)) + in + return_some (block, next_block) | exn -> tzfail (Inconsistent_cemented_file (Naming.file_path cycle.file, Printexc.to_string exn))) -let find_activation_block_in_floating block_store ~head ~savepoint_level +let find_activation_blocks_in_floating block_store ~head ~savepoint_level ~proto_level = let open Lwt_result_syntax in - let rec loop block_proto_level block = + let rec loop block_proto_level succ block = if Compare.Int32.(Block_repr.level block <= savepoint_level) then let* () = fail_unless (Block_repr.proto_level block = proto_level) (Corrupted_store (Cannot_find_activation_block proto_level)) in - return block + return (block, succ) else let* predecessor_opt = Block_store.read_block @@ -876,11 +880,11 @@ let find_activation_block_in_floating block_store ~head ~savepoint_level predecessor_proto_level < block_proto_level && block_proto_level = proto_level) then (* Found *) - return block + return (block, succ) else (* Continue *) - loop predecessor_proto_level predecessor + loop predecessor_proto_level block predecessor in - loop (Block_repr.proto_level head) head + loop (Block_repr.proto_level head) head head let craft_activation_block context_index block = let open Lwt_result_syntax in @@ -894,25 +898,18 @@ let craft_activation_block context_index block = return_some (Protocol_levels.commit_info_of_tuple tup)) (fun _ -> return_none) in - let*! protocol = - let*! ctxt = - Context.checkout_exn context_index (Block_repr.context block) - in - Context.get_protocol ctxt - in - return - {Protocol_levels.block = Block_repr.descriptor block; protocol; commit_info} + return {Protocol_levels.block = Block_repr.descriptor block; commit_info} -let find_lowest_block_with_proto_level block_store ~head ~savepoint_level +let find_two_lowest_block_with_proto_level block_store ~head ~savepoint_level proto_level = let open Lwt_result_syntax in - let* activation_block = - find_activation_block_in_cemented block_store ~savepoint_level ~proto_level + let* activation_blocks = + find_activation_blocks_in_cemented block_store ~savepoint_level ~proto_level in - match activation_block with + match activation_blocks with | Some b -> return b | None -> - find_activation_block_in_floating + find_activation_blocks_in_floating block_store ~head ~savepoint_level @@ -929,11 +926,12 @@ let find_lowest_block_with_proto_level block_store ~head ~savepoint_level - block store is valid and available, - head is valid and available. - savepoint is valid and available. *) -let fix_protocol_levels chain_dir block_store context_index +let fix_protocol_levels chain_dir block_store context_index genesis ~savepoint:(savepoint_hash, _) ~head = let open Lwt_result_syntax in (* Attempt to recover with the previous protocol table. *) - let*! (stored_protocol_levels : 'a Protocol_levels.t) = + let*! (stored_protocol_levels : + Protocol_levels.protocol_info Protocol_levels.t) = let*! r = Stored_data.load (Naming.protocol_levels_file chain_dir) in match r with | Error _ -> Lwt.return Protocol_levels.empty @@ -958,9 +956,9 @@ let fix_protocol_levels chain_dir block_store context_index (fun invalid_protocol_levels proto_level -> match Protocol_levels.find_opt proto_level stored_protocol_levels with | None -> return (proto_level :: invalid_protocol_levels) - | Some activation_block -> ( + | Some proto_info -> ( let activation_block_level = - snd activation_block.Protocol_levels.block + snd proto_info.Protocol_levels.activation_block.block in let level_to_read = if @@ -990,7 +988,7 @@ let fix_protocol_levels chain_dir block_store context_index let* protocol_matches = check_block_protocol_hash context_index - ~expected:activation_block.protocol + ~expected:proto_info.protocol b in if protocol_matches then return invalid_protocol_levels @@ -1020,24 +1018,65 @@ let fix_protocol_levels chain_dir block_store context_index let* fixed_protocol_levels = List.fold_left_es (fun fixed_protocol_levels invalid_proto_level -> - let* b = - find_lowest_block_with_proto_level + (* We need the activation block and its successor. The + successor is used when the protocol uses the predecessor's + context semantics to retrieve the context to checkout. *) + let* lowest, succ_lowest = + find_two_lowest_block_with_proto_level block_store ~head ~savepoint_level:(Block_repr.level savepoint) invalid_proto_level in + let is_genesis = Block_repr.hash lowest = genesis.Genesis.block in + (* We fail when the successor of the activation block has a + different protocol which should not happen in most cases + unless a protocol activates right after an existing + one. This is to make sure we are able to retrieve the + activation block's context through the protocol's + semantics. *) + let* () = + fail_unless + (is_genesis + || Compare.Int.( + Block_repr.proto_level lowest + = Block_repr.proto_level succ_lowest)) + (Corrupted_store (Cannot_find_activation_block invalid_proto_level)) + in let* activation_block = craft_activation_block context_index b in + let*! protocol = + let*! ctxt = + if is_genesis then + Context.checkout_exn context_index (Block_repr.context lowest) + else + (* The successor of the lowest has the same protocol + committed as the lowest and it will be correct + whether the context hash semantics is the current's + or the predecessor's resulting context. *) + Context.checkout_exn + context_index + (Block_repr.context succ_lowest) + in + Context.get_protocol ctxt + in + (* Protocol above savepoints should be registered *) + let* (module Proto) = Registered_protocol.get_result protocol in let*! () = Store_events.( - emit - restore_protocol_activation - (invalid_proto_level, activation_block.protocol)) + emit restore_protocol_activation (invalid_proto_level, protocol)) + in + let proto_info = + { + Protocol_levels.protocol; + activation_block = Block_repr.descriptor lowest; + expect_predecessor_context = + Proto.expected_context_hash = Predecessor_resulting_context; + } in return (Protocol_levels.add invalid_proto_level - activation_block + proto_info fixed_protocol_levels)) correct_protocol_levels invalid_proto_levels @@ -1259,7 +1298,13 @@ let fix_consistency ?history_mode chain_dir context_index genesis = savepoint in let* protocol_levels = - fix_protocol_levels chain_dir block_store context_index ~savepoint ~head + fix_protocol_levels + chain_dir + block_store + context_index + genesis + ~savepoint + ~head in let* () = fix_chain_state diff --git a/src/lib_store/unix/floating_block_index.ml b/src/lib_store/unix/floating_block_index.ml index 67735faa0e59..56b8e65eb637 100644 --- a/src/lib_store/unix/floating_block_index.ml +++ b/src/lib_store/unix/floating_block_index.ml @@ -27,7 +27,11 @@ time: if a block has fewer than 12 predecessors then it is padded so its size remain constant. *) module Block_info = struct - type t = {offset : int; predecessors : Block_hash.t list} + type t = { + offset : int; + predecessors : Block_hash.t list; + resulting_context_hash : Context_hash.t; + } let max_predecessors = 12 @@ -36,17 +40,25 @@ module Block_info = struct assert (r < 1 lsl 16) ; r - let encoded_size = 8 + 1 + encoded_list_size + let encoded_size = 8 + 1 + encoded_list_size + Context_hash.size - (* Format: - (8) + (1) + (list_size * Block_hash.size) *) + let context_hash_t = + let open Repr in + map + (bytes_of (`Fixed Context_hash.size)) + (fun res -> Context_hash.of_bytes_exn res) + (fun ch -> Context_hash.to_bytes ch) + (* Format: + (8) + (1) + (list_size * Block_hash.size) + (31)*) let t = let open Repr in map - (pair int (list ~len:`Int16 Block_key.t)) - (fun (offset, predecessors) -> {offset; predecessors}) - (fun {offset; predecessors} -> (offset, predecessors)) + (triple int (list ~len:`Int16 Block_key.t) context_hash_t) + (fun (offset, predecessors, resulting_context_hash) -> + {offset; predecessors; resulting_context_hash}) + (fun {offset; predecessors; resulting_context_hash} -> + (offset, predecessors, resulting_context_hash)) let encode v = let bytes = Bytes.create encoded_size in @@ -64,9 +76,16 @@ module Block_info = struct (8 + 1 + (i * Block_hash.size)) Block_hash.size) v.predecessors ; + Bytes.blit + (Context_hash.to_bytes v.resulting_context_hash) + 0 + bytes + (encoded_size - Context_hash.size) + Context_hash.size ; Bytes.unsafe_to_string bytes let decode str i = + let total_size = i + encoded_size in let current_offset = ref i in (* The Int64.to_int conversion is not likely to fail as it was written based on Int64.of_int and the encoded offset won't @@ -77,28 +96,106 @@ module Block_info = struct (* Setting current_offset right after the (8)*) current_offset := !current_offset + 8 ; let list_size = TzEndian.get_int8_string str !current_offset in + (* Setting current_offset right after the (1)*) current_offset := !current_offset + 1 ; let predecessors = ref [] in - let limit = !current_offset in - current_offset := limit + ((list_size - 1) * Block_hash.size) ; - while !current_offset >= limit do + let predecessors_limit = !current_offset in + current_offset := predecessors_limit + ((list_size - 1) * Block_hash.size) ; + while !current_offset >= predecessors_limit do predecessors := (String.sub str !current_offset Block_hash.size |> Block_hash.of_string_exn) :: !predecessors ; current_offset := !current_offset - Block_hash.size done ; - {offset; predecessors = !predecessors} + let resulting_context_hash = + let offset = total_size - Context_hash.size in + String.sub str offset Context_hash.size |> Context_hash.of_string_exn + in + {offset; predecessors = !predecessors; resulting_context_hash} let pp fmt v = let open Format in fprintf fmt - "@[offset: %d, predecessors : [ @[%a @]]@]" + "@[offset: %d,@ predecessors: [@[%a @]],@ resulting context hash: \ + %a@]" v.offset (pp_print_list ~pp_sep:(fun fmt () -> fprintf fmt " ;@,") Block_hash.pp) v.predecessors + Context_hash.pp + v.resulting_context_hash end (* Hashmap from block's hashes to location *) include Index_unix.Make (Block_key) (Block_info) (Index.Cache.Unbounded) + +module Legacy = struct + module Legacy_block_info = struct + type t = {offset : int; predecessors : Block_hash.t list} + + let t = + let open Repr in + map + (pair int (list ~len:`Int16 Block_key.t)) + (fun (offset, predecessors) -> {offset; predecessors}) + (fun {offset; predecessors} -> (offset, predecessors)) + + let max_predecessors = 12 + + let encoded_list_size = + let r = max_predecessors * Block_hash.size (* uint16 *) in + assert (r < 1 lsl 16) ; + r + + let encoded_size = 8 + 1 + encoded_list_size + + (* Format: + (8) + (1) + (list_size * Block_hash.size) *) + + let encode v = + let bytes = Bytes.create encoded_size in + Bytes.set_int64_be bytes 0 (Int64.of_int v.offset) ; + let len = List.length v.predecessors in + (* Start reading after the (8) *) + Bytes.set_int8 bytes 8 len ; + List.iteri + (fun i h -> + (* Start reading after the (8) + (1) *) + Bytes.blit + (Block_hash.to_bytes h) + 0 + bytes + (8 + 1 + (i * Block_hash.size)) + Block_hash.size) + v.predecessors ; + Bytes.unsafe_to_string bytes + + let decode str i = + let current_offset = ref i in + (* The Int64.to_int conversion is not likely to fail as it was + written based on Int64.of_int and the encoded offset won't + reach the int64 max value. *) + let offset = + TzEndian.get_int64_string str !current_offset |> Int64.to_int + in + (* Setting current_offset right after the (8)*) + current_offset := !current_offset + 8 ; + let list_size = TzEndian.get_int8_string str !current_offset in + current_offset := !current_offset + 1 ; + let predecessors = ref [] in + let limit = !current_offset in + current_offset := limit + ((list_size - 1) * Block_hash.size) ; + while !current_offset >= limit do + predecessors := + (String.sub str !current_offset Block_hash.size + |> Block_hash.of_string_exn) + :: !predecessors ; + current_offset := !current_offset - Block_hash.size + done ; + {offset; predecessors = !predecessors} + end + + include + Index_unix.Make (Block_key) (Legacy_block_info) (Index.Cache.Unbounded) +end diff --git a/src/lib_store/unix/floating_block_index.mli b/src/lib_store/unix/floating_block_index.mli index 17b9afa5f040..4a7d0ecaed25 100644 --- a/src/lib_store/unix/floating_block_index.mli +++ b/src/lib_store/unix/floating_block_index.mli @@ -38,6 +38,8 @@ module Block_info : sig type t = { offset : int; (** offset in the file *) predecessors : Block_hash.t list; (** predecessors of the block *) + resulting_context_hash : Context_hash.t; + (** context hash after block application*) } (** Pretty-printer for {!t} *) @@ -47,3 +49,19 @@ end (** Key/value index associated to a floating block store where the key is a {!Block_hash.t} and the value is {!Block_info.t}. *) include Index.S with type key = Block_hash.t and type value = Block_info.t + +module Legacy : sig + module Legacy_block_info : sig + (** The maximum number of predecessors stored per block. *) + val max_predecessors : int + + (** The type for storing the block's info. *) + type t = { + offset : int; (** offset in the file *) + predecessors : Block_hash.t list; (** predecessors of the block *) + } + end + + include + Index.S with type key = Block_hash.t and type value = Legacy_block_info.t +end diff --git a/src/lib_store/unix/floating_block_store.ml b/src/lib_store/unix/floating_block_store.ml index 4917c7e4f3b4..e362c7117500 100644 --- a/src/lib_store/unix/floating_block_store.ml +++ b/src/lib_store/unix/floating_block_store.ml @@ -41,11 +41,16 @@ type t = { scheduler : Lwt_idle_waiter.t; } +type info = { + predecessors : Block_hash.t list; + resulting_context_hash : Context_hash.t; +} + (* The log_size corresponds to the maximum size of the memory zone allocated in memory before flushing it onto the disk. It is basically a cache which is use for the index. The cache size is `log_size * log_entry` where a `log_entry` is roughly 56 bytes. *) -let floating_blocks_log_size = 10_000 +let default_floating_blocks_log_size = 10_000 let default_block_buffer_size = 500_000 @@ -58,39 +63,46 @@ let mem floating_store hash = Lwt.return (Floating_block_index.mem floating_store.floating_block_index hash)) -let find_predecessors floating_store hash = +let find_info floating_store hash = Lwt_idle_waiter.task floating_store.scheduler (fun () -> try - let {predecessors; _} = + let {predecessors; resulting_context_hash; _} = Floating_block_index.find floating_store.floating_block_index hash in - Lwt.return_some predecessors + Lwt.return_some {predecessors; resulting_context_hash} with Not_found -> Lwt.return_none) -let read_block_and_predecessors floating_store hash = +let ( let*? ) t k = let open Lwt_syntax in + let* v_opt = t in + match v_opt with None -> return_none | Some v -> k v + +let find_predecessors floating_store hash = + let*? {predecessors; _} = find_info floating_store hash in + Lwt.return_some predecessors + +let find_resulting_context_hash floating_store hash = + let*? {resulting_context_hash; _} = find_info floating_store hash in + Lwt.return_some resulting_context_hash + +let read_block_and_info floating_store hash = Lwt_idle_waiter.task floating_store.scheduler (fun () -> Option.catch_os (fun () -> - let {offset; predecessors} = + let {offset; predecessors; resulting_context_hash} = Floating_block_index.find floating_store.floating_block_index hash in - let* o = + let*? block, _ = Block_repr_unix.pread_block floating_store.fd ~file_offset:offset in - match o with - | Some (block, _) -> Lwt.return_some (block, predecessors) - | None -> - (* May be the case when a stored block is corrupted *) - Lwt.return_none)) + Lwt.return_some (block, {predecessors; resulting_context_hash}))) let read_block floating_store hash = let open Lwt_syntax in - let* o = read_block_and_predecessors floating_store hash in - match o with - | Some (block, _) -> Lwt.return_some block - | None -> Lwt.return_none + let*? block, _ = read_block_and_info floating_store hash in + return_some block -let locked_write_block floating_store ~offset ~block ~predecessors = +let locked_write_block floating_store ~offset ~block ~predecessors + ~resulting_context_hash = let open Lwt_result_syntax in let* block_bytes = match Data_encoding.Binary.to_bytes_opt Block_repr.encoding block with @@ -108,16 +120,21 @@ let locked_write_block floating_store ~offset ~block ~predecessors = Floating_block_index.replace floating_store.floating_block_index block.Block_repr.hash - {offset; predecessors} ; + {offset; predecessors; resulting_context_hash} ; return block_length -let append_block ?(flush = true) ?(log_metrics = false) floating_store - predecessors (block : Block_repr.t) = +let append_block floating_store ?(flush = true) ?(log_metrics = false) + ({predecessors; resulting_context_hash} : info) (block : Block_repr.t) = let open Lwt_result_syntax in Lwt_idle_waiter.force_idle floating_store.scheduler (fun () -> let*! offset = Lwt_unix.lseek floating_store.fd 0 Unix.SEEK_END in let* written_len = - locked_write_block floating_store ~offset ~block ~predecessors + locked_write_block + floating_store + ~offset + ~block + ~predecessors + ~resulting_context_hash in if flush then Floating_block_index.flush floating_store.floating_block_index ; @@ -127,16 +144,20 @@ let append_block ?(flush = true) ?(log_metrics = false) floating_store (Int.to_float written_len) ; return_unit) -let append_all floating_store - (blocks : (Block_hash.t list * Block_repr.t) Seq.t) = +let append_all floating_store (blocks : (Block_repr.t * info) Seq.t) = let open Lwt_result_syntax in Lwt_idle_waiter.force_idle floating_store.scheduler (fun () -> let*! eof_offset = Lwt_unix.lseek floating_store.fd 0 Unix.SEEK_END in let* _last_offset = Seq.fold_left_es - (fun offset (predecessors, block) -> + (fun offset (block, ({predecessors; resulting_context_hash} : info)) -> let* written_len = - locked_write_block floating_store ~offset ~block ~predecessors + locked_write_block + floating_store + ~offset + ~block + ~predecessors + ~resulting_context_hash in return (offset + written_len)) eof_offset @@ -162,14 +183,14 @@ let iter_s_raw_fd f fd = loop eof_offset (* May raise [Not_found] if index does not contain the block. *) -let iter_with_pred_s_raw_fd f fd block_index = +let iter_with_info_s_raw_fd f fd block_index = protect (fun () -> iter_s_raw_fd (fun block -> - let {predecessors; _} = + let {predecessors; resulting_context_hash; _} = Floating_block_index.find block_index block.hash in - f (block, predecessors)) + f (block, {predecessors; resulting_context_hash})) fd) let folder f floating_store = @@ -204,13 +225,13 @@ let fold_left_s f e floating_store = return !acc) floating_store -let fold_left_with_pred_s f e floating_store = +let fold_left_with_info_s f e floating_store = let open Lwt_result_syntax in folder (fun fd -> let acc = ref e in let* () = - iter_with_pred_s_raw_fd + iter_with_info_s_raw_fd (fun (b, preds) -> let* new_acc = f !acc (b, preds) in acc := new_acc ; @@ -224,8 +245,8 @@ let fold_left_with_pred_s f e floating_store = (* Iter sequentially on every blocks in the file *) let iter_s f floating_store = fold_left_s (fun () e -> f e) () floating_store -let iter_with_pred_s f floating_store = - fold_left_with_pred_s (fun () e -> f e) () floating_store +let iter_with_info_s f floating_store = + fold_left_with_info_s (fun () e -> f e) () floating_store let retrieve_block_from_stores floating_stores block_hash = let open Lwt_result_syntax in @@ -248,7 +269,7 @@ let raw_copy_block ~block_buffer ~src_floating_stores ~block_hash ~dst_floating_store = let open Lwt_result_syntax in protect (fun () -> - let* {offset; predecessors}, src_floating_store = + let* {offset; predecessors; resulting_context_hash}, src_floating_store = retrieve_block_from_stores src_floating_stores block_hash in let length_size = 4 in @@ -285,7 +306,7 @@ let raw_copy_block ~block_buffer ~src_floating_stores ~block_hash Floating_block_index.replace dst_floating_store.floating_block_index block_hash - {offset = new_offset; predecessors} ; + {offset = new_offset; predecessors; resulting_context_hash} ; return_unit) let raw_copy_all ~src_floating_stores ~block_hashes ~dst_floating_store = @@ -306,7 +327,7 @@ let raw_retrieve_blocks_seq ~src_floating_stores ~block_hashes = List.to_seq block_hashes |> Seq.map (fun block_hash -> protect (fun () -> - let* {offset; predecessors = _}, src_floating_store = + let* {offset; _}, src_floating_store = retrieve_block_from_stores src_floating_stores block_hash in let length_size = 4 in @@ -375,8 +396,51 @@ let raw_iterate f floating_store = in protect (fun () -> folder iterate floating_store) +let raw_iterate_fd f fd = + let open Lwt_result_syntax in + let block_buffer = ref (Bytes.create default_block_buffer_size) in + let block_size_buffer = Bytes.create 4 in + let iterate fd = + let*! end_of_file_offset = Lwt_unix.lseek fd 0 Unix.SEEK_END in + let rec loop current_offset = + if current_offset = end_of_file_offset then return_unit + else + let*! () = + Lwt_utils_unix.read_bytes + ~file_offset:current_offset + ~pos:0 + ~len:4 + fd + block_size_buffer + in + let block_length = + Bytes.get_int32_be block_size_buffer 0 |> Int32.to_int + in + let required_length = block_length + 4 in + let buffer_length = Bytes.length !block_buffer in + if buffer_length < required_length then + block_buffer := Bytes.create required_length ; + let*! () = + Lwt_utils_unix.read_bytes + ~file_offset:current_offset + ~pos:0 + ~len:required_length + fd + !block_buffer + in + let* () = f (!block_buffer, required_length) in + loop (current_offset + required_length) + in + loop 0 + in + protect (fun () -> iterate fd) + let raw_append floating_store - (block_hash, block_bytes, required_length, predecessors) = + ( block_hash, + block_bytes, + required_length, + predecessors, + resulting_context_hash ) = let open Lwt_result_syntax in protect @@ fun () -> let*! new_offset = Lwt_unix.lseek floating_store.fd 0 Unix.SEEK_END in @@ -390,7 +454,7 @@ let raw_append floating_store Floating_block_index.replace floating_store.floating_block_index block_hash - {offset = new_offset; predecessors} ; + {offset = new_offset; predecessors; resulting_context_hash} ; return_unit let init chain_dir ~readonly kind = @@ -421,7 +485,7 @@ let init chain_dir ~readonly kind = in let floating_block_index = Floating_block_index.v - ~log_size:floating_blocks_log_size + ~log_size:default_floating_blocks_log_size ~readonly (Naming.dir_path floating_index_dir) in @@ -440,8 +504,8 @@ let append_floating_store ~from ~into = let open Lwt_result_syntax in protect (fun () -> let* () = - iter_with_pred_s - (fun (block, preds) -> append_block ~flush:false into preds block) + iter_with_info_s + (fun (block, info) -> append_block ~flush:false into info block) from in Floating_block_index.flush ~with_fsync:true into.floating_block_index ; @@ -509,7 +573,7 @@ let full_integrity_check chain_dir kind = in let index = Floating_block_index.v - ~log_size:floating_blocks_log_size + ~log_size:default_floating_blocks_log_size ~readonly:false floating_blocks_index_dir_path in @@ -577,7 +641,7 @@ let fix_integrity chain_dir kind = iter_s (fun block -> let*! o = - find_predecessors + find_info inconsistent_floating_store (Block_repr.hash block) in diff --git a/src/lib_store/unix/floating_block_store.mli b/src/lib_store/unix/floating_block_store.mli index 4425e89b121c..7b097f349e68 100644 --- a/src/lib_store/unix/floating_block_store.mli +++ b/src/lib_store/unix/floating_block_store.mli @@ -26,22 +26,23 @@ (** Persistent block store with arborescent history The floating block store is an append-only store where blocks are - stored arbitrarily. This structure possess an indexed map - {!Block_hash.t} -> (offset × predecessors) which points to its - offset in the associated file along with an exponential list of - predecessors block hashes (as implemented in - {!Block_store.compute_predecessors}). The structure - access/modification is protected by a mutex ({!Lwt_idle_waiter}) - and thus can be manipulated concurrently. Stored blocks may or may - not contain metadata. The instance maintains an opened file - descriptor. Therefore it must be properly closed or it might lead - to a file descriptor leak. + stored arbitrarily. This structure possess an indexed map + {!Block_hash.t} -> (offset × predecessors x + resulting_context_hash) which points to its offset in the + associated file along with an exponential list of predecessors + block hashes (as implemented in + {!Block_store.compute_predecessors}) and the block's resulting + context hash. The structure access/modification is protected by a + mutex ({!Lwt_idle_waiter}) and thus can be manipulated + concurrently. Stored blocks may or may not contain metadata. The + instance maintains an opened file descriptor. Therefore it must be + properly closed or it might lead to a file descriptor leak. Four different kind of instances are allowed to co-exist for an - identical path: - RO, a read-only instance; - RW, a read-write - instance - RO_TMP, RW_TMP, read-write instances; - Restore is a - generic instance used to wrap leftover instances while fixing a - crashed storage. See {!Block_store}. + identical path: - RO, a read-only instance; - RW, a read-write + instance - RO_TMP, RW_TMP, read-write instances; - Restore is a + generic instance used to wrap leftover instances while fixing a + crashed storage. See {!Block_store}. {1 Invariants} @@ -72,6 +73,16 @@ type floating_kind = Naming.floating_kind = | RO_TMP | Restore of floating_kind +(** The type for informations stored in floating store indexes. *) +type info = { + predecessors : Block_hash.t list; + resulting_context_hash : Context_hash.t; +} + +(** The default value of the floating blocks cache size used by index + library. *) +val default_floating_blocks_log_size : int + (** [kind floating_store] returns the floating store's kind. *) val kind : t -> floating_kind @@ -79,40 +90,49 @@ val kind : t -> floating_kind [floating_store]. *) val mem : t -> Block_hash.t -> bool Lwt.t +(** [find_info floating_store block_hash] reads from the index the + info of [block_hash] if the block is stored in [floating_store], + returns [None] otherwise. *) +val find_info : t -> Block_hash.t -> info option Lwt.t + (** [find_predecessors floating_store block_hash] reads from the index the list of [block_hash]'s predecessors if the block is stored in [floating_store], returns [None] otherwise. *) val find_predecessors : t -> Block_hash.t -> Block_hash.t list option Lwt.t +(** [find_resulting_context_hash floating_store block_hash] reads from + the index the resulting context hash if the block is stored in + [floating_store], returns [None] otherwise. *) +val find_resulting_context_hash : + t -> Block_hash.t -> Context_hash.t option Lwt.t + (** [read_block floating_store hash] reads from the file the block of [hash] if the block is stored in [floating_store], returns [None] otherwise. *) val read_block : t -> Block_hash.t -> Block_repr.t option Lwt.t -(** [read_block_and_predecessors floating_store hash] same as - [read_block] but also returns the block's predecessors. Returns +(** [read_block_and_info floating_store hash] same as + [read_block] but also returns the block's info. Returns [None] if it fails to resolve the given [hash].*) -val read_block_and_predecessors : - t -> Block_hash.t -> (Block_repr.t * Block_hash.t list) option Lwt.t - -(** [append_block floating_store ?flush preds block] stores the - [block] in [floating_store] updating its index with the given - predecessors [preds] and flushing if [flush] is set to [true] - (defaults to [true]). [log_metrics] enables logs of the amount of - written bytes using the node's metrics. *) +val read_block_and_info : + t -> Block_hash.t -> (Block_repr.t * info) option Lwt.t + +(** [append_block floating_store ?flush ?log_metrics info block] + stores the [block] in [floating_store] updating its index with the + given [info] and flushing if [flush] is set to [true] (defaults to + [true]). [log_metrics] enables logs of the amount of written bytes + using the node's metrics. *) val append_block : + t -> ?flush:bool -> ?log_metrics:bool -> - t -> - Block_hash.t trace -> + info -> Block_repr.block -> unit tzresult Lwt.t -(** [append_all floating_store chunk] stores the [chunk] of - (predecessors × blocks) in [floating_store] updating its index - accordingly. *) -val append_all : - t -> (Block_hash.t list * Block_repr.t) Seq.t -> unit tzresult Lwt.t +(** [append_all floating_store chunk] stores the [chunk] of (blocks x + info) in [floating_store] updating its index accordingly. *) +val append_all : t -> (Block_repr.t * info) Seq.t -> unit tzresult Lwt.t (** [iter_s_raw_fd f fd] unsafe sequential iterator on a file descriptor [fd]. Applies [f] on every block encountered. @@ -129,12 +149,12 @@ val iter_s_raw_fd : val fold_left_s : ('a -> Block_repr.block -> 'a tzresult Lwt.t) -> 'a -> t -> 'a tzresult Lwt.t -(** [fold_left_with_pred_s f e floating_store] sequential fold left on the +(** [fold_left_with_info_s f e floating_store] sequential fold left on the [floating_store] using [f] on every block and [e] as initial element. The function [f] is given the last read block along with - its predecessors. *) -val fold_left_with_pred_s : - ('a -> Block_repr.block * Block_hash.t list -> 'a tzresult Lwt.t) -> + its info. *) +val fold_left_with_info_s : + ('a -> Block_repr.block * info -> 'a tzresult Lwt.t) -> 'a -> t -> 'a tzresult Lwt.t @@ -143,13 +163,11 @@ val fold_left_with_pred_s : [floating_store]. Applies [f] on every block read. *) val iter_s : (Block_repr.t -> unit tzresult Lwt.t) -> t -> unit tzresult Lwt.t -(** [iter_with_pred_s f floating_store] sequential iterator on the - [floating_store]. Applies [f] on every block with its predecessors +(** [iter_with_info_s f floating_store] sequential iterator on the + [floating_store]. Applies [f] on every block (and its info) read. *) -val iter_with_pred_s : - (Block_repr.t * Block_hash.t list -> unit tzresult Lwt.t) -> - t -> - unit tzresult Lwt.t +val iter_with_info_s : + (Block_repr.t * info -> unit tzresult Lwt.t) -> t -> unit tzresult Lwt.t (** [init ~chain_dir ~readonly kind] creates or load an existing floating store at path [chain_dir] with [kind]. @@ -194,11 +212,13 @@ val fix_integrity : (** Unsafe set of functions intended for merging optimizations. *) -(** [raw_append dst_store (hash, buffer, total_length, predecessors)] - appends a block with its [hash] in [dst_store] contained in the - [buffer]. *) +(** [raw_append dst_store (hash, buffer, total_length, predecessors, + resulting_context_hash)] appends a block with its [hash] in + [dst_store] contained in the [buffer]. *) val raw_append : - t -> Block_hash.t * bytes * int * Block_hash.t list -> unit tzresult Lwt.t + t -> + Block_hash.t * bytes * int * Block_hash.t list * Context_hash.t -> + unit tzresult Lwt.t (** [raw_copy_all src_stores block_hashes dst_store] retrieves [block_hashes] from [src_stores] and copy them (without decoding) @@ -232,6 +252,22 @@ val raw_retrieve_blocks_seq : is consistent. {b Warning}: the [buffer] is modified through the iteration and - therefore must not be used outside of the definition of [f]. *) + therefore must not be used outside of the definition of [f]. + + For internal use only! *) val raw_iterate : (bytes * int -> unit tzresult Lwt.t) -> t -> unit tzresult Lwt.t + +(** [raw_iterate_fd f fd] iterate over all blocks in a floating store + block file and calling [f] providing it with a [buffer] and the + [total_block_length] such that [(decode Block_repr.encoding + (Bytes.sub buffer 0 total_block_length))] is consistent. + + {b Warning}: the [buffer] is modified through the iteration and + therefore must not be used outside of the definition of [f]. + + For internal use only! *) +val raw_iterate_fd : + (bytes * int -> unit tzresult Lwt.t) -> + Lwt_unix.file_descr -> + unit tzresult Lwt.t diff --git a/src/lib_store/unix/reconstruction.ml b/src/lib_store/unix/reconstruction.ml index e7419926917b..fb89bf1bd097 100644 --- a/src/lib_store/unix/reconstruction.ml +++ b/src/lib_store/unix/reconstruction.ml @@ -25,7 +25,6 @@ type failure_kind = | Nothing_to_reconstruct - | Context_hash_mismatch of Block_header.t * Context_hash.t * Context_hash.t | Cannot_read_block_hash of Block_hash.t | Cannot_read_block_level of Int32.t @@ -41,22 +40,12 @@ let failure_kind_encoding = (fun () -> Nothing_to_reconstruct); case (Tag 1) - ~title:"context_hash_mismatch" - (obj3 - (req "block_header" Block_header.encoding) - (req "expected" Context_hash.encoding) - (req "got" Context_hash.encoding)) - (function - | Context_hash_mismatch (h, e, g) -> Some (h, e, g) | _ -> None) - (fun (h, e, g) -> Context_hash_mismatch (h, e, g)); - case - (Tag 2) ~title:"cannot_read_block_hash" Block_hash.encoding (function Cannot_read_block_hash h -> Some h | _ -> None) (fun h -> Cannot_read_block_hash h); case - (Tag 3) + (Tag 2) ~title:"cannot_read_block_level" int32 (function Cannot_read_block_level l -> Some l | _ -> None) @@ -65,18 +54,6 @@ let failure_kind_encoding = let failure_kind_pp ppf = function | Nothing_to_reconstruct -> Format.fprintf ppf "nothing to reconstruct" - | Context_hash_mismatch (h, e, g) -> - Format.fprintf - ppf - "resulting context hash for block %a (level %ld) does not match. \ - Context hash expected %a, got %a" - Block_hash.pp - (Block_header.hash h) - h.shell.level - Context_hash.pp - e - Context_hash.pp - g | Cannot_read_block_hash h -> Format.fprintf ppf "Unexpected missing block in store: %a" Block_hash.pp h | Cannot_read_block_level l -> @@ -156,14 +133,6 @@ let cemented_metadata_status cemented_store in search (Int32.succ start_level) (Int32.pred end_level)) -let check_context_hash_consistency block_validation_result block_header = - let expected = block_header.Block_header.shell.context in - let got = block_validation_result.Block_validation.context_hash in - fail_unless - (Context_hash.equal expected got) - (Reconstruction_failure - (Context_hash_mismatch (block_header, expected, got))) - (* We assume that the given list is not empty. *) let compute_block_metadata_hash block_metadata = Some (Block_metadata_hash.hash_bytes [block_metadata]) @@ -193,20 +162,20 @@ let compute_all_operations_metadata_hash block = let apply_context context_index chain_id ~user_activated_upgrades ~user_activated_protocol_overrides ~operation_metadata_size_limit ~predecessor_block_metadata_hash ~predecessor_ops_metadata_hash - ~predecessor_block block = + ~predecessor_block ~expected_context_hash block = let open Lwt_result_syntax in let block_header = Store.Block.header block in let operations = Store.Block.operations block in let predecessor_block_header = Store.Block.header predecessor_block in - let context_hash = predecessor_block_header.shell.context in - let* predecessor_context = - let*! o = Context_ops.checkout context_index context_hash in - match o with - | Some ctxt -> return ctxt - | None -> - tzfail - (Store_errors.Cannot_checkout_context - (Store.Block.hash predecessor_block, context_hash)) + let predecessor_resulting_context_hash = + if + expected_context_hash + = Tezos_protocol_environment__Environment_context.Resulting_context + then Store.Block.context_hash predecessor_block + else Store.Block.context_hash block + in + let*! predecessor_context = + Context_ops.checkout_exn context_index predecessor_resulting_context_hash in let apply_environment = { @@ -217,6 +186,7 @@ let apply_context context_index chain_id ~user_activated_upgrades predecessor_context; predecessor_block_metadata_hash; predecessor_ops_metadata_hash; + predecessor_resulting_context_hash; user_activated_upgrades; user_activated_protocol_overrides; operation_metadata_size_limit; @@ -239,7 +209,6 @@ let apply_context context_index chain_id ~user_activated_upgrades block to the next one. *) in - let* () = check_context_hash_consistency validation_store block_header in return ( validation_store.message, validation_store.max_operations_ttl, @@ -248,7 +217,7 @@ let apply_context context_index chain_id ~user_activated_upgrades ops_metadata ) (** Returns the protocol environment version of a given protocol level. *) -let protocol_env_of_protocol_level chain_store protocol_level block_hash = +let protocol_of_protocol_level chain_store protocol_level block_hash = let open Lwt_result_syntax in let* protocol_hash = let*! o = Store.Chain.find_protocol chain_store ~protocol_level in @@ -256,12 +225,10 @@ let protocol_env_of_protocol_level chain_store protocol_level block_hash = | Some ph -> return ph | None -> tzfail (Store_errors.Cannot_find_protocol protocol_level) in - match Registered_protocol.get protocol_hash with - | None -> - tzfail - (Block_validator_errors.Unavailable_protocol - {block = block_hash; protocol = protocol_hash}) - | Some (module Proto) -> return Proto.environment_version + trace + (Block_validator_errors.Unavailable_protocol + {block = block_hash; protocol = protocol_hash}) + (Registered_protocol.get_result protocol_hash) (* Restores the block and operations metadata hash of a given block, if needed. *) @@ -342,6 +309,12 @@ let reconstruct_chunk chain_store context_index ~user_activated_upgrades "Cannot read block in cemented store. The storage is corrupted." | Some b -> return b in + let* (module Proto) = + protocol_of_protocol_level + chain_store + (Store.Block.proto_level block) + (Store.Block.hash block) + in let* ( message, max_operations_ttl, last_allowed_fork_level, @@ -386,17 +359,13 @@ let reconstruct_chunk chain_store context_index ~user_activated_upgrades ~predecessor_block_metadata_hash ~predecessor_ops_metadata_hash ~predecessor_block + ~expected_context_hash:Proto.expected_context_hash block in let*! () = Event.(emit reconstruct_block_success) (Store.Block.descriptor block) in - let* block_protocol_env = - protocol_env_of_protocol_level - chain_store - (Store.Block.proto_level block) - (Store.Block.hash block) - in + let block_protocol_env = Proto.environment_version in let reconstructed_block = restore_block_contents chain_store @@ -420,7 +389,7 @@ let store_chunk cemented_store chunk = | Some e -> return e in let* _, higher_env_version = - match List.hd (List.rev chunk) with + match List.last_opt chunk with | None -> failwith "Cannot read chunk to cement." | Some e -> return e in @@ -561,13 +530,13 @@ let reconstruct_cemented chain_store context_index ~user_activated_upgrades let* available_metadata = List.map_es (fun br -> - let* proto_env_version = - protocol_env_of_protocol_level + let* (module Proto) = + protocol_of_protocol_level chain_store (Block_repr.proto_level br) (Block_repr.hash br) in - return (br, proto_env_version)) + return (br, Proto.environment_version)) brs in let* () = @@ -616,9 +585,15 @@ let reconstruct_floating chain_store context_index ~user_activated_upgrades List.iter_es (fun fs -> let* () = - Floating_block_store.iter_with_pred_s - (fun (block, predecessors) -> + Floating_block_store.iter_with_info_s + (fun (block, info) -> let level = Block_repr.level block in + let* (module Proto) = + protocol_of_protocol_level + chain_store + (Block_repr.proto_level block) + (Block_repr.hash block) + in (* If the block is genesis then just retrieve its metadata. *) let* ( message, max_operations_ttl, @@ -697,6 +672,7 @@ let reconstruct_floating chain_store context_index ~user_activated_upgrades (Store.Block.all_operations_metadata_hash predecessor_block) ~predecessor_block + ~expected_context_hash:Proto.expected_context_hash block in let*! () = @@ -735,16 +711,10 @@ let reconstruct_floating chain_store context_index ~user_activated_upgrades block_metadata, operations_metadata ) in - let* block_protocol_env = - protocol_env_of_protocol_level - chain_store - (Block_repr.proto_level block) - (Block_repr.hash block) - in let reconstructed_block = restore_block_contents chain_store - block_protocol_env + Proto.environment_version ~block_metadata ~operations_metadata message @@ -755,7 +725,7 @@ let reconstruct_floating chain_store context_index ~user_activated_upgrades let* () = Floating_block_store.append_block new_ro_store - predecessors + info reconstructed_block in let*! () = notify () in diff --git a/src/lib_store/unix/reconstruction.mli b/src/lib_store/unix/reconstruction.mli index 05a3b296ca49..016b1b3ca2b4 100644 --- a/src/lib_store/unix/reconstruction.mli +++ b/src/lib_store/unix/reconstruction.mli @@ -53,7 +53,6 @@ type failure_kind = | Nothing_to_reconstruct - | Context_hash_mismatch of Block_header.t * Context_hash.t * Context_hash.t | Cannot_read_block_hash of Block_hash.t | Cannot_read_block_level of Int32.t diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index bc800b9c9110..3b23ba8230ba 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -2328,21 +2328,27 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct let protocol_levels = Protocol_levels.( filter - (fun level {block; _} -> + (fun level {activation_block; _} -> + let block = activation_block.block in level >= Store.Block.proto_level minimum_block || Store.Block.is_genesis chain_store (fst block)) protocol_levels) in + let* pred_resulting_context = + Store.Block.resulting_context_hash chain_store pred_block + in return ( export_mode, export_block, pred_block, + pred_resulting_context, protocol_levels, (return_unit, floating_block_stream) ) in let* ( export_mode, export_block, pred_block, + pred_resulting_context, protocol_levels, (return_unit, floating_block_stream) ) = Store.Unsafe.open_for_snapshot_export @@ -2355,6 +2361,7 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct ( export_mode, export_block, pred_block, + pred_resulting_context, protocol_levels, (return_unit, floating_block_stream) ) @@ -2417,10 +2424,14 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct Compare.Int32.( max_cemented_level > Store.Block.level export_block) in + let* pred_resulting_context = + Store.Block.resulting_context_hash chain_store pred_block + in return ( export_mode, export_block, pred_block, + pred_resulting_context, protocol_levels, cemented_table, (ro_fd, rw_fd), @@ -2434,6 +2445,7 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct let* ( export_mode, export_block, pred_block, + pred_resulting_context, protocol_levels, cemented_table, (floating_ro_fd, floating_rw_fd), @@ -2480,6 +2492,7 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct ( export_mode, export_block, pred_block, + pred_resulting_context, protocol_levels, (reading_thread, floating_block_stream) ) @@ -2512,6 +2525,7 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct let* ( export_mode, export_block, pred_block, + pred_resulting_context, protocol_levels, (reading_thread, floating_block_stream) ) = if rolling then @@ -2531,18 +2545,13 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct now, it is performed outside the lock to allow the node from getting stuck while waiting a merge. *) let*! context_index = Context.init ~readonly:true context_dir in - let*! context = - Context.checkout_exn - context_index - (Store.Block.context_hash export_block) - in (* Retrieve predecessor block metadata hash and operations metadata hash from the context of the exported block *) - let*! predecessor_block_metadata_hash = - Context.find_predecessor_block_metadata_hash context + let predecessor_block_metadata_hash = + Store.Block.block_metadata_hash pred_block in - let*! predecessor_ops_metadata_hash = - Context.find_predecessor_ops_metadata_hash context + let predecessor_ops_metadata_hash = + Store.Block.all_operations_metadata_hash pred_block in let*! () = Exporter.write_block_data @@ -2556,7 +2565,7 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct Exporter.dump_context snapshot_exporter context_index - (Store.Block.context_hash pred_block) + pred_resulting_context ~on_disk progress_display_mode in @@ -2758,7 +2767,7 @@ module type IMPORTER = sig unit tzresult Lwt.t val load_protocol_table : - t -> Protocol_levels.activation_block Protocol_levels.t tzresult Lwt.t + t -> Protocol_levels.protocol_info Protocol_levels.t tzresult Lwt.t val load_and_validate_protocol_filenames : t -> Protocol_hash.t list tzresult Lwt.t @@ -2858,7 +2867,6 @@ module Raw_importer : IMPORTER = struct ~in_memory ~progress_display_mode in - (* FIXME: Is this test really usefull? *) let*! current = Lwt_unix.lseek fd 0 Lwt_unix.SEEK_CUR in let*! stats = Lwt_unix.fstat fd in let total = stats.Lwt_unix.st_size in @@ -2876,9 +2884,37 @@ module Raw_importer : IMPORTER = struct Data_encoding.Binary.of_string_opt Protocol_levels.encoding table_bytes with | Some table -> return table - | None -> - tzfail - (Cannot_read {kind = `Protocol_table; path = protocol_tbl_filename}) + | None -> ( + (* Try with legacy's encoding *) + match + Data_encoding.Binary.of_string_opt + Protocol_levels.Legacy.encoding + table_bytes + with + | Some table -> + Protocol_levels.Legacy.fold_es + (fun proto_level activation_block map -> + let protocol_info = + { + Protocol_levels.protocol = + activation_block.Protocol_levels.Legacy.protocol; + activation_block = + { + Protocol_levels.block = activation_block.block; + commit_info = activation_block.commit_info; + }; + (* Only snapshot with legacy semantics will have + legacy encoding. *) + expect_predecessor_context = false; + } + in + return (Protocol_levels.add proto_level protocol_info map)) + table + Protocol_levels.empty + | None -> + tzfail + (Cannot_read + {kind = `Protocol_table; path = protocol_tbl_filename})) let load_and_validate_protocol_filenames t = let open Lwt_result_syntax in @@ -3133,15 +3169,49 @@ module Tar_importer : IMPORTER = struct Onthefly.load_from_filename t.tar ~filename:protocol_tbl_filename in match o with - | Some str -> - let _ofs, res = - Data_encoding.Binary.read_exn + | Some str -> ( + let res = + Data_encoding.Binary.read_opt Protocol_levels.encoding str 0 (String.length str) in - return res + match res with + | Some (_ofs, res) -> return res + | None -> ( + (* Try with legacy's encoding *) + match + Data_encoding.Binary.read_opt + Protocol_levels.Legacy.encoding + str + 0 + (String.length str) + with + | Some (_ofs, table) -> + Protocol_levels.Legacy.fold_es + (fun proto_level activation_block map -> + let protocol_info = + { + Protocol_levels.protocol = + activation_block.Protocol_levels.Legacy.protocol; + activation_block = + { + Protocol_levels.block = activation_block.block; + commit_info = activation_block.commit_info; + }; + (* Only snapshot with legacy semantics will have + legacy encoding. *) + expect_predecessor_context = false; + } + in + return (Protocol_levels.add proto_level protocol_info map)) + table + Protocol_levels.empty + | None -> + tzfail + (Cannot_read + {kind = `Protocol_table; path = protocol_tbl_filename}))) | None -> tzfail (Cannot_read {kind = `Protocol_table; path = protocol_tbl_filename}) @@ -3483,21 +3553,26 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct in Event.(emit import_loading ()) - let check_context_hash_consistency validation_store block_header = + let check_context_hash_consistency ~expect_predecessor_context + validation_store block_header = + (* FIXME: provide the resulting block context in the snapshot and + check against it when the [expect_predecessor_context] + semantics is used. *) fail_unless - (Context_hash.equal - validation_store.Tezos_validation.Block_validation.context_hash - block_header.Block_header.shell.context) + (expect_predecessor_context + || Context_hash.equal + validation_store.Tezos_validation.Block_validation.context_hash + block_header.Block_header.shell.context) (Inconsistent_context_hash { expected = block_header.Block_header.shell.context; got = validation_store.Tezos_validation.Block_validation.context_hash; }) - let restore_and_apply_context snapshot_importer ?user_expected_block - ~context_index ~user_activated_upgrades ~user_activated_protocol_overrides - ~operation_metadata_size_limit ~legacy ~in_memory ~progress_display_mode - snapshot_metadata genesis chain_id = + let restore_and_apply_context snapshot_importer protocol_levels + ?user_expected_block ~context_index ~user_activated_upgrades + ~user_activated_protocol_overrides ~operation_metadata_size_limit ~legacy + ~in_memory ~progress_display_mode snapshot_metadata genesis chain_id = let open Lwt_result_syntax in (* Start by committing genesis *) let* genesis_ctxt_hash = @@ -3535,23 +3610,39 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct (Inconsistent_imported_block (block_header_hash, snapshot_metadata.block_hash)) in + let expected_context_hash, expect_predecessor_context = + match + Protocol_levels.find + predecessor_header.shell.proto_level + protocol_levels + with + | None -> Stdlib.failwith "unknown protocol" + | Some {Protocol_levels.expect_predecessor_context; _} -> + if expect_predecessor_context then + (block_header.Block_header.shell.context, expect_predecessor_context) + else + ( predecessor_header.Block_header.shell.context, + expect_predecessor_context ) + in + let predecessor_resulting_context_hash = expected_context_hash in (* Restore context *) let* () = Importer.restore_context snapshot_importer context_index - ~expected_context_hash:predecessor_header.Block_header.shell.context + ~expected_context_hash ~nb_context_elements:snapshot_metadata.context_elements ~legacy ~in_memory ~progress_display_mode in - let pred_context_hash = predecessor_header.shell.context in let* predecessor_context = - let*! o = Context.checkout context_index pred_context_hash in + let*! o = + Context.checkout context_index predecessor_resulting_context_hash + in match o with | Some ch -> return ch - | None -> tzfail (Inconsistent_context pred_context_hash) + | None -> tzfail (Inconsistent_context predecessor_resulting_context_hash) in let predecessor_context = Tezos_shell_context.Shell_context.wrap_disk_context predecessor_context @@ -3563,6 +3654,7 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct chain_id; predecessor_block_header = predecessor_header; predecessor_context; + predecessor_resulting_context_hash; predecessor_block_metadata_hash; predecessor_ops_metadata_hash; user_activated_upgrades; @@ -3592,6 +3684,7 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct in let* () = check_context_hash_consistency + ~expect_predecessor_context block_validation_result.validation_store block_header in @@ -3688,10 +3781,15 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct ?patch_context dst_context_dir in + (* Restore protocols *) + let* protocol_levels = + restore_protocols snapshot_importer progress_display_mode + in (* Restore context *) let* block_data, genesis_context_hash, block_validation_result = restore_and_apply_context snapshot_importer + protocol_levels ?user_expected_block ~context_index ~user_activated_upgrades @@ -3705,10 +3803,6 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct ~in_memory in (* Restore store *) - (* Restore protocols *) - let* protocol_levels = - restore_protocols snapshot_importer progress_display_mode - in (* Restore cemented dir *) let* () = restore_cemented_blocks @@ -3721,7 +3815,12 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct let* reading_thread, floating_blocks_stream = read_floating_blocks snapshot_importer ~genesis_hash:genesis.block in - let {Block_validation.validation_store; block_metadata; ops_metadata} = + let { + Block_validation.validation_store; + block_metadata; + ops_metadata; + shell_header_hash = _; + } = block_validation_result in let contents = @@ -3781,6 +3880,7 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct ~genesis_context_hash ~floating_blocks_stream ~new_head_with_metadata + ~new_head_resulting_context_hash:validation_store.context_hash ~protocol_levels ~history_mode) in diff --git a/src/lib_store/unix/store.ml b/src/lib_store/unix/store.ml index 3ea97afccde7..1a68825fd452 100644 --- a/src/lib_store/unix/store.ml +++ b/src/lib_store/unix/store.ml @@ -93,7 +93,7 @@ and chain_state = { checkpoint_data : block_descriptor Stored_data.t; (* Following fields are safe to update directly *) protocol_levels_data : - Protocol_levels.activation_block Protocol_levels.t Stored_data.t; + Protocol_levels.protocol_info Protocol_levels.t Stored_data.t; invalid_blocks_data : invalid_block Block_hash.Map.t Stored_data.t; forked_chains_data : Block_hash.t Chain_id.Map.t Stored_data.t; (* In memory-only: *) @@ -162,6 +162,49 @@ let locked_is_acceptable_block chain_state (hash, level) = Lwt.return @@ Block_hash.equal hash target_hash else Lwt.return_true +(* Shared protocols accessors *) + +let find_protocol_info chain_store ~protocol_level = + let open Lwt_syntax in + Shared.use chain_store.chain_state (fun {protocol_levels_data; _} -> + let* protocol_levels = Stored_data.get protocol_levels_data in + return (Protocol_levels.find protocol_level protocol_levels)) + +let find_activation_block chain_store ~protocol_level = + let open Lwt_syntax in + let* protocol_info = find_protocol_info chain_store ~protocol_level in + match protocol_info with + | Some {activation_block; _} -> return_some activation_block + | None -> return_none + +let find_protocol chain_store ~protocol_level = + let open Lwt_syntax in + let* o = find_protocol_info chain_store ~protocol_level in + match o with + | None -> return_none + | Some {Protocol_levels.protocol; _} -> return_some protocol + +let expect_predecessor_context_hash_exn chain_store protocol_level = + let open Lwt_syntax in + let* protocol_info = find_protocol_info chain_store ~protocol_level in + match protocol_info with + | Some {expect_predecessor_context; _} -> return expect_predecessor_context + | None -> + Format.ksprintf + Stdlib.failwith + "cannot find protocol info for level: %d" + protocol_level + +let expect_predecessor_context_hash chain_store ~protocol_level = + let open Lwt_result_syntax in + Lwt.catch + (fun () -> + let*! b = + expect_predecessor_context_hash_exn chain_store protocol_level + in + return b) + (fun _ -> tzfail (Protocol_not_found {protocol_level})) + let create_lockfile chain_dir = let open Lwt_syntax in protect (fun () -> @@ -404,7 +447,7 @@ module Block = struct let { Block_validation.validation_store = { - context_hash; + context_hash = resulting_context_hash; timestamp = _; message; max_operations_ttl; @@ -412,6 +455,7 @@ module Block = struct }; block_metadata; ops_metadata; + shell_header_hash = _; } = validation_result in @@ -500,12 +544,6 @@ module Block = struct known_invalid Store_errors.(Cannot_store_block (hash, Invalid_block)) in - let* () = - fail_unless - (Context_hash.equal block_header.shell.context context_hash) - (Validation_errors.Inconsistent_hash - (context_hash, block_header.shell.context)) - in let contents = { Block_repr.header = block_header; @@ -533,7 +571,12 @@ module Block = struct } in let block = {Block_repr.hash; contents; metadata} in - let* () = Block_store.store_block chain_store.block_store block in + let* () = + Block_store.store_block + chain_store.block_store + block + resulting_context_hash + in let*! () = Store_events.(emit store_block) (hash, block_header.shell.level) in @@ -583,13 +626,70 @@ module Block = struct in return_unit + let resulting_context_hash chain_store block = + let open Lwt_result_syntax in + let fetch_expect_predecessor_context () = + expect_predecessor_context_hash + chain_store + ~protocol_level:(Block_repr.proto_level block) + in + let hash = Block_repr.hash block in + let* resulting_context_hash_opt = + Block_store.resulting_context_hash + ~fetch_expect_predecessor_context + chain_store.block_store + (Block (hash, 0)) + in + match resulting_context_hash_opt with + | None -> + tzfail + (Resulting_context_hash_not_found + {hash; level = Block_repr.level block}) + | Some resulting_context_hash -> return resulting_context_hash + let context_exn chain_store block = + let open Lwt_syntax in let context_index = chain_store.global_store.context_index in - Context_ops.checkout_exn context_index (Block_repr.context block) + let fetch_expect_predecessor_context () = + let* expect_pred_context_opt = + expect_predecessor_context_hash + chain_store + ~protocol_level:(Block_repr.proto_level block) + in + match expect_pred_context_opt with + | Ok expect_pred_context -> Lwt.return_ok expect_pred_context + | Error _ -> + Format.kasprintf + Stdlib.failwith + "cannot find the resulting context of block %a" + pp_block_descriptor + (Block_repr.descriptor block) + in + let* context_to_checkout = + let* r = + Block_store.resulting_context_hash + chain_store.block_store + ~fetch_expect_predecessor_context + (Block (Block_repr.hash block, 0)) + in + match r with + | Ok (Some resulting_context) -> return resulting_context + | Ok None | Error _ -> + Format.kasprintf + Stdlib.failwith + "cannot find the resulting context of block %a" + pp_block_descriptor + (Block_repr.descriptor block) + in + Context_ops.checkout_exn context_index context_to_checkout let context_opt chain_store block = - let context_index = chain_store.global_store.context_index in - Context_ops.checkout context_index (Block_repr.context block) + let open Lwt_syntax in + Lwt.catch + (fun () -> + let* ctxt = context_exn chain_store block in + return_some ctxt) + (fun _exn -> Lwt.return_none) let context chain_store block = let open Lwt_result_syntax in @@ -1124,8 +1224,8 @@ module Chain = struct in match Protocol_levels.find proto_level protocol_levels with | None -> Lwt.return_none - | Some {block; _} -> ( - let block_activation_level = snd block in + | Some {activation_block; _} -> ( + let block_activation_level = snd activation_block.block in (* proto level's lower bound found, now retrieving the upper bound *) let head_proto_level = Block_repr.proto_level chain_state.current_head @@ -1141,8 +1241,8 @@ module Chain = struct Protocol_levels.find (succ proto_level) protocol_levels with | None -> Lwt.return_none - | Some {block; _} -> ( - let next_activation_level = snd block in + | Some {activation_block; _} -> ( + let next_activation_level = snd activation_block.block in let last_level_in_protocol = Int32.(pred next_activation_level) in @@ -1845,6 +1945,14 @@ module Chain = struct ~some:(fun metadata -> Block.last_allowed_fork_level metadata) (Block_repr.metadata genesis_block) in + let activation_block = + {Protocol_levels.block = genesis_descr; commit_info = genesis_commit_info} + in + let* expect_predecessor_context = + let open Lwt_result_syntax in + let* (module Proto) = Registered_protocol.get_result genesis_protocol in + return (Proto.expected_context_hash = Predecessor_resulting_context) + in let* protocol_levels_data = Stored_data.init (Naming.protocol_levels_file chain_dir) @@ -1853,9 +1961,9 @@ module Chain = struct add genesis_proto_level { - block = genesis_descr; protocol = genesis_protocol; - commit_info = genesis_commit_info; + activation_block; + expect_predecessor_context; } empty) in @@ -2309,6 +2417,18 @@ module Chain = struct (* Protocols *) + let find_protocol_info chain_store ~protocol_level = + find_protocol_info chain_store ~protocol_level + + let find_activation_block chain_store ~protocol_level = + find_activation_block chain_store ~protocol_level + + let find_protocol chain_store ~protocol_level = + find_protocol chain_store ~protocol_level + + let expect_predecessor_context_hash chain_store ~protocol_level = + expect_predecessor_context_hash chain_store ~protocol_level + let compute_commit_info chain_store block = let open Lwt_result_syntax in let index = chain_store.global_store.context_index in @@ -2318,7 +2438,8 @@ module Chain = struct let* commit_info = get_commit_info index block in return_some commit_info) - let set_protocol_level chain_store ~protocol_level (block, protocol_hash) = + let set_protocol_level chain_store ~protocol_level + (block, protocol_hash, expect_predecessor_context) = let open Lwt_result_syntax in Shared.locked_use chain_store.chain_state (fun {protocol_levels_data; _} -> let* commit_info_opt = @@ -2326,14 +2447,21 @@ module Chain = struct in let* () = Stored_data.update_with protocol_levels_data (fun protocol_levels -> + let activation_block = + Protocol_levels. + { + block = Block.descriptor block; + commit_info = commit_info_opt; + } + in Lwt.return Protocol_levels.( add protocol_level { - block = Block.descriptor block; protocol = protocol_hash; - commit_info = commit_info_opt; + activation_block; + expect_predecessor_context; } protocol_levels)) in @@ -2348,21 +2476,8 @@ module Chain = struct in return_unit) - let find_activation_block chain_store ~protocol_level = - let open Lwt_syntax in - Shared.use chain_store.chain_state (fun {protocol_levels_data; _} -> - let* protocol_levels = Stored_data.get protocol_levels_data in - Lwt.return (Protocol_levels.find protocol_level protocol_levels)) - - let find_protocol chain_store ~protocol_level = - let open Lwt_syntax in - let* o = find_activation_block chain_store ~protocol_level in - match o with - | None -> Lwt.return_none - | Some {Protocol_levels.protocol; _} -> Lwt.return_some protocol - let may_update_protocol_level chain_store ?pred ?protocol_level - (block, protocol_hash) = + ~expect_predecessor_context (block, protocol_hash) = let open Lwt_result_syntax in let* pred = match pred with @@ -2378,18 +2493,22 @@ module Chain = struct match o with | Some {block = bh, _; _} -> if Block_hash.(bh <> Block.hash block) then - set_protocol_level chain_store ~protocol_level (block, protocol_hash) + set_protocol_level + chain_store + ~protocol_level + (block, protocol_hash, expect_predecessor_context) else return_unit | None -> - set_protocol_level chain_store ~protocol_level (block, protocol_hash) + set_protocol_level + chain_store + ~protocol_level + (block, protocol_hash, expect_predecessor_context) else return_unit let may_update_ancestor_protocol_level chain_store ~head = let open Lwt_result_syntax in let head_proto_level = Block.proto_level head in - let*! o = - find_activation_block chain_store ~protocol_level:head_proto_level - in + let*! o = find_protocol_info chain_store ~protocol_level:head_proto_level in match o with | None -> (* If no activation block is registered for a given protocol @@ -2415,11 +2534,22 @@ module Chain = struct (* The head must have an associated context stored. *) let* context = Block.context chain_store head in let*! activated_protocol = Context_ops.get_protocol context in + let* (module Proto) = + Registered_protocol.get_result activated_protocol + in + let expected_context_hash = + Proto.expected_context_hash = Predecessor_resulting_context + in set_protocol_level chain_store ~protocol_level - (activation_block, activated_protocol) - | Some {block; protocol; _} -> ( + (activation_block, activated_protocol, expected_context_hash) + | Some + { + Protocol_levels.protocol; + activation_block = {block; _}; + expect_predecessor_context; + } -> ( let*! _, savepoint_level = savepoint chain_store in if Compare.Int32.(savepoint_level > snd block) then (* the block is too far in the past *) @@ -2443,7 +2573,10 @@ module Chain = struct match o with | None -> return_unit | Some ancestor -> - may_update_protocol_level chain_store (ancestor, protocol))) + may_update_protocol_level + chain_store + (ancestor, protocol) + ~expect_predecessor_context)) let all_protocol_levels chain_store = Shared.use chain_store.chain_state (fun {protocol_levels_data; _} -> @@ -2463,7 +2596,7 @@ module Chain = struct let* protocol = if Compare.Int32.(Block.level pred < save_point_level) then let* o = - find_activation_block + find_protocol_info chain_store ~protocol_level:(Block.proto_level pred) in @@ -2955,8 +3088,9 @@ let rec make_pp_chain_store (chain_store : chain_store) = forked_chains, active_testchain )) in - let pp_protocol_level fmt - (proto_level, {Protocol_levels.block; protocol; commit_info}) = + let pp_proto_info fmt + (proto_level, {Protocol_levels.protocol; activation_block; _}) = + let Protocol_levels.{block; commit_info} = activation_block in Format.fprintf fmt "proto level: %d, transition block: %a, protocol: %a, commit info: %a" @@ -3027,7 +3161,7 @@ let rec make_pp_chain_store (chain_store : chain_store) = merge_status (option_pp ~default:"n/a" pp_block_descriptor) target - (Format.pp_print_list ~pp_sep:Format.pp_print_cut pp_protocol_level) + (Format.pp_print_list ~pp_sep:Format.pp_print_cut pp_proto_info) (Protocol_levels.bindings protocol_levels_data) (Format.pp_print_list ~pp_sep:Format.pp_print_cut Block_hash.pp) (Block_hash.Map.bindings invalid_blocks_data |> List.map fst) @@ -3062,6 +3196,87 @@ let make_pp_store (store : store) = pp_testchain_store ()) +let upgrade_protocol_levels ~chain_dir ~cleanups ~finalizers = + let open Lwt_result_syntax in + let cleanup ~tmp_protocol_levels_path ~protocol_levels_path = + let*! exists = Lwt_unix.file_exists tmp_protocol_levels_path in + if exists then Lwt_unix.rename tmp_protocol_levels_path protocol_levels_path + else Lwt.return_unit + in + let protocol_levels_path = + Naming.legacy_protocol_levels_file chain_dir |> Naming.encoded_file_path + in + let tmp_protocol_levels_path = protocol_levels_path ^ ".tmp" in + let*! () = cleanup ~tmp_protocol_levels_path ~protocol_levels_path in + let* legacy_protocol_levels_data = + Stored_data.load (Naming.legacy_protocol_levels_file chain_dir) + in + let*! legacy_protocol_levels = Stored_data.get legacy_protocol_levels_data in + let bindings = Protocol_levels.Legacy.bindings legacy_protocol_levels in + let*! protocol_levels = + List.fold_left_s + (fun map + ( level, + (legacy_activation_block : Protocol_levels.Legacy.activation_block) + ) -> + let protocol_info = + Protocol_levels. + { + protocol = legacy_activation_block.protocol; + activation_block = + { + block = legacy_activation_block.block; + commit_info = legacy_activation_block.commit_info; + }; + expect_predecessor_context = + false + (* the shell cannot have stored blocks that use the new semantics *); + } + in + Lwt.return (Protocol_levels.add level protocol_info map)) + Protocol_levels.empty + bindings + in + cleanups := + (fun () -> cleanup ~tmp_protocol_levels_path ~protocol_levels_path) + :: !cleanups ; + finalizers := + (fun () -> + let*! () = + Lwt_unix.rename protocol_levels_path tmp_protocol_levels_path + in + let*! _unit_error = + Stored_data.write_file + (Naming.protocol_levels_file chain_dir) + protocol_levels + in + let*! () = Lwt_unix.unlink tmp_protocol_levels_path in + Lwt.return_unit) + :: !finalizers ; + return_unit + +let v_3_0_upgrade ~store_dir genesis = + let open Lwt_result_syntax in + (* Hypothesis: all present blocks were validated with the + preexisting semantics *) + let*! () = Store_events.(emit upgrade_store_started ()) in + let cleanups : (unit -> unit Lwt.t) list ref = ref [] in + let finalizers : (unit -> unit Lwt.t) list ref = ref [] in + let chain_id = Chain_id.of_block_hash genesis.Genesis.block in + let chain_dir = + Naming.chain_dir (Naming.store_dir ~dir_path:store_dir) chain_id + in + protect + ~on_error:(fun err -> + let*! () = Store_events.(emit upgrade_store_failed) err in + let*! () = List.iter_s (fun f -> f ()) !cleanups in + Lwt.return_error err) + (fun () -> + let* () = upgrade_protocol_levels ~chain_dir ~cleanups ~finalizers in + let* () = Block_store.v_3_0_upgrade chain_dir ~cleanups ~finalizers in + let*! () = List.iter_s (fun f -> f ()) !finalizers in + return_unit) + (************ For testing and internal purposes only **************) module Unsafe = struct let repr_of_block b = b @@ -3098,8 +3313,8 @@ module Unsafe = struct let set_caboose chain_store new_caboose = Chain.unsafe_set_caboose chain_store new_caboose - let set_protocol_level chain_store ~protocol_level (b, ph) = - Chain.set_protocol_level chain_store ~protocol_level (b, ph) + let set_protocol_level chain_store ~protocol_level (b, ph, epc) = + Chain.set_protocol_level chain_store ~protocol_level (b, ph, epc) let load_testchain = Chain.load_testchain @@ -3134,7 +3349,8 @@ module Unsafe = struct let restore_from_snapshot ?(notify = fun () -> Lwt.return_unit) store_dir ~genesis ~genesis_context_hash ~floating_blocks_stream - ~new_head_with_metadata ~protocol_levels ~history_mode = + ~new_head_with_metadata ~new_head_resulting_context_hash ~protocol_levels + ~history_mode = let open Lwt_result_syntax in let chain_id = Chain_id.of_block_hash genesis.Genesis.block in let chain_dir = Naming.chain_dir store_dir chain_id in @@ -3223,24 +3439,31 @@ module Unsafe = struct Block_store.load chain_dir ~genesis_block ~readonly:false in (* Store the floating (in the correct order!) *) + (* No context under the head will be accessible: we do not need + to store consistent resulting_context_hash *) let*! () = Lwt_stream.iter_s (fun block -> - let*! _ = Block_store.store_block block_store block in + let*! _ = + Block_store.store_block block_store block Context_hash.zero + in notify ()) floating_blocks_stream in (* Store the head *) - let* () = Block_store.store_block block_store new_head_with_metadata in + let* () = + Block_store.store_block + block_store + new_head_with_metadata + new_head_resulting_context_hash + in (* Check correctness of protocol transition blocks *) let* () = List.iter_es - (fun ( _, - { - Protocol_levels.block = bh, _; - protocol; - commit_info = commit_info_opt; - } ) -> + (fun (_, {Protocol_levels.protocol; activation_block; _}) -> + let {Protocol_levels.block = bh, _; commit_info = commit_info_opt} = + activation_block + in let* block_opt = Block_store.read_block block_store diff --git a/src/lib_store/unix/store.mli b/src/lib_store/unix/store.mli index a0d780408158..aec83f579641 100644 --- a/src/lib_store/unix/store.mli +++ b/src/lib_store/unix/store.mli @@ -429,10 +429,7 @@ module Block : sig the newly created block is returned. If the block was successfully stored, then the block is removed - from the prechecked block cache. - - {b Warning} The store will refuse to store blocks with no - associated context's commit. *) + from the prechecked block cache. *) val store_block : chain_store -> block_header:Block_header.t -> @@ -450,8 +447,17 @@ module Block : sig operations:Operation.t trace trace -> unit tzresult Lwt.t - (** [context_exn chain_store block] checkouts the context of the - [block]. *) + (** [resulting_context_hash chain_store block] returns the resulting + context hash of the [block]. This context depends on the + [block]'s protocol associated semantics, i.e., it can either be + the one contained in its block header or the stored result of + its application. *) + val resulting_context_hash : + chain_store -> block -> Context_hash.t tzresult Lwt.t + + (** [context_exn chain_store block] checkouts the {b resulting} + context of the [block] which may differ from its block header's + one depending on the block's associated protocol semantics. *) val context_exn : chain_store -> block -> Tezos_protocol_environment.Context.t Lwt.t @@ -865,6 +871,11 @@ module Chain : sig (** {2 Chain's protocols} *) + (** [find_protocol_info chain_store ~protocol_level] returns the + protocol info associated to the given [protocol_level]. *) + val find_protocol_info : + t -> protocol_level:int -> Protocol_levels.protocol_info option Lwt.t + (** [find_activation_block chain_store ~protocol_level] returns the block that activated the protocol of level [protocol_level]. *) val find_activation_block : @@ -877,31 +888,41 @@ module Chain : sig val find_protocol : chain_store -> protocol_level:int -> Protocol_hash.t option Lwt.t + (** [expects_predecessor_context_hash chain_store ~protocol_level] + returns whether or not a protocol requires the context hash of a + block to target resulting context of it's predecessor. This + depends on the environment of each protocol.*) + val expect_predecessor_context_hash : + chain_store -> protocol_level:int -> bool tzresult Lwt.t + (** [all_protocol_levels chain_store] returns all the protocols registered in [chain_store]. *) val all_protocol_levels : - chain_store -> Protocol_levels.activation_block Protocol_levels.t Lwt.t + chain_store -> Protocol_levels.protocol_info Protocol_levels.t Lwt.t (** [may_update_protocol_level chain_store ?pred ?protocol_level - (block, ph)] updates the protocol level for the protocol [ph] in - [chain_store] with the activation [block]. If [pred] is not - provided, it reads the [block]'s predecessor and check that the - [block]'s protocol level is increasing compared to its - predecessor. If [protocol_level] is provided, we use this value - instead of the protocol level found in [block]. If a previous - entry is found, it overwrites it. *) + ~expect_predecessor_context (block, ph)] updates the protocol + level for the protocol [ph] in [chain_store] with the activation + [block]. If [pred] is not provided, it reads the [block]'s + predecessor and check that the [block]'s protocol level is + increasing compared to its predecessor. If [protocol_level] is + provided, we use this value instead of the protocol level found + in [block]. If a previous entry is found, it overwrites it. The + [expect_predecessor_context] argument specifies which context + hash semantics should be used. *) val may_update_protocol_level : chain_store -> ?pred:Block.block -> ?protocol_level:int -> + expect_predecessor_context:bool -> Block.block * Protocol_hash.t -> unit tzresult Lwt.t (** [may_update_ancestor_protocol_level chain_store ~head] tries to - find the activation block of the [head]'s protocol, checks that - its an ancestor and tries to update it if that's not the case. If - the registered activation block is not reachable (already - pruned), this function does nothing. *) + find the activation block of the [head]'s protocol, checks that + its an ancestor and tries to update it if that's not the + case. If the registered activation block is not reachable + (already pruned), this function does nothing. *) val may_update_ancestor_protocol_level : chain_store -> head:Block.block -> unit tzresult Lwt.t @@ -1004,6 +1025,12 @@ module Chain_traversal : sig (Block.t * Block.t list) Lwt.t end +(** Upgrade a v_2 to v_3 store by rewriting the block store and the + protocol level's table. + + {b Warning} Not backward-compatible. *) +val v_3_0_upgrade : store_dir:string -> Genesis.t -> unit tzresult Lwt.t + (**/**) (** Unsafe set of functions intended for internal store manipulation @@ -1045,13 +1072,14 @@ module Unsafe : sig [chain_store] without checks. *) val set_caboose : chain_store -> block_descriptor -> unit tzresult Lwt.t - (** [set_protocol_level chain_store protocol_level - (block, ph)] updates the protocol level for the protocol [ph] in - [chain_store] with the activation [block]. *) + (** [set_protocol_level chain_store protocol_level (block, ph, + expect_predecessor_context)] updates the protocol level for the + protocol [ph] in [chain_store] with the activation + [block] and specifies the [expect_predecessor_context] semantics. *) val set_protocol_level : chain_store -> protocol_level:int -> - Block.block * Protocol_hash.t -> + Block.block * Protocol_hash.t * bool -> unit tzresult Lwt.t (** Snapshots utility functions *) @@ -1075,9 +1103,10 @@ module Unsafe : sig (** [restore_from_snapshot ?notify ~store_dir ~context_index ~genesis ~genesis_context_hash ~floating_blocks_stream - ~new_head_with_metadata ~protocol_levels ~history_mode] - initialises a coherent store in [store_dir] with all the given - info retrieved from a snapshot. *) + ~new_head_with_metadata ~new_head_resulting_context_hash + ~protocol_levels ~history_mode] initialises a coherent store in + [store_dir] with all the given info retrieved from a + snapshot. *) val restore_from_snapshot : ?notify:(unit -> unit Lwt.t) -> [`Store_dir] Naming.directory -> @@ -1085,7 +1114,8 @@ module Unsafe : sig genesis_context_hash:Context_hash.t -> floating_blocks_stream:Block_repr.block Lwt_stream.t -> new_head_with_metadata:Block_repr.block -> - protocol_levels:Protocol_levels.activation_block Protocol_levels.t -> + new_head_resulting_context_hash:Context_hash.t -> + protocol_levels:Protocol_levels.protocol_info Protocol_levels.t -> history_mode:History_mode.t -> unit tzresult Lwt.t end diff --git a/src/lib_store/unix/test/alpha_utils.ml b/src/lib_store/unix/test/alpha_utils.ml index f058b708b63c..659d3f8ac36c 100644 --- a/src/lib_store/unix/test/alpha_utils.ml +++ b/src/lib_store/unix/test/alpha_utils.ml @@ -666,6 +666,7 @@ let apply_and_store chain_store ?(synchronous_merge = true) ?policy }; block_metadata = (block_header_metadata, block_metadata_hash); ops_metadata; + shell_header_hash = Block_validation.Shell_header_hash.zero; } in let operations = diff --git a/src/lib_store/unix/test/test_block_store.ml b/src/lib_store/unix/test/test_block_store.ml index 6326002094c5..22a2234fde94 100644 --- a/src/lib_store/unix/test/test_block_store.ml +++ b/src/lib_store/unix/test/test_block_store.ml @@ -142,7 +142,11 @@ let test_storing_and_access_predecessors block_store = let*! blocks, _head = make_raw_block_list ~kind:`Full (genesis_hash, -1l) 50 in - let* () = List.iter_es (Block_store.store_block block_store) blocks in + let* () = + List.iter_es + (fun b -> Block_store.store_block block_store b Context_hash.zero) + blocks + in let* () = assert_presence_in_block_store block_store blocks in let* () = List.iter_es @@ -231,7 +235,11 @@ let test_simple_merge block_store = Block_repr.metadata head |> WithExceptions.Option.get ~loc:__LOC__ in let all_blocks = List.concat cycles in - let* () = List.iter_es (Block_store.store_block block_store) all_blocks in + let* () = + List.iter_es + (fun b -> Block_store.store_block block_store b Context_hash.zero) + all_blocks + in let* () = Block_store.merge_stores block_store @@ -260,7 +268,11 @@ let test_consecutive_concurrent_merges block_store = Block_repr.metadata head |> WithExceptions.Option.get ~loc:__LOC__ in let all_blocks = List.concat cycles in - let* () = List.iter_es (Block_store.store_block block_store) all_blocks in + let* () = + List.iter_es + (fun b -> Block_store.store_block block_store b Context_hash.zero) + all_blocks + in let cycles_to_merge = List.fold_left (fun (acc, pred_cycle_lafl) cycle -> @@ -321,7 +333,11 @@ let test_ten_cycles_merge block_store = ~nb_cycles:10 in let all_blocks = List.concat cycles in - let* () = List.iter_es (Block_store.store_block block_store) all_blocks in + let* () = + List.iter_es + (fun b -> Block_store.store_block block_store b Context_hash.zero) + all_blocks + in let* () = Block_store.merge_stores block_store @@ -352,7 +368,11 @@ let test_merge_with_branches block_store = make_n_initial_consecutive_cycles block_store ~cycle_length:100 ~nb_cycles:2 in let all_blocks = List.concat cycles in - let* () = List.iter_es (Block_store.store_block block_store) all_blocks in + let* () = + List.iter_es + (fun b -> Block_store.store_block block_store b Context_hash.zero) + all_blocks + in let branches_fork_points_to_gc = [20; 40; 60; 80; 98] in (* 448 => we also keep the checkpoint *) let* blocks_to_gc = @@ -378,7 +398,11 @@ let test_merge_with_branches block_store = {metadata with Block_repr.last_allowed_fork_level = 99l}) block.metadata) blocks ; - let* () = List.iter_es (Block_store.store_block block_store) blocks in + let* () = + List.iter_es + (fun b -> Block_store.store_block block_store b Context_hash.zero) + blocks + in return blocks) branches_fork_points_to_gc in @@ -406,7 +430,11 @@ let test_merge_with_branches block_store = {metadata with Block_repr.last_allowed_fork_level = 199l}) block.metadata) blocks ; - let* () = List.iter_es (Block_store.store_block block_store) blocks in + let* () = + List.iter_es + (fun b -> Block_store.store_block block_store b Context_hash.zero) + blocks + in return blocks) branches_fork_points_to_keep in @@ -439,7 +467,11 @@ let perform_n_cycles_merge ?(cycle_length = 10) block_store history_mode make_n_initial_consecutive_cycles block_store ~cycle_length ~nb_cycles in let all_blocks = List.concat cycles in - let* () = List.iter_es (Block_store.store_block block_store) all_blocks in + let* () = + List.iter_es + (fun b -> Block_store.store_block block_store b Context_hash.zero) + all_blocks + in let* () = assert_presence_in_block_store ~with_metadata:true block_store all_blocks in diff --git a/src/lib_store/unix/test/test_consistency.ml b/src/lib_store/unix/test/test_consistency.ml index 11134830e4de..c62869c3aac8 100644 --- a/src/lib_store/unix/test/test_consistency.ml +++ b/src/lib_store/unix/test/test_consistency.ml @@ -83,6 +83,7 @@ let init_protocols store history_mode = chain_store ~pred ~protocol_level + ~expect_predecessor_context:true (first_block_of_cycle, proto_hash) in let*! () = @@ -96,7 +97,7 @@ let init_protocols store history_mode = let*! _savepoint, savepoint_level = Store.Chain.savepoint chain_store in let* () = Protocol_levels.iter_es - (fun proto_level activation_block -> + (fun proto_level {Protocol_levels.activation_block; _} -> let activation_block_hash, activation_block_level = activation_block.Protocol_levels.block in @@ -200,7 +201,7 @@ let check_protocol_levels_availability chain_store ~expected_protocols in match recovered_activation_block with | None -> assert false - | Some {Protocol_levels.commit_info; _} -> + | Some {Protocol_levels.activation_block = {commit_info; _}; _} -> assert (Option.is_some commit_info) else assert false ; return_unit) diff --git a/src/lib_store/unix/test/test_locator.ml b/src/lib_store/unix/test/test_locator.ml index 3ca1629aa95b..765ece5f7e67 100644 --- a/src/lib_store/unix/test/test_locator.ml +++ b/src/lib_store/unix/test/test_locator.ml @@ -126,6 +126,7 @@ let make_empty_chain chain_store n : Block_hash.t Lwt.t = }; block_metadata = (zero, None); ops_metadata = Block_validation.No_metadata_hash []; + shell_header_hash = Block_validation.Shell_header_hash.zero; } in let* _ = @@ -203,6 +204,7 @@ let make_multiple_protocol_chain (chain_store : Store.Chain.t) {empty_result with last_allowed_fork_level}; block_metadata = (zero, block_metadata_hash); ops_metadata = Block_validation.No_metadata_hash []; + shell_header_hash = Block_validation.Shell_header_hash.zero; } in let* o = @@ -222,6 +224,7 @@ let make_multiple_protocol_chain (chain_store : Store.Chain.t) Store.Chain.may_update_protocol_level chain_store ~protocol_level:proto_level + ~expect_predecessor_context:true (b, genesis_protocol) in loop remaining_fork_points (lvl + 1) header diff --git a/src/lib_store/unix/test/test_utils.ml b/src/lib_store/unix/test/test_utils.ml index b3b08da6d56a..9234cf2291dd 100644 --- a/src/lib_store/unix/test/test_utils.ml +++ b/src/lib_store/unix/test/test_utils.ml @@ -473,6 +473,7 @@ let store_raw_block chain_store (raw_block : Block_repr.t) = ( Block_repr.block_metadata metadata, Block_repr.block_metadata_hash raw_block ); ops_metadata; + shell_header_hash = Block_validation.Shell_header_hash.zero; } in let* r = -- GitLab From 46a0e2de0e044a4ea5e60f8cd2e4d9a4431127a1 Mon Sep 17 00:00:00 2001 From: vbot Date: Thu, 17 Nov 2022 08:10:41 +0100 Subject: [PATCH 05/19] Store: remove commit info from protocol table --- src/lib_context/disk/context.ml | 92 ----------- src/lib_context/disk/context.mli | 1 - src/lib_context/memory/context.ml | 85 ---------- src/lib_context/sigs/context.ml | 29 ---- .../context_ops/context_ops.ml | 6 - src/lib_store/mocked/store.ml | 79 ++------- src/lib_store/shared/store_types.ml | 151 +++++++----------- src/lib_store/shared/store_types.mli | 104 +++--------- src/lib_store/store.mli | 4 +- src/lib_store/unix/consistency.ml | 17 +- src/lib_store/unix/snapshots.ml | 14 +- src/lib_store/unix/store.ml | 132 ++------------- src/lib_store/unix/store.mli | 4 +- src/lib_store/unix/test/test_consistency.ml | 20 +-- 14 files changed, 126 insertions(+), 612 deletions(-) diff --git a/src/lib_context/disk/context.ml b/src/lib_context/disk/context.ml index 6cd41bd65867..bc4c72d53899 100644 --- a/src/lib_context/disk/context.ml +++ b/src/lib_context/disk/context.ml @@ -34,7 +34,6 @@ module Proof = Tezos_context_sigs.Context.Proof_types type error += | Cannot_create_file of string | Cannot_open_file of string - | Cannot_retrieve_commit_info of Tezos_crypto.Context_hash.t | Cannot_find_protocol | Suspicious_file of int @@ -65,20 +64,6 @@ let () = Data_encoding.(obj1 (req "context_restore_cannot_open" string)) (function Cannot_open_file e -> Some e | _ -> None) (fun e -> Cannot_open_file e) ; - register_error_kind - `Permanent - ~id:"cannot_retrieve_commit_info" - ~title:"Cannot retrieve commit info" - ~description:"" - ~pp:(fun ppf hash -> - Format.fprintf - ppf - "@[Cannot retrieve commit info associated to context hash %a@]" - Tezos_crypto.Context_hash.pp - hash) - Data_encoding.(obj1 (req "context_hash" Tezos_crypto.Context_hash.encoding)) - (function Cannot_retrieve_commit_info e -> Some e | _ -> None) - (fun e -> Cannot_retrieve_commit_info e) ; register_error_kind `Permanent ~id:"context_dump.cannot_find_protocol" @@ -106,7 +91,6 @@ module type TEZOS_CONTEXT_UNIX = sig type error += | Cannot_create_file of string | Cannot_open_file of string - | Cannot_retrieve_commit_info of Tezos_crypto.Context_hash.t | Cannot_find_protocol | Suspicious_file of int @@ -229,7 +213,6 @@ module Make (Encoding : module type of Tezos_context_encoding.Context) = struct type error += | Cannot_create_file = Cannot_create_file | Cannot_open_file = Cannot_open_file - | Cannot_retrieve_commit_info = Cannot_retrieve_commit_info | Cannot_find_protocol = Cannot_find_protocol | Suspicious_file = Suspicious_file @@ -285,8 +268,6 @@ module Make (Encoding : module type of Tezos_context_encoding.Context) = struct let current_test_chain_key = ["test_chain"] - let current_data_key = ["data"] - let current_predecessor_block_metadata_hash_key = ["predecessor_block_metadata_hash"] @@ -1146,79 +1127,6 @@ module Make (Encoding : module type of Tezos_context_encoding.Context) = struct module Block_header = Block_header end - (* Protocol data *) - - let data_node_hash context = - let open Lwt_syntax in - let+ tree = Store.Tree.get_tree context.tree current_data_key in - Hash.to_context_hash (Store.Tree.hash tree) - - let retrieve_commit_info index block_header = - let open Lwt_result_syntax in - let context_hash = block_header.Block_header.shell.context in - let* context = - let*! r = checkout index context_hash in - match r with - | Some c -> return c - | None -> tzfail (Cannot_retrieve_commit_info context_hash) - in - let irmin_info = Dumpable_context.context_info context in - let author = Info.author irmin_info in - let message = Info.message irmin_info in - let timestamp = Time.Protocol.of_seconds (Info.date irmin_info) in - let*! protocol_hash = get_protocol context in - let*! test_chain_status = get_test_chain context in - let*! predecessor_block_metadata_hash = - find_predecessor_block_metadata_hash context - in - let*! predecessor_ops_metadata_hash = - find_predecessor_ops_metadata_hash context - in - let*! data_key = data_node_hash context in - let parents_contexts = Dumpable_context.context_parents context in - return - ( protocol_hash, - author, - message, - timestamp, - test_chain_status, - data_key, - predecessor_block_metadata_hash, - predecessor_ops_metadata_hash, - parents_contexts ) - - let check_protocol_commit_consistency ~expected_context_hash - ~given_protocol_hash ~author ~message ~timestamp ~test_chain_status - ~predecessor_block_metadata_hash ~predecessor_ops_metadata_hash - ~data_merkle_root ~parents_contexts = - let open Lwt_syntax in - let data_merkle_root = Hash.of_context_hash data_merkle_root in - let parents = List.map Hash.of_context_hash parents_contexts in - let info = Info.v ~author (Time.Protocol.to_seconds timestamp) ~message in - let tree = Store.Tree.empty () in - let* tree = Root_tree.add_test_chain tree test_chain_status in - let* tree = Root_tree.add_protocol tree given_protocol_hash in - let* tree = - Option.fold - predecessor_block_metadata_hash - ~none:(Lwt.return tree) - ~some:(Root_tree.add_predecessor_block_metadata_hash tree) - in - let* tree = - Option.fold - predecessor_ops_metadata_hash - ~none:(Lwt.return tree) - ~some:(Root_tree.add_predecessor_ops_metadata_hash tree) - in - let data_t = Store.Tree.pruned (`Node data_merkle_root) in - let+ new_tree = Store.Tree.add_tree tree current_data_key data_t in - let node = Store.Tree.hash new_tree in - let ctxt_h = - P.Commit_portable.v ~info ~parents ~node - |> Commit_hash.hash |> Hash.to_context_hash - in - Tezos_crypto.Context_hash.equal ctxt_h expected_context_hash - (* Context dumper *) open Tezos_context_dump diff --git a/src/lib_context/disk/context.mli b/src/lib_context/disk/context.mli index 6a6368c96c07..fb4e531b64bd 100644 --- a/src/lib_context/disk/context.mli +++ b/src/lib_context/disk/context.mli @@ -30,7 +30,6 @@ module type TEZOS_CONTEXT_UNIX = sig type error += | Cannot_create_file of string | Cannot_open_file of string - | Cannot_retrieve_commit_info of Tezos_crypto.Context_hash.t | Cannot_find_protocol | Suspicious_file of int diff --git a/src/lib_context/memory/context.ml b/src/lib_context/memory/context.ml index 3e78b9860c80..edc165d944d5 100644 --- a/src/lib_context/memory/context.ml +++ b/src/lib_context/memory/context.ml @@ -109,8 +109,6 @@ module Make (Encoding : module type of Tezos_context_encoding.Context) = struct let current_test_chain_key = ["test_chain"] - let current_data_key = Tezos_context_sigs.Context.current_data_key - let current_predecessor_block_metadata_hash_key = ["predecessor_block_metadata_hash"] @@ -658,87 +656,4 @@ module Make (Encoding : module type of Tezos_context_encoding.Context) = struct match o with | None -> assert false | Some commit -> Store.Branch.set index.repo Store.Branch.main commit - - module Dumpable_context = struct - let context_parents ctxt = - match ctxt with - | {parents = [commit]; _} -> - let parents = Store.Commit.parents commit in - let parents = - List.map - (fun k -> P.Commit.Key.to_hash k |> Hash.to_context_hash) - parents - in - List.sort Tezos_crypto.Context_hash.compare parents - | _ -> assert false - - let context_info = function - | {parents = [c]; _} -> Store.Commit.info c - | _ -> assert false - end - - let data_node_hash context = - let open Lwt_syntax in - let+ tree = Store.Tree.get_tree context.tree current_data_key in - Hash.to_context_hash (Store.Tree.hash tree) - - let retrieve_commit_info index block_header = - let open Lwt_syntax in - let* context = checkout_exn index block_header.Block_header.shell.context in - let irmin_info = Dumpable_context.context_info context in - let author = Info.author irmin_info in - let message = Info.message irmin_info in - let timestamp = Time.Protocol.of_seconds (Info.date irmin_info) in - let* protocol_hash = get_protocol context in - let* test_chain_status = get_test_chain context in - let* predecessor_block_metadata_hash = - find_predecessor_block_metadata_hash context - in - let* predecessor_ops_metadata_hash = - find_predecessor_ops_metadata_hash context - in - let* data_key = data_node_hash context in - let parents_contexts = Dumpable_context.context_parents context in - return_ok - ( protocol_hash, - author, - message, - timestamp, - test_chain_status, - data_key, - predecessor_block_metadata_hash, - predecessor_ops_metadata_hash, - parents_contexts ) - - let check_protocol_commit_consistency ~expected_context_hash - ~given_protocol_hash ~author ~message ~timestamp ~test_chain_status - ~predecessor_block_metadata_hash ~predecessor_ops_metadata_hash - ~data_merkle_root ~parents_contexts = - let open Lwt_syntax in - let data_merkle_root = Hash.of_context_hash data_merkle_root in - let parents = List.map Hash.of_context_hash parents_contexts in - let info = Info.v ~author (Time.Protocol.to_seconds timestamp) ~message in - let tree = Store.Tree.empty () in - let* tree = Root_tree.add_test_chain tree test_chain_status in - let* tree = Root_tree.add_protocol tree given_protocol_hash in - let* tree = - Option.fold - predecessor_block_metadata_hash - ~none:(Lwt.return tree) - ~some:(Root_tree.add_predecessor_block_metadata_hash tree) - in - let* tree = - Option.fold - predecessor_ops_metadata_hash - ~none:(Lwt.return tree) - ~some:(Root_tree.add_predecessor_ops_metadata_hash tree) - in - let data_t = Store.Tree.pruned (`Node data_merkle_root) in - let+ new_tree = Store.Tree.add_tree tree current_data_key data_t in - let node = Store.Tree.hash new_tree in - let ctxt_h = - P.Commit_portable.v ~info ~parents ~node - |> Commit_hash.hash |> Hash.to_context_hash - in - Tezos_crypto.Context_hash.equal ctxt_h expected_context_hash end diff --git a/src/lib_context/sigs/context.ml b/src/lib_context/sigs/context.ml index 8bd5c0800f28..cbf87f90b815 100644 --- a/src/lib_context/sigs/context.ml +++ b/src/lib_context/sigs/context.ml @@ -806,35 +806,6 @@ module type TEZOS_CONTEXT = sig val add_predecessor_ops_metadata_hash : context -> Tezos_crypto.Operation_metadata_list_list_hash.t -> context Lwt.t - - val retrieve_commit_info : - index -> - Block_header.t -> - (Tezos_crypto.Protocol_hash.t - * string - * string - * Time.Protocol.t - * Test_chain_status.t - * Tezos_crypto.Context_hash.t - * Tezos_crypto.Block_metadata_hash.t option - * Tezos_crypto.Operation_metadata_list_list_hash.t option - * Tezos_crypto.Context_hash.t list) - tzresult - Lwt.t - - val check_protocol_commit_consistency : - expected_context_hash:Tezos_crypto.Context_hash.t -> - given_protocol_hash:Tezos_crypto.Protocol_hash.t -> - author:string -> - message:string -> - timestamp:Time.Protocol.t -> - test_chain_status:Test_chain_status.t -> - predecessor_block_metadata_hash:Tezos_crypto.Block_metadata_hash.t option -> - predecessor_ops_metadata_hash: - Tezos_crypto.Operation_metadata_list_list_hash.t option -> - data_merkle_root:Tezos_crypto.Context_hash.t -> - parents_contexts:Tezos_crypto.Context_hash.t list -> - bool Lwt.t end (** Functor `With_get_data` adds a `get_data` function to modules of signature `S`. diff --git a/src/lib_protocol_environment/context_ops/context_ops.ml b/src/lib_protocol_environment/context_ops/context_ops.ml index 56cca8e1a09b..8110558a2033 100644 --- a/src/lib_protocol_environment/context_ops/context_ops.ml +++ b/src/lib_protocol_environment/context_ops/context_ops.ml @@ -280,12 +280,6 @@ let exists context_index context_hash = | Disk_index index -> Context.exists index context_hash | Memory_index index -> Tezos_context_memory.Context.exists index context_hash -let retrieve_commit_info context_index header = - match context_index with - | Disk_index index -> Context.retrieve_commit_info index header - | Memory_index index -> - Tezos_context_memory.Context.retrieve_commit_info index header - let close context_index = match context_index with | Disk_index index -> Context.close index diff --git a/src/lib_store/mocked/store.ml b/src/lib_store/mocked/store.ml index 8f1ede5f34b8..a5e07f9c3daf 100644 --- a/src/lib_store/mocked/store.ml +++ b/src/lib_store/mocked/store.ml @@ -1110,8 +1110,8 @@ module Chain = struct in match Protocol_levels.find proto_level protocol_levels with | None -> Lwt.return_none - | Some {activation_block = {block; _}; _} -> ( - let block_activation_level = snd block in + | Some {activation_block; _} -> ( + let block_activation_level = snd activation_block in (* proto level's lower bound found, now retrieving the upper bound *) let head_proto_level = Block_repr.proto_level chain_state.current_head @@ -1127,8 +1127,8 @@ module Chain = struct Protocol_levels.find (succ proto_level) protocol_levels with | None -> Lwt.return_none - | Some {activation_block = {block; _}; _} -> ( - let next_activation_level = snd block in + | Some {activation_block; _} -> ( + let next_activation_level = snd activation_block in let last_level_in_protocol = Int32.(pred next_activation_level) in @@ -1613,8 +1613,7 @@ module Chain = struct (* Create / Load / Close *) - let create_chain_state ?target ~genesis_block ~genesis_protocol - ~genesis_commit_info chain_dir = + let create_chain_state ?target ~genesis_block ~genesis_protocol chain_dir = let open Lwt_result_syntax in let genesis_proto_level = Block_repr.proto_level genesis_block in let ((_, genesis_level) as genesis_descr) = @@ -1641,8 +1640,7 @@ module Chain = struct genesis_proto_level { protocol = genesis_protocol; - activation_block = - {block = genesis_descr; commit_info = genesis_commit_info}; + activation_block = genesis_descr; expect_predecessor_context; } empty) @@ -1704,24 +1702,6 @@ module Chain = struct prechecked_blocks; } - let get_commit_info index header = - let open Lwt_result_syntax in - protect - ~on_error:(fun err -> - Format.kasprintf - (fun e -> tzfail (Missing_commit_info e)) - "%a" - Error_monad.pp_print_trace - err) - (fun () -> - let* tup = Context_ops.retrieve_commit_info index header in - return (Protocol_levels.commit_info_of_tuple tup)) - - let get_commit_info_opt index header = - let open Lwt_syntax in - let* r = get_commit_info index header in - match r with Ok v -> Lwt.return_some v | Error _ -> Lwt.return_none - let create_chain_store ?block_cache_limit global_store chain_dir ?target ~chain_id ?(expiration = None) ?genesis_block ~genesis ~genesis_context history_mode = @@ -1740,18 +1720,12 @@ module Chain = struct let* () = Stored_data.write_file (Naming.chain_config_file chain_dir) chain_config in - let*! genesis_commit_info = - get_commit_info_opt - global_store.context_index - (Block.header genesis_block) - in let* chain_state = create_chain_state chain_dir ?target ~genesis_block ~genesis_protocol:genesis.Genesis.protocol - ~genesis_commit_info in let* genesis_block_data = Stored_data.init @@ -1806,22 +1780,10 @@ module Chain = struct let expect_predecessor_context_hash chain_store ~protocol_level = expect_predecessor_context_hash chain_store ~protocol_level - let compute_commit_info chain_store block = - let open Lwt_result_syntax in - let index = chain_store.global_store.context_index in - protect - ~on_error:(fun _ -> return_none) - (fun () -> - let* commit_info = get_commit_info index block in - return_some commit_info) - let set_protocol_level chain_store ~protocol_level (block, protocol_hash, expect_predecessor_context) = let open Lwt_result_syntax in Shared.locked_use chain_store.chain_state (fun {protocol_levels_data; _} -> - let* commit_info_opt = - compute_commit_info chain_store (Block.header block) - in let* () = Stored_data.update_with protocol_levels_data (fun protocol_levels -> Lwt.return @@ -1830,11 +1792,7 @@ module Chain = struct protocol_level { protocol = protocol_hash; - activation_block = - { - block = Block.descriptor block; - commit_info = commit_info_opt; - }; + activation_block = Block.descriptor block; expect_predecessor_context; } protocol_levels)) @@ -1882,7 +1840,7 @@ module Chain = struct if Compare.Int.(prev_proto_level < protocol_level) then let*! o = find_activation_block chain_store ~protocol_level in match o with - | Some {block = bh, _; _} -> + | Some (bh, _) -> if Block_hash.(bh <> Block.hash block) then set_protocol_level chain_store @@ -1933,13 +1891,11 @@ module Chain = struct ~protocol_level (activation_block, activated_protocol, expected_predecessor_context) | Some - { - Protocol_levels.protocol; - activation_block = {block; _}; - expect_predecessor_context; - } -> ( + {Protocol_levels.protocol; activation_block; expect_predecessor_context} + -> ( let*! _, savepoint_level = savepoint chain_store in - if Compare.Int32.(savepoint_level > snd block) then + let activation_block_level = snd activation_block in + if Compare.Int32.(savepoint_level > activation_block_level) then (* the block is too far in the past *) return_unit else @@ -1947,13 +1903,13 @@ module Chain = struct is_ancestor chain_store ~head:(Block.descriptor head) - ~ancestor:block + ~ancestor:activation_block in match b with | true -> (* nothing to do *) return_unit | false -> ( let distance = - Int32.(sub (Block.level head) (snd block) |> to_int) + Int32.(sub (Block.level head) activation_block_level |> to_int) in let*! o = Block.read_block_opt chain_store ~distance (Block.hash head) @@ -2269,17 +2225,14 @@ let rec make_pp_chain_store (chain_store : chain_store) = in let pp_proto_info fmt (proto_level, {Protocol_levels.protocol; activation_block; _}) = - let Protocol_levels.{block; commit_info} = activation_block in Format.fprintf fmt - "proto level: %d, transition block: %a, protocol: %a, commit info: %a" + "proto level: %d, transition block: %a, protocol: %a" proto_level pp_block_descriptor - block + activation_block Protocol_hash.pp protocol - (option_pp ~default:"n/a" (fun fmt _ -> Format.fprintf fmt "available")) - commit_info in let make_pp_test_chain_opt = function | None -> Lwt.return (fun fmt () -> Format.fprintf fmt "n/a") diff --git a/src/lib_store/shared/store_types.ml b/src/lib_store/shared/store_types.ml index 4aa865d01f22..0df400553469 100644 --- a/src/lib_store/shared/store_types.ml +++ b/src/lib_store/shared/store_types.ml @@ -70,99 +70,9 @@ module Protocol_levels = struct let compare = Compare.Int.compare end) - type commit_info = { - author : string; - message : string; - test_chain_status : Test_chain_status.t; - predecessor_block_metadata_hash : Block_metadata_hash.t option; - predecessor_ops_metadata_hash : Operation_metadata_list_list_hash.t option; - data_merkle_root : Context_hash.t; - parents_contexts : Context_hash.t list; - } - - let commit_info_of_tuple - ( _protocol_hash, - author, - message, - _timestamp, - test_chain_status, - data_merkle_root, - predecessor_block_metadata_hash, - predecessor_ops_metadata_hash, - parents_contexts ) = - { - author; - message; - test_chain_status; - data_merkle_root; - predecessor_block_metadata_hash; - predecessor_ops_metadata_hash; - parents_contexts; - } - - let commit_info_encoding = - let open Data_encoding in - conv - (fun { - author; - message; - test_chain_status; - data_merkle_root; - predecessor_block_metadata_hash; - predecessor_ops_metadata_hash; - parents_contexts; - } -> - ( author, - message, - test_chain_status, - data_merkle_root, - predecessor_block_metadata_hash, - predecessor_ops_metadata_hash, - parents_contexts )) - (fun ( author, - message, - test_chain_status, - data_merkle_root, - predecessor_block_metadata_hash, - predecessor_ops_metadata_hash, - parents_contexts ) -> - { - author; - message; - test_chain_status; - data_merkle_root; - predecessor_block_metadata_hash; - predecessor_ops_metadata_hash; - parents_contexts; - }) - (obj7 - (req "author" string) - (req "message" string) - (req "test_chain_status" Test_chain_status.encoding) - (req "data_merkle_root" Context_hash.encoding) - (opt "predecessor_block_metadata_hash" Block_metadata_hash.encoding) - (opt - "predecessor_ops_metadata_hash" - Operation_metadata_list_list_hash.encoding) - (req "parents_contexts" (list Context_hash.encoding))) - - type activation_block = { - block : block_descriptor; - commit_info : commit_info option; - } - - let activation_block_encoding = - let open Data_encoding in - conv - (fun {block; commit_info} -> (block, commit_info)) - (fun (block, commit_info) -> {block; commit_info}) - (obj2 - (req "block" block_descriptor_encoding) - (opt "commit_info" commit_info_encoding)) - type protocol_info = { protocol : Protocol_hash.t; - activation_block : activation_block; + activation_block : block_descriptor; expect_predecessor_context : bool; } @@ -175,7 +85,7 @@ module Protocol_levels = struct {protocol; activation_block; expect_predecessor_context}) (obj3 (req "protocol" Protocol_hash.encoding) - (req "activation_block" activation_block_encoding) + (req "activation_block" block_descriptor_encoding) (req "expect_predecessor_context" bool)) let encoding = @@ -186,6 +96,63 @@ module Protocol_levels = struct Data_encoding.(list (tup2 uint8 protocol_info_encoding)) module Legacy = struct + type commit_info = { + author : string; + message : string; + test_chain_status : Test_chain_status.t; + predecessor_block_metadata_hash : Block_metadata_hash.t option; + predecessor_ops_metadata_hash : + Operation_metadata_list_list_hash.t option; + data_merkle_root : Context_hash.t; + parents_contexts : Context_hash.t list; + } + + let commit_info_encoding = + let open Data_encoding in + conv + (fun { + author; + message; + test_chain_status; + data_merkle_root; + predecessor_block_metadata_hash; + predecessor_ops_metadata_hash; + parents_contexts; + } -> + ( author, + message, + test_chain_status, + data_merkle_root, + predecessor_block_metadata_hash, + predecessor_ops_metadata_hash, + parents_contexts )) + (fun ( author, + message, + test_chain_status, + data_merkle_root, + predecessor_block_metadata_hash, + predecessor_ops_metadata_hash, + parents_contexts ) -> + { + author; + message; + test_chain_status; + data_merkle_root; + predecessor_block_metadata_hash; + predecessor_ops_metadata_hash; + parents_contexts; + }) + (obj7 + (req "author" string) + (req "message" string) + (req "test_chain_status" Test_chain_status.encoding) + (req "data_merkle_root" Context_hash.encoding) + (opt "predecessor_block_metadata_hash" Block_metadata_hash.encoding) + (opt + "predecessor_ops_metadata_hash" + Operation_metadata_list_list_hash.encoding) + (req "parents_contexts" (list Context_hash.encoding))) + type activation_block = { block : block_descriptor; protocol : Protocol_hash.t; diff --git a/src/lib_store/shared/store_types.mli b/src/lib_store/shared/store_types.mli index f3725b7e8baa..05d5f5304f5a 100644 --- a/src/lib_store/shared/store_types.mli +++ b/src/lib_store/shared/store_types.mli @@ -63,97 +63,43 @@ module Block_lru_cache : Aches_lwt.Lache.MAP_OPTION with type key = Block_hash.t module Protocol_levels : sig include Map.S with type key = int - (** The type representing a subset of the commit information. These - are used to easily check that a given [Context_hash.t], with the - associated context not present on disk, is consistent. It is - used to verify that an announced protocol is indeed the one that - was committed on disk. Fields are: - - [author] is the commit's author; - - [message] is the commit's message; - - [test_chain_status] is the status of the test chain at commit - time; - - [data_merkle_root] is the merkle root of the context's data - main node; - - [parents_contexts] are the context hashes of this commit's - parents. - - This structure should be populated with the result of - {!Tezos_context.Context.retrieve_commit_info}. The consistency - check is done by - {!Tezos_context.Context.check_protocol_commit_consistency} when - a context in imported in the leger state, for example, when - importing a snapshot. *) - type commit_info = { - author : string; - message : string; - test_chain_status : Test_chain_status.t; - predecessor_block_metadata_hash : Block_metadata_hash.t option; - predecessor_ops_metadata_hash : Operation_metadata_list_list_hash.t option; - data_merkle_root : Context_hash.t; - parents_contexts : Context_hash.t list; - } - - val commit_info_of_tuple : - Protocol_hash.t - * string - * string - * Time.Protocol.t - * Test_chain_status.t - * Context_hash.t - * Block_metadata_hash.t option - * Operation_metadata_list_list_hash.t option - * Context_hash.t list -> - commit_info - - (** Encoding for {!commit_info}. *) - val commit_info_encoding : commit_info Data_encoding.t - - (** The type for activation blocks. - - An activation block refers to a block which activated a new - protocol [N+1]. Thus, the block header related to that activation - block will have a protocol level which differs from its - predecessor. As this block aims to activate a new protocol, the - [next_protocol] and [protocol] fields from the metadata differs: - [next_protocol] refers to the hash of the activated protocol. As - a consequence, this block handled by protocol [N] and is the last - block of that protocol. - - - {b WARNING.} Commit information are optional to allow - retro-compatibility: the LMDB legacy store does not contain such - information. Thus, populating the protocol levels' map while - upgrading the storage would prevent us from storing an activation - block which is used to retrieve the [protocol] to load, and - therefore, being unable to decode stored blocks. In the future, - when a sufficient number of nodes have fully migrated, we can - stitch the missing commit information by hard-coding them, - allowing us to remove the option. *) - type activation_block = { - block : block_descriptor; - commit_info : commit_info option; - } - (** The type for protocol info. [expect_predecessor_context] is a flag which reflects what is - referenced by the context hash of a block, depending on its - protocol. If the flag is true, the block contains the - predecessors context (context before the application of the - block). Otherwise, it contains the resulting context hash of the - application of the block. *) + referenced by the context hash of a block, depending on its + protocol. If the flag is true, the block contains the predecessors + context (context before the application of the block). Otherwise, + it contains the resulting context hash of the application of the + block. An activation block refers to a block which activated a new + protocol [N+1]. Thus, the block header related to that activation + block will have a protocol level which differs from its + predecessor. As this block aims to activate a new protocol, the + [next_protocol] and [protocol] fields from the metadata differs: + [next_protocol] refers to the hash of the activated protocol. As a + consequence, this block handled by protocol [N] and is the last + block of that protocol. +*) type protocol_info = { protocol : Protocol_hash.t; - activation_block : activation_block; + activation_block : block_descriptor; expect_predecessor_context : bool; } - val activation_block_encoding : activation_block Data_encoding.t - (** Encoding for the protocol level's association map. *) val encoding : protocol_info t Data_encoding.t module Legacy : sig + type commit_info = { + author : string; + message : string; + test_chain_status : Test_chain_status.t; + predecessor_block_metadata_hash : Block_metadata_hash.t option; + predecessor_ops_metadata_hash : + Operation_metadata_list_list_hash.t option; + data_merkle_root : Context_hash.t; + parents_contexts : Context_hash.t list; + } + type activation_block = { block : block_descriptor; protocol : Protocol_hash.t; diff --git a/src/lib_store/store.mli b/src/lib_store/store.mli index b681ec755464..893fd5b6771c 100644 --- a/src/lib_store/store.mli +++ b/src/lib_store/store.mli @@ -878,9 +878,7 @@ module Chain : sig (** [find_activation_block chain_store ~protocol_level] returns the block that activated the protocol of level [protocol_level]. *) val find_activation_block : - chain_store -> - protocol_level:int -> - Protocol_levels.activation_block option Lwt.t + chain_store -> protocol_level:int -> block_descriptor option Lwt.t (** [find_protocol chain_store ~protocol_level] returns the protocol with the level [protocol_level]. *) diff --git a/src/lib_store/unix/consistency.ml b/src/lib_store/unix/consistency.ml index d82083716e14..a394e90d472c 100644 --- a/src/lib_store/unix/consistency.ml +++ b/src/lib_store/unix/consistency.ml @@ -886,20 +886,6 @@ let find_activation_blocks_in_floating block_store ~head ~savepoint_level in loop (Block_repr.proto_level head) head head -let craft_activation_block context_index block = - let open Lwt_result_syntax in - protect @@ fun () -> - let* commit_info = - Lwt.catch - (fun () -> - let* tup = - Context.retrieve_commit_info context_index (Block_repr.header block) - in - return_some (Protocol_levels.commit_info_of_tuple tup)) - (fun _ -> return_none) - in - return {Protocol_levels.block = Block_repr.descriptor block; commit_info} - let find_two_lowest_block_with_proto_level block_store ~head ~savepoint_level proto_level = let open Lwt_result_syntax in @@ -958,7 +944,7 @@ let fix_protocol_levels chain_dir block_store context_index genesis | None -> return (proto_level :: invalid_protocol_levels) | Some proto_info -> ( let activation_block_level = - snd proto_info.Protocol_levels.activation_block.block + snd proto_info.Protocol_levels.activation_block in let level_to_read = if @@ -1043,7 +1029,6 @@ let fix_protocol_levels chain_dir block_store context_index genesis = Block_repr.proto_level succ_lowest)) (Corrupted_store (Cannot_find_activation_block invalid_proto_level)) in - let* activation_block = craft_activation_block context_index b in let*! protocol = let*! ctxt = if is_genesis then diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index 3b23ba8230ba..2865796ece6b 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -2329,7 +2329,7 @@ module Make_snapshot_exporter (Exporter : EXPORTER) : Snapshot_exporter = struct Protocol_levels.( filter (fun level {activation_block; _} -> - let block = activation_block.block in + let block = activation_block in level >= Store.Block.proto_level minimum_block || Store.Block.is_genesis chain_store (fst block)) protocol_levels) @@ -2898,11 +2898,7 @@ module Raw_importer : IMPORTER = struct { Protocol_levels.protocol = activation_block.Protocol_levels.Legacy.protocol; - activation_block = - { - Protocol_levels.block = activation_block.block; - commit_info = activation_block.commit_info; - }; + activation_block = activation_block.block; (* Only snapshot with legacy semantics will have legacy encoding. *) expect_predecessor_context = false; @@ -3195,11 +3191,7 @@ module Tar_importer : IMPORTER = struct { Protocol_levels.protocol = activation_block.Protocol_levels.Legacy.protocol; - activation_block = - { - Protocol_levels.block = activation_block.block; - commit_info = activation_block.commit_info; - }; + activation_block = activation_block.block; (* Only snapshot with legacy semantics will have legacy encoding. *) expect_predecessor_context = false; diff --git a/src/lib_store/unix/store.ml b/src/lib_store/unix/store.ml index 1a68825fd452..62c47ae4e11f 100644 --- a/src/lib_store/unix/store.ml +++ b/src/lib_store/unix/store.ml @@ -1225,7 +1225,7 @@ module Chain = struct match Protocol_levels.find proto_level protocol_levels with | None -> Lwt.return_none | Some {activation_block; _} -> ( - let block_activation_level = snd activation_block.block in + let block_activation_level = snd activation_block in (* proto level's lower bound found, now retrieving the upper bound *) let head_proto_level = Block_repr.proto_level chain_state.current_head @@ -1242,7 +1242,7 @@ module Chain = struct with | None -> Lwt.return_none | Some {activation_block; _} -> ( - let next_activation_level = snd activation_block.block in + let next_activation_level = snd activation_block in let last_level_in_protocol = Int32.(pred next_activation_level) in @@ -1932,8 +1932,7 @@ module Chain = struct in {Block_repr.hash = genesis_hash; contents; metadata} - let create_chain_state ?target ~genesis_block ~genesis_protocol - ~genesis_commit_info chain_dir = + let create_chain_state ?target ~genesis_block ~genesis_protocol chain_dir = let open Lwt_result_syntax in let genesis_proto_level = Block_repr.proto_level genesis_block in let ((_, genesis_level) as genesis_descr) = @@ -1945,9 +1944,7 @@ module Chain = struct ~some:(fun metadata -> Block.last_allowed_fork_level metadata) (Block_repr.metadata genesis_block) in - let activation_block = - {Protocol_levels.block = genesis_descr; commit_info = genesis_commit_info} - in + let activation_block = genesis_descr in let* expect_predecessor_context = let open Lwt_result_syntax in let* (module Proto) = Registered_protocol.get_result genesis_protocol in @@ -2124,24 +2121,6 @@ module Chain = struct prechecked_blocks; } - let get_commit_info index header = - let open Lwt_result_syntax in - protect - ~on_error:(fun err -> - Format.kasprintf - (fun e -> tzfail (Missing_commit_info e)) - "%a" - Error_monad.pp_print_trace - err) - (fun () -> - let* tup = Context_ops.retrieve_commit_info index header in - return (Protocol_levels.commit_info_of_tuple tup)) - - let get_commit_info_opt index header = - let open Lwt_syntax in - let* r = get_commit_info index header in - match r with Ok v -> Lwt.return_some v | Error _ -> Lwt.return_none - let create_chain_store ?block_cache_limit global_store chain_dir ?target ~chain_id ?(expiration = None) ?genesis_block ~genesis ~genesis_context history_mode = @@ -2160,18 +2139,12 @@ module Chain = struct let* () = Stored_data.write_file (Naming.chain_config_file chain_dir) chain_config in - let*! genesis_commit_info = - get_commit_info_opt - global_store.context_index - (Block.header genesis_block) - in let* chain_state = create_chain_state chain_dir ?target ~genesis_block ~genesis_protocol:genesis.Genesis.protocol - ~genesis_commit_info in let* genesis_block_data = Stored_data.init @@ -2429,31 +2402,13 @@ module Chain = struct let expect_predecessor_context_hash chain_store ~protocol_level = expect_predecessor_context_hash chain_store ~protocol_level - let compute_commit_info chain_store block = - let open Lwt_result_syntax in - let index = chain_store.global_store.context_index in - protect - ~on_error:(fun _ -> return_none) - (fun () -> - let* commit_info = get_commit_info index block in - return_some commit_info) - let set_protocol_level chain_store ~protocol_level (block, protocol_hash, expect_predecessor_context) = let open Lwt_result_syntax in Shared.locked_use chain_store.chain_state (fun {protocol_levels_data; _} -> - let* commit_info_opt = - compute_commit_info chain_store (Block.header block) - in let* () = Stored_data.update_with protocol_levels_data (fun protocol_levels -> - let activation_block = - Protocol_levels. - { - block = Block.descriptor block; - commit_info = commit_info_opt; - } - in + let activation_block = Block.descriptor block in Lwt.return Protocol_levels.( add @@ -2491,7 +2446,7 @@ module Chain = struct if Compare.Int.(prev_proto_level < protocol_level) then let*! o = find_activation_block chain_store ~protocol_level in match o with - | Some {block = bh, _; _} -> + | Some (bh, _) -> if Block_hash.(bh <> Block.hash block) then set_protocol_level chain_store @@ -2545,13 +2500,11 @@ module Chain = struct ~protocol_level (activation_block, activated_protocol, expected_context_hash) | Some - { - Protocol_levels.protocol; - activation_block = {block; _}; - expect_predecessor_context; - } -> ( + {Protocol_levels.protocol; activation_block; expect_predecessor_context} + -> ( + let activation_block_level = snd activation_block in let*! _, savepoint_level = savepoint chain_store in - if Compare.Int32.(savepoint_level > snd block) then + if Compare.Int32.(savepoint_level > activation_block_level) then (* the block is too far in the past *) return_unit else @@ -2559,13 +2512,13 @@ module Chain = struct is_ancestor chain_store ~head:(Block.descriptor head) - ~ancestor:block + ~ancestor:activation_block in match b with | true -> (* nothing to do *) return_unit | false -> ( let distance = - Int32.(sub (Block.level head) (snd block) |> to_int) + Int32.(sub (Block.level head) activation_block_level |> to_int) in let*! o = Block.read_block_opt chain_store ~distance (Block.hash head) @@ -3090,17 +3043,14 @@ let rec make_pp_chain_store (chain_store : chain_store) = in let pp_proto_info fmt (proto_level, {Protocol_levels.protocol; activation_block; _}) = - let Protocol_levels.{block; commit_info} = activation_block in Format.fprintf fmt - "proto level: %d, transition block: %a, protocol: %a, commit info: %a" + "proto level: %d, transition block: %a, protocol: %a" proto_level pp_block_descriptor - block + activation_block Protocol_hash.pp protocol - (option_pp ~default:"n/a" (fun fmt _ -> Format.fprintf fmt "available")) - commit_info in let make_pp_test_chain_opt = function | None -> Lwt.return (fun fmt () -> Format.fprintf fmt "n/a") @@ -3223,11 +3173,7 @@ let upgrade_protocol_levels ~chain_dir ~cleanups ~finalizers = Protocol_levels. { protocol = legacy_activation_block.protocol; - activation_block = - { - block = legacy_activation_block.block; - commit_info = legacy_activation_block.commit_info; - }; + activation_block = legacy_activation_block.block; expect_predecessor_context = false (* the shell cannot have stored blocks that use the new semantics *); @@ -3357,7 +3303,6 @@ module Unsafe = struct let genesis_block = Block_repr.create_genesis_block ~genesis genesis_context_hash in - let real_genesis_hash = Block_header.hash (Block.header genesis_block) in let new_head_descr = ( Block_repr.hash new_head_with_metadata, Block_repr.level new_head_with_metadata ) @@ -3457,53 +3402,6 @@ module Unsafe = struct new_head_with_metadata new_head_resulting_context_hash in - (* Check correctness of protocol transition blocks *) - let* () = - List.iter_es - (fun (_, {Protocol_levels.protocol; activation_block; _}) -> - let {Protocol_levels.block = bh, _; commit_info = commit_info_opt} = - activation_block - in - let* block_opt = - Block_store.read_block - block_store - ~read_metadata:false - (Block (bh, 0)) - in - match (block_opt, commit_info_opt) with - | None, _ -> ( - match history_mode with - | Rolling _ -> - (* If we are importing a rolling snapshot then allow the - absence of block. *) - return_unit - | _ -> - fail_unless - (Block_hash.equal real_genesis_hash bh) - (Missing_activation_block (bh, protocol, history_mode))) - | Some _block, None -> return_unit - | Some block, Some commit_info -> - let*! is_consistent = - Context.check_protocol_commit_consistency - ~expected_context_hash:(Block.context_hash block) - ~given_protocol_hash:protocol - ~author:commit_info.author - ~message:commit_info.message - ~timestamp:(Block.timestamp block) - ~test_chain_status:commit_info.test_chain_status - ~predecessor_block_metadata_hash: - commit_info.predecessor_block_metadata_hash - ~predecessor_ops_metadata_hash: - commit_info.predecessor_ops_metadata_hash - ~data_merkle_root:commit_info.data_merkle_root - ~parents_contexts:commit_info.parents_contexts - in - fail_unless - (is_consistent - || Compare.Int32.(equal (Block_repr.level block) 0l)) - (Inconsistent_protocol_commit_info (Block.hash block, protocol))) - (Protocol_levels.bindings protocol_levels) - in let*! () = Block_store.close block_store in let chain_config = {history_mode; genesis; expiration = None} in let* () = diff --git a/src/lib_store/unix/store.mli b/src/lib_store/unix/store.mli index aec83f579641..342e54f53aeb 100644 --- a/src/lib_store/unix/store.mli +++ b/src/lib_store/unix/store.mli @@ -879,9 +879,7 @@ module Chain : sig (** [find_activation_block chain_store ~protocol_level] returns the block that activated the protocol of level [protocol_level]. *) val find_activation_block : - chain_store -> - protocol_level:int -> - Protocol_levels.activation_block option Lwt.t + chain_store -> protocol_level:int -> block_descriptor option Lwt.t (** [find_protocol chain_store ~protocol_level] returns the protocol with the level [protocol_level]. *) diff --git a/src/lib_store/unix/test/test_consistency.ml b/src/lib_store/unix/test/test_consistency.ml index c62869c3aac8..e5b088f86929 100644 --- a/src/lib_store/unix/test/test_consistency.ml +++ b/src/lib_store/unix/test/test_consistency.ml @@ -98,9 +98,7 @@ let init_protocols store history_mode = let* () = Protocol_levels.iter_es (fun proto_level {Protocol_levels.activation_block; _} -> - let activation_block_hash, activation_block_level = - activation_block.Protocol_levels.block - in + let activation_block_hash, activation_block_level = activation_block in (* We cannot consider gc-ed blocks *) if activation_block_level < Int32.succ savepoint_level then return_unit else @@ -191,21 +189,13 @@ let check_protocol_levels_availability chain_store ~expected_protocols let*! savepoint_hash, _ = Store.Chain.savepoint chain_store in let* savepoint = Store.Block.read_block chain_store savepoint_hash in let savepoint_proto_level = Store.Block.proto_level savepoint in - Protocol_levels.iter_es + Protocol_levels.iter (fun proto_level _ -> if proto_level < savepoint_proto_level then assert (not (Protocol_levels.mem proto_level recovered_protocols)) - else if proto_level >= savepoint_proto_level then - let recovered_activation_block = - Protocol_levels.find proto_level recovered_protocols - in - match recovered_activation_block with - | None -> assert false - | Some {Protocol_levels.activation_block = {commit_info; _}; _} -> - assert (Option.is_some commit_info) - else assert false ; - return_unit) - expected_protocols + else assert (Protocol_levels.mem proto_level recovered_protocols)) + expected_protocols ; + return_unit let test_protocol_level_consistency_remove_file history_mode (store_dir, context_dir) store = -- GitLab From 787fd8e7f3dbb13d954fe9a2ec7f8d4e255afa09 Mon Sep 17 00:00:00 2001 From: vbot Date: Tue, 15 Nov 2022 15:39:27 +0100 Subject: [PATCH 06/19] Store/Tests: adapt framework to new semantics --- manifest/main.ml | 1 + src/lib_store/unix/test/alpha_utils.ml | 61 +++++---- src/lib_store/unix/test/dune | 2 + src/lib_store/unix/test/test_consistency.ml | 62 +++++---- src/lib_store/unix/test/test_snapshots.ml | 1 - src/lib_store/unix/test/test_testchain.ml | 27 +--- src/lib_store/unix/test/test_utils.ml | 135 +++++++++++--------- src/lib_test/assert_lib.ml | 5 +- 8 files changed, 163 insertions(+), 131 deletions(-) diff --git a/manifest/main.ml b/manifest/main.ml index bb339a26400a..c0da4f2e9c88 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -5330,6 +5330,7 @@ let _octez_store_tests = octez_shell_services |> open_; octez_stdlib_unix |> open_; octez_validation |> open_; + octez_protocol_updater |> open_; Protocol.(embedded demo_noops); Protocol.(embedded genesis); Protocol.(embedded alpha); diff --git a/src/lib_store/unix/test/alpha_utils.ml b/src/lib_store/unix/test/alpha_utils.ml index 659d3f8ac36c..a179eeb96281 100644 --- a/src/lib_store/unix/test/alpha_utils.ml +++ b/src/lib_store/unix/test/alpha_utils.ml @@ -274,7 +274,7 @@ module Forge = struct } let make_shell ~level ~predecessor ~timestamp ~fitness ~operations_hash - ~proto_level = + ~proto_level ~pred_resulting_context_hash = Tezos_base.Block_header. { level; @@ -284,7 +284,7 @@ module Forge = struct operations_hash; proto_level; validation_passes = List.length Main.validation_passes; - context = Context_hash.zero (* to update later *); + context = pred_resulting_context_hash; } let set_seed_nonce_hash seed_nonce_hash {baker; shell; contents} = @@ -308,7 +308,8 @@ module Forge = struct in Block_header.{shell; protocol_data = {contents; signature}} |> return - let forge_header rpc_ctxt ?(policy = By_round 0) ?timestamp ~operations pred = + let forge_header rpc_ctxt ?(policy = By_round 0) ?timestamp ~operations pred + pred_resulting_context_hash = let open Lwt_result_syntax in let predecessor_round = match Fitness.from_raw (Store.Block.fitness pred) with @@ -349,6 +350,7 @@ module Forge = struct ~fitness ~operations_hash ~proto_level + ~pred_resulting_context_hash in let contents = make_contents @@ -492,9 +494,10 @@ let finalize_validation_and_application (validation_state, application_state) let* () = Main.finalize_validation validation_state in Main.finalize_application application_state shell_header -let apply ctxt chain_id ~policy ?(operations = empty_operations) pred = +let apply pred_resulting_ctxt chain_id ~policy ?(operations = empty_operations) + pred pred_resulting_context_hash = let open Lwt_result_syntax in - let* rpc_ctxt = make_rpc_context ~chain_id ctxt pred in + let* rpc_ctxt = make_rpc_context ~chain_id pred_resulting_ctxt pred in let element_of_key ~chain_id ~predecessor_context ~predecessor_timestamp ~predecessor_level ~predecessor_fitness ~predecessor ~timestamp = let*! f = @@ -514,34 +517,39 @@ let apply ctxt chain_id ~policy ?(operations = empty_operations) pred = return r) in let* {shell; contents; baker} = - Forge.forge_header rpc_ctxt ?policy ~operations pred + Forge.forge_header + rpc_ctxt + ?policy + ~operations + pred + pred_resulting_context_hash in let protocol_data = {Block_header.contents; signature = Signature.zero} in - let*! context = + let*! ctxt = match Store.Block.block_metadata_hash pred with - | None -> Lwt.return ctxt - | Some hash -> Context_ops.add_predecessor_block_metadata_hash ctxt hash + | None -> Lwt.return pred_resulting_ctxt + | Some hash -> + Context_ops.add_predecessor_block_metadata_hash pred_resulting_ctxt hash in let*! ctxt = match Store.Block.all_operations_metadata_hash pred with - | None -> Lwt.return context - | Some hash -> Context_ops.add_predecessor_ops_metadata_hash context hash + | None -> Lwt.return ctxt + | Some hash -> Context_ops.add_predecessor_ops_metadata_hash ctxt hash in - let predecessor_context = ctxt in let* element_of_key = element_of_key ~chain_id - ~predecessor_context + ~predecessor_context:ctxt ~predecessor_timestamp:(Store.Block.timestamp pred) ~predecessor_level:(Store.Block.level pred) ~predecessor_fitness:(Store.Block.fitness pred) ~predecessor:(Store.Block.hash pred) ~timestamp:shell.timestamp in - let* predecessor_context = + let* pred_ctxt = Tezos_protocol_environment.Context.load_cache (Store.Block.hash pred) - predecessor_context + ctxt `Lazy element_of_key in @@ -550,7 +558,7 @@ let apply ctxt chain_id ~policy ?(operations = empty_operations) pred = let open Environment.Error_monad in let* vstate = begin_validation_and_application - predecessor_context + pred_ctxt chain_id (Construction { @@ -582,7 +590,7 @@ let apply ctxt chain_id ~policy ?(operations = empty_operations) pred = in let validation = {validation with max_operations_ttl} in let context = Shell_context.unwrap_disk_context validation.context in - let*! context_hash = + let*! resulting_context_hash = Context.commit ~time:shell.timestamp ?message:validation.message context in let block_header_metadata = @@ -599,7 +607,6 @@ let apply ctxt chain_id ~policy ?(operations = empty_operations) pred = hashes in let contents = {contents with payload_hash} in - let shell = {shell with context = context_hash} in let* header = Forge.sign_header ~chain_id {baker; shell; contents} in let protocol_data = Data_encoding.Binary.to_bytes_exn @@ -623,21 +630,31 @@ let apply ctxt chain_id ~policy ?(operations = empty_operations) pred = block_header_metadata, block_hash_metadata, operations_metadata_hashes, + resulting_context_hash, validation ) let apply_and_store chain_store ?(synchronous_merge = true) ?policy ?(operations = empty_operations) pred = let open Lwt_result_syntax in - let* ctxt = Store.Block.context chain_store pred in + let* pred_resulting_ctxt = Store.Block.context chain_store pred in + let* pred_resulting_context_hash = + Store.Block.resulting_context_hash chain_store pred + in let chain_id = Store.Chain.chain_id chain_store in let* ( block_header, block_header_metadata, block_metadata_hash, ops_metadata_hashes, + resulting_context_hash, validation ) = - apply ctxt chain_id ~policy ~operations pred + apply + pred_resulting_ctxt + chain_id + ~policy + ~operations + pred + pred_resulting_context_hash in - let context_hash = block_header.shell.context in let ops_metadata = let operations_metadata = WithExceptions.List.init ~loc:__LOC__ 4 (fun _ -> []) @@ -658,7 +675,7 @@ let apply_and_store chain_store ?(synchronous_merge = true) ?policy { Tezos_validation.Block_validation.validation_store = { - context_hash; + context_hash = resulting_context_hash; timestamp = block_header.shell.timestamp; message = validation.Tezos_protocol_environment.message; max_operations_ttl = validation.max_operations_ttl; diff --git a/src/lib_store/unix/test/dune b/src/lib_store/unix/test/dune index 649d9ff36bc9..c4f4b4715e9b 100644 --- a/src/lib_store/unix/test/dune +++ b/src/lib_store/unix/test/dune @@ -22,6 +22,7 @@ tezos-shell-services tezos-stdlib-unix tezos-validation + tezos-protocol-updater tezos-embedded-protocol-demo-noops tezos-embedded-protocol-genesis tezos-embedded-protocol-alpha @@ -42,6 +43,7 @@ -open Tezos_shell_services -open Tezos_stdlib_unix -open Tezos_validation + -open Tezos_protocol_updater -open Tezos_protocol_alpha_parameters -open Tezos_protocol_plugin_alpha)) diff --git a/src/lib_store/unix/test/test_consistency.ml b/src/lib_store/unix/test/test_consistency.ml index e5b088f86929..7e0a2e993256 100644 --- a/src/lib_store/unix/test/test_consistency.ml +++ b/src/lib_store/unix/test/test_consistency.ml @@ -27,31 +27,50 @@ open Test_utils let nb_protocols = 5 -let init_protocols store history_mode = - let open Lwt_result_syntax in - let chain_store = Store.main_chain_store store in - assert (History_mode.equal (Store.Chain.history_mode chain_store) history_mode) ; +let register_protocol ~hash ~sources = + let module M = struct + include + Registered_protocol.Register_embedded_V8 + (Tezos_protocol_environment_demo_noops) + (Tezos_protocol_demo_noops.Protocol) + (struct + let hash = Some hash + + let sources = sources + end) + end in + () + +let test_protocols = let mk i = let s = string_of_int i in let proto = { - Protocol.expected_env = V0; + Protocol.expected_env = V8; components = [{name = s; interface = None; implementation = s}]; } in - (Protocol.hash proto, proto) + let proto_hash = Protocol.hash proto in + register_protocol ~hash:proto_hash ~sources:proto ; + (proto_hash, proto) in - let protos = Stdlib.List.init nb_protocols mk in + Stdlib.List.init nb_protocols mk + +let init_protocols store history_mode = + let open Lwt_result_syntax in + let chain_store = Store.main_chain_store store in + assert (History_mode.equal (Store.Chain.history_mode chain_store) history_mode) ; let*! () = List.iter_s (fun (h, p) -> let*! r = Store.Protocol.store store h p in assert (r <> None) ; + (* register protocol *) Lwt.return_unit) - protos + test_protocols in - assert (List.for_all (Store.Protocol.mem store) (List.map fst protos)) ; - let blocks_proto = List.mapi (fun i p -> (succ i, fst p)) protos in + assert (List.for_all (Store.Protocol.mem store) (List.map fst test_protocols)) ; + let blocks_proto = List.mapi (fun i p -> (succ i, fst p)) test_protocols in (* 10 blocks per cycle *) let nb_blocks_per_cycle = 10 in let constants = @@ -64,28 +83,17 @@ let init_protocols store history_mode = let* () = List.iter_es (fun (protocol_level, proto_hash) -> - let*! pred = Store.Chain.current_head chain_store in - let* bl, _ = + let* _ = append_blocks ~constants ~max_operations_ttl:8 ~kind:`Full ~should_set_head:true - ~should_commit:true ~protocol_level ~set_protocol:proto_hash chain_store nb_blocks_per_cycle in - let first_block_of_cycle = Stdlib.List.hd bl in - let* () = - Store.Chain.may_update_protocol_level - chain_store - ~pred - ~protocol_level - ~expect_predecessor_context:true - (first_block_of_cycle, proto_hash) - in let*! () = Block_store.await_merging (Store.Unsafe.get_block_store chain_store) in @@ -173,9 +181,14 @@ let test_protocol_level_consistency_drop_one history_mode nth "unexpected missing proto level %d in the protocol levels" (Store.Block.proto_level b) in - let*! ctxt = Store.Block.context_exn chain_store b in + let* ctxt = Store.Block.context chain_store b in let*! proto_block = Context_ops.get_protocol ctxt in - assert (Protocol_hash.(proto = proto_block)) ; + Assert.equal + ~eq:Protocol_hash.equal + ~pp:Protocol_hash.pp + ~loc:__LOC__ + proto + proto_block ; return_unit) (Int32.to_int savepoint_level -- Int32.to_int (Store.Block.level current_head)) @@ -286,7 +299,6 @@ let make_tests = let () = Lwt_main.run @@ Alcotest_lwt.run - ~argv:[|""|] "tezos-store" [ ( "consistency", diff --git a/src/lib_store/unix/test/test_snapshots.ml b/src/lib_store/unix/test/test_snapshots.ml index 2a8cabe62efd..867d260769f4 100644 --- a/src/lib_store/unix/test/test_snapshots.ml +++ b/src/lib_store/unix/test/test_snapshots.ml @@ -109,7 +109,6 @@ let export_import ~test_descr ~previously_baked_blocks ?exported_block_hash | None -> `Alias (`Checkpoint, 0) | Some hash -> `Hash (hash, 0) in - (*FIXME test over Raw formats as well *) let* () = Snapshots.export ~snapshot_path diff --git a/src/lib_store/unix/test/test_testchain.ml b/src/lib_store/unix/test/test_testchain.ml index 68fa63ce809b..63767080b50d 100644 --- a/src/lib_store/unix/test/test_testchain.ml +++ b/src/lib_store/unix/test/test_testchain.ml @@ -39,8 +39,11 @@ let fork_testchain chain_store (blocks, forked_block) = let context_index = Store.context_index global_store in (* Call [Context.fork_test_chain] then commit so we are able to gather commit info *) + let* resulting_context_hash = + Store.Block.resulting_context_hash chain_store forked_block + in let*! context = - Context_ops.checkout_exn context_index head_header.shell.context + Context_ops.checkout_exn context_index resulting_context_hash in let*! context = Context_ops.fork_test_chain context ~protocol:test_protocol ~expiration @@ -72,7 +75,6 @@ let fork_testchain chain_store (blocks, forked_block) = let* test_blocks, head = append_blocks ~min_lafl:genesis_header.shell.level - ~should_commit:true ~should_set_head:true testchain_store ~kind:`Full @@ -87,12 +89,7 @@ let test_simple store = let open Lwt_result_syntax in let chain_store = Store.main_chain_store store in let* blocks, head = - append_blocks - ~should_commit:true - ~should_set_head:true - chain_store - ~kind:`Full - 10 + append_blocks ~should_set_head:true chain_store ~kind:`Full 10 in let* _ = fork_testchain chain_store (blocks, head) in return_unit @@ -101,12 +98,7 @@ let test_inner store = let open Lwt_result_syntax in let chain_store = Store.main_chain_store store in let* blocks, head = - append_blocks - ~should_commit:true - ~should_set_head:true - chain_store - ~kind:`Full - 10 + append_blocks ~should_set_head:true chain_store ~kind:`Full 10 in let* testchain, blocks, head = fork_testchain chain_store (blocks, head) in let testchain_store = Store.Chain.testchain_store testchain in @@ -117,12 +109,7 @@ let test_shutdown store = let open Lwt_result_syntax in let chain_store = Store.main_chain_store store in let* blocks, head = - append_blocks - ~should_commit:true - ~should_set_head:true - chain_store - ~kind:`Full - 10 + append_blocks ~should_set_head:true chain_store ~kind:`Full 10 in let* testchain, blocks, _head = fork_testchain chain_store (blocks, head) in let testchain_store = Store.Chain.testchain_store testchain in diff --git a/src/lib_store/unix/test/test_utils.ml b/src/lib_store/unix/test/test_utils.ml index 9234cf2291dd..0dc264e95482 100644 --- a/src/lib_store/unix/test/test_utils.ml +++ b/src/lib_store/unix/test/test_utils.ml @@ -440,7 +440,7 @@ let pp_raw_block fmt b = let h, lvl = raw_descriptor b in Format.fprintf fmt "%a (%ld)" Block_hash.pp h lvl -let store_raw_block chain_store (raw_block : Block_repr.t) = +let store_raw_block chain_store ?resulting_context (raw_block : Block_repr.t) = let open Lwt_syntax in let metadata = WithExceptions.Option.get ~loc:__LOC__ (Block_repr.metadata raw_block) @@ -463,7 +463,10 @@ let store_raw_block chain_store (raw_block : Block_repr.t) = { Tezos_validation.Block_validation.validation_store = { - context_hash = Block_repr.context raw_block; + context_hash = + Option.value + ~default:(Block_repr.context raw_block) + resulting_context; timestamp = Block_repr.timestamp raw_block; message = Block_repr.message metadata; max_operations_ttl = Block_repr.max_operations_ttl metadata; @@ -543,8 +546,7 @@ let incr_fitness b = | _ -> assert false let append_blocks ?min_lafl ?constants ?max_operations_ttl ?root ?(kind = `Full) - ?(should_set_head = false) ?(should_commit = false) ?protocol_level - ?set_protocol chain_store n = + ?(should_set_head = false) ?protocol_level ?set_protocol chain_store n = let open Lwt_result_syntax in let*! root = match root with @@ -554,10 +556,13 @@ let append_blocks ?min_lafl ?constants ?max_operations_ttl ?root ?(kind = `Full) Lwt.return (Store.Block.descriptor block) in let* root_b = Store.Block.read_block chain_store (fst root) in - let*! ctxt_opt = - Context_ops.checkout + let* resulting_context_hash = + Store.Block.resulting_context_hash chain_store root_b + in + let*! ctxt = + Context_ops.checkout_exn (Store.context_index (Store.Chain.global_store chain_store)) - (Store.Block.context_hash root_b) + resulting_context_hash in let*! blocks, _last = make_raw_block_list ?min_lafl ?constants ?max_operations_ttl ~kind root n @@ -569,68 +574,76 @@ let append_blocks ?min_lafl ?constants ?max_operations_ttl ?root ?(kind = `Full) in let* _, _, blocks = List.fold_left_es - (fun (ctxt_opt, last_opt, blocks) b -> - let* ctxt, last_opt, b = - if should_commit then - let ctxt = WithExceptions.Option.get ~loc:__LOC__ ctxt_opt in - let*! ctxt = - Context_ops.add - ctxt - ["level"] - (Bytes.of_string (Format.asprintf "%ld" (Block_repr.level b))) - in - let*! ctxt = - match set_protocol with - | None -> Lwt.return ctxt - | Some proto -> Context_ops.add_protocol ctxt proto - in - let*! ctxt_hash = - Context_ops.commit ~time:Time.Protocol.epoch ctxt - in - let predecessor = - match List.hd blocks with None -> root_b | Some pred -> pred - in - let shell = + (fun (ctxt, pred, blocks) b -> + let* ctxt, resulting_context, b = + let*! ctxt = + Context_ops.add + ctxt + ["level"] + (Bytes.of_string (Format.asprintf "%ld" (Block_repr.level b))) + in + let*! ctxt = + match set_protocol with + | None -> Lwt.return ctxt + | Some proto -> Context_ops.add_protocol ctxt proto + in + let* pred_resulting_context = + Store.Block.resulting_context_hash chain_store pred + in + let shell = + { + b.contents.header.Block_header.shell with + fitness = incr_fitness (Store.Block.fitness pred); + proto_level; + context = pred_resulting_context; + predecessor = Store.Block.hash pred; + } + in + let header = + { + Block_header.shell; + protocol_data = b.contents.header.protocol_data; + } + in + let hash = Block_header.hash header in + let*! resulting_context = + Context_ops.commit ~time:Time.Protocol.epoch ctxt + in + return + ( ctxt, + resulting_context, { - b.contents.header.Block_header.shell with - fitness = incr_fitness (Store.Block.fitness predecessor); - proto_level; - context = ctxt_hash; - predecessor = Store.Block.hash predecessor; - } - in - let header = - { - Block_header.shell; - protocol_data = b.contents.header.protocol_data; - } - in - let hash = Block_header.hash header in - return - ( Some ctxt, - Some hash, - { - Block_repr.hash; - contents = - { - header; - operations = b.contents.operations; - block_metadata_hash = None; - operations_metadata_hashes = None; - }; - metadata = b.metadata; - } ) - else return (ctxt_opt, last_opt, b) + Block_repr.hash; + contents = + { + header; + operations = b.contents.operations; + block_metadata_hash = None; + operations_metadata_hashes = None; + }; + metadata = b.metadata; + } ) + in + let* b = store_raw_block chain_store ~resulting_context b in + let* () = + match set_protocol with + | None -> return_unit + | Some proto -> + Store.Chain.may_update_protocol_level + chain_store + ~pred + ~protocol_level:(Store.Block.proto_level b) + ~expect_predecessor_context:true + (b, proto) in - let* b = store_raw_block chain_store b in let* () = if should_set_head then let* _ = Store.Chain.set_head chain_store b in return_unit else return_unit in - return (ctxt, last_opt, b :: blocks)) - (ctxt_opt, None, []) + return (ctxt, b, b :: blocks)) + (ctxt, root_b, []) blocks in let head = List.hd blocks |> WithExceptions.Option.get ~loc:__LOC__ in diff --git a/src/lib_test/assert_lib.ml b/src/lib_test/assert_lib.ml index 7ff0e6fb62ce..9110b17eca04 100644 --- a/src/lib_test/assert_lib.ml +++ b/src/lib_test/assert_lib.ml @@ -44,10 +44,11 @@ module Crypto = struct let pp ppf st = Format.fprintf ppf - "%a (%ld)" + "@[%a@ %a@]" Tezos_crypto.Block_hash.pp (Tezos_base.Block_header.hash st) - st.shell.level + Data_encoding.Json.pp + (Data_encoding.Json.construct Tezos_base.Block_header.encoding st) in Assert.equal ?loc ?msg ~pp ~eq st1 st2 -- GitLab From 14cb119854399c951a0359e5d1137a8bebe130c4 Mon Sep 17 00:00:00 2001 From: vbot Date: Mon, 14 Nov 2022 14:11:10 +0100 Subject: [PATCH 07/19] Snapshots: bump snapshot version --- src/lib_store/unix/snapshots.ml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index 2865796ece6b..4ed4c12c3603 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -591,11 +591,13 @@ module Version = struct * - 1: snapshot exported with storage 0.0.1 to 0.0.4 * - 2: snapshot exported with storage 0.0.4 to 0.0.6 * - 3: snapshot exported with storage 0.0.7 - * - 4: snapshot exported with storage 0.0.8 to current *) - let current_version = 4 + * - 4: snapshot exported with storage 0.0.8 to current + * - 5: snapshot exported with new protocol tables *) + let current_version = 5 (* List of versions that are supported *) - let supported_versions = [(2, `Legacy); (3, `Legacy); (4, `Current)] + let supported_versions = + [(2, `Legacy); (3, `Legacy); (4, `Current); (5, `Current)] let is_supported version = match List.assq_opt version supported_versions with -- GitLab From 3672a71a70892f934aea2140b589a92923314570 Mon Sep 17 00:00:00 2001 From: vbot Date: Mon, 21 Nov 2022 14:41:33 +0100 Subject: [PATCH 08/19] Shell/Validation: open Tezos_crypto by default and make fmt --- manifest/main.ml | 1 + opam/tezos-validation.opam | 1 + src/lib_validation/block_validation.ml | 92 +++++++++------------- src/lib_validation/block_validation.mli | 56 ++++++------- src/lib_validation/dune | 2 + src/lib_validation/external_validation.ml | 92 ++++++++++------------ src/lib_validation/external_validation.mli | 34 ++++---- src/lib_validation/external_validator.ml | 25 +++--- src/lib_validation/patch_context.ml | 6 +- 9 files changed, 137 insertions(+), 172 deletions(-) diff --git a/manifest/main.ml b/manifest/main.ml index c0da4f2e9c88..6e23382c241a 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -2226,6 +2226,7 @@ let octez_validation = ~deps: [ octez_base |> open_ ~m:"TzPervasives"; + octez_crypto |> open_; octez_rpc; octez_context |> open_; octez_context_ops |> open_; diff --git a/opam/tezos-validation.opam b/opam/tezos-validation.opam index 0401c233c1a1..074b08617a06 100644 --- a/opam/tezos-validation.opam +++ b/opam/tezos-validation.opam @@ -10,6 +10,7 @@ license: "MIT" depends: [ "dune" { >= "3.0" } "tezos-base" + "tezos-crypto" "tezos-rpc" "tezos-context" "tezos-context-ops" diff --git a/src/lib_validation/block_validation.ml b/src/lib_validation/block_validation.ml index d47b390e6cb9..ea12d1a82b60 100644 --- a/src/lib_validation/block_validation.ml +++ b/src/lib_validation/block_validation.ml @@ -37,12 +37,12 @@ module Event = struct ~name:"block_validation_inconsistent_cache" ~msg:"applied block {hash} with an inconsistent cache: reloading cache" ~level:Warning - ~pp1:Tezos_crypto.Block_hash.pp - ("hash", Tezos_crypto.Block_hash.encoding) + ~pp1:Block_hash.pp + ("hash", Block_hash.encoding) end type validation_store = { - context_hash : Tezos_crypto.Context_hash.t; + context_hash : Context_hash.t; timestamp : Time.Protocol.t; message : string option; max_operations_ttl : int; @@ -77,7 +77,7 @@ let validation_store_encoding = last_allowed_fork_level; }) (obj5 - (req "context_hash" Tezos_crypto.Context_hash.encoding) + (req "context_hash" Context_hash.encoding) (req "timestamp" Time.Protocol.encoding) (req "message" (option string)) (req "max_operations_ttl" int31) @@ -114,12 +114,11 @@ let operation_metadata_encoding = type ops_metadata = | No_metadata_hash of operation_metadata list list - | Metadata_hash of - (operation_metadata * Tezos_crypto.Operation_metadata_hash.t) list list + | Metadata_hash of (operation_metadata * Operation_metadata_hash.t) list list type result = { validation_store : validation_store; - block_metadata : Bytes.t * Tezos_crypto.Block_metadata_hash.t option; + block_metadata : Bytes.t * Block_metadata_hash.t option; ops_metadata : ops_metadata; } @@ -150,7 +149,7 @@ let update_testchain_status ctxt ~predecessor_hash timestamp = let genesis = Context_ops.compute_testchain_genesis ctxt predecessor_hash in - let chain_id = Tezos_crypto.Chain_id.of_block_hash genesis in + let chain_id = Chain_id.of_block_hash genesis in (* legacy semantics *) Context_ops.add_test_chain ctxt @@ -199,7 +198,7 @@ let result_encoding = (list (tup2 operation_metadata_encoding - Tezos_crypto.Operation_metadata_hash.encoding))) + Operation_metadata_hash.encoding))) (function | Metadata_hash ops_metadata -> Some ops_metadata | _ -> None) (fun ops_metadata -> Metadata_hash ops_metadata); @@ -212,9 +211,7 @@ let result_encoding = {validation_store; block_metadata; ops_metadata}) (obj3 (req "validation_store" validation_store_encoding) - (req - "block_metadata" - (tup2 bytes (option Tezos_crypto.Block_metadata_hash.encoding))) + (req "block_metadata" (tup2 bytes (option Block_metadata_hash.encoding))) (req "ops_metadata" ops_metadata_encoding)) let preapply_result_encoding : @@ -269,7 +266,7 @@ let may_patch_protocol ~user_activated_upgrades module Make (Proto : Registered_protocol.T) = struct type 'operation_data preapplied_operation = { - hash : Tezos_crypto.Operation_hash.t; + hash : Operation_hash.t; raw : Operation.t; protocol_data : 'operation_data; } @@ -279,8 +276,8 @@ module Make (Proto : Registered_protocol.T) = struct application_state : Proto.application_state; applied : (Proto.operation_data preapplied_operation * Proto.operation_receipt) list; - live_blocks : Tezos_crypto.Block_hash.Set.t; - live_operations : Tezos_crypto.Operation_hash.Set.t; + live_blocks : Block_hash.Set.t; + live_operations : Operation_hash.Set.t; } type preapply_result = @@ -440,7 +437,7 @@ module Make (Proto : Registered_protocol.T) = struct in let metadata_hash_opt = if should_include_metadata_hashes then - Some (Tezos_crypto.Block_metadata_hash.hash_bytes [metadata]) + Some (Block_metadata_hash.hash_bytes [metadata]) else None in (metadata, metadata_hash_opt) @@ -479,7 +476,7 @@ module Make (Proto : Registered_protocol.T) = struct (List.map (List.map (fun (bytes, metadata) -> let metadata_hash = - Tezos_crypto.Operation_metadata_hash.hash_bytes [bytes] + Operation_metadata_hash.hash_bytes [bytes] in (metadata, metadata_hash))) metadata_list_list)) @@ -562,7 +559,7 @@ module Make (Proto : Registered_protocol.T) = struct (block_header : Proto.block_header) block_hash (validation_result : Tezos_protocol_environment.validation_result) = let open Lwt_result_syntax in - if Tezos_crypto.Protocol_hash.equal new_protocol Proto.hash then + if Protocol_hash.equal new_protocol Proto.hash then return (validation_result, Proto.environment_version) else match Registered_protocol.get new_protocol with @@ -594,7 +591,7 @@ module Make (Proto : Registered_protocol.T) = struct let block_hash = Block_header.hash block_header in match cached_result with | Some (({result; _} as cached_result), context) - when Tezos_crypto.Context_hash.equal + when Context_hash.equal result.validation_store.context_hash block_header.shell.context && Time.Protocol.equal @@ -615,9 +612,7 @@ module Make (Proto : Registered_protocol.T) = struct context in assert ( - Tezos_crypto.Context_hash.equal - context_hash - result.validation_store.context_hash) ; + Context_hash.equal context_hash result.validation_store.context_hash) ; return cached_result | Some _ | None -> let* () = @@ -659,7 +654,7 @@ module Make (Proto : Registered_protocol.T) = struct let context = validation_result.context in let*! new_protocol = Context_ops.get_protocol context in let expected_proto_level = - if Tezos_crypto.Protocol_hash.equal new_protocol Proto.hash then + if Protocol_hash.equal new_protocol Proto.hash then predecessor_block_header.shell.proto_level else (predecessor_block_header.shell.proto_level + 1) mod 256 in @@ -723,9 +718,7 @@ module Make (Proto : Registered_protocol.T) = struct in let* () = fail_unless - (Tezos_crypto.Context_hash.equal - context_hash - block_header.shell.context) + (Context_hash.equal context_hash block_header.shell.context) (Validation_errors.Inconsistent_hash (context_hash, block_header.shell.context)) in @@ -788,8 +781,7 @@ module Make (Proto : Registered_protocol.T) = struct let preapply_operation pv op = let open Lwt_syntax in - if Tezos_crypto.Operation_hash.Set.mem op.hash pv.live_operations then - return Outdated + if Operation_hash.Set.mem op.hash pv.live_operations then return Outdated else let+ r = protect (fun () -> @@ -814,7 +806,7 @@ module Make (Proto : Registered_protocol.T) = struct applied = (op, receipt) :: pv.applied; live_blocks = pv.live_blocks; live_operations = - Tezos_crypto.Operation_hash.Set.add op.hash pv.live_operations; + Operation_hash.Set.add op.hash pv.live_operations; } in match @@ -937,7 +929,7 @@ module Make (Proto : Registered_protocol.T) = struct Lwt.return ({preapp with applied}, t, receipt :: receipts) | Branch_delayed errors -> let branch_delayed = - Tezos_crypto.Operation_hash.Map.add + Operation_hash.Map.add op.hash (op.raw, errors) preapp.branch_delayed @@ -945,7 +937,7 @@ module Make (Proto : Registered_protocol.T) = struct Lwt.return ({preapp with branch_delayed}, t, receipts) | Branch_refused errors -> let branch_refused = - Tezos_crypto.Operation_hash.Map.add + Operation_hash.Map.add op.hash (op.raw, errors) preapp.branch_refused @@ -953,10 +945,7 @@ module Make (Proto : Registered_protocol.T) = struct Lwt.return ({preapp with branch_refused}, t, receipts) | Refused errors -> let refused = - Tezos_crypto.Operation_hash.Map.add - op.hash - (op.raw, errors) - preapp.refused + Operation_hash.Map.add op.hash (op.raw, errors) preapp.refused in Lwt.return ({preapp with refused}, t, receipts) | Outdated -> Lwt.return (preapp, t, receipts) @@ -1007,10 +996,10 @@ module Make (Proto : Registered_protocol.T) = struct let applied_ops_metadata = List.rev receipts_rev in let preapply_state = validation_state in let operations_hash = - Tezos_crypto.Operation_list_list_hash.compute + Operation_list_list_hash.compute (List.rev_map (fun r -> - Tezos_crypto.Operation_list_hash.compute + Operation_list_hash.compute (List.map fst r.Preapply_result.applied)) validation_result_list_rev) in @@ -1023,7 +1012,7 @@ module Make (Proto : Registered_protocol.T) = struct timestamp; validation_passes; operations_hash; - context = Tezos_crypto.Context_hash.zero (* place holder *); + context = Context_hash.zero (* place holder *); fitness = []; } in @@ -1044,7 +1033,7 @@ module Make (Proto : Registered_protocol.T) = struct Tezos_protocol_environment.Context.get_protocol validation_result.context in let proto_level = - if Tezos_crypto.Protocol_hash.equal protocol Proto.hash then + if Protocol_hash.equal protocol Proto.hash then predecessor_shell_header.proto_level else (predecessor_shell_header.proto_level + 1) mod 256 in @@ -1052,7 +1041,7 @@ module Make (Proto : Registered_protocol.T) = struct {shell_header with proto_level; fitness = validation_result.fitness} in let* validation_result, cache, new_protocol_env_version = - if Tezos_crypto.Protocol_hash.equal protocol Proto.hash then + if Protocol_hash.equal protocol Proto.hash then let (Tezos_protocol_environment.Context.Context {cache; _}) = validation_result.context in @@ -1066,7 +1055,7 @@ module Make (Proto : Registered_protocol.T) = struct | Some (module NewProto) -> let*? () = check_proto_environment_version_increasing - Tezos_crypto.Block_hash.zero + Block_hash.zero Proto.environment_version NewProto.environment_version in @@ -1183,9 +1172,9 @@ let assert_no_duplicate_operations block_hash live_operations operations = (List.fold_left (List.fold_left (fun live_operations op -> let oph = Operation.hash op in - if Tezos_crypto.Operation_hash.Set.mem oph live_operations then + if Operation_hash.Set.mem oph live_operations then raise (Duplicate (Replayed_operation oph)) - else Tezos_crypto.Operation_hash.Set.add oph live_operations)) + else Operation_hash.Set.add oph live_operations)) live_operations operations) with Duplicate err -> tzfail (invalid_block block_hash err) @@ -1197,11 +1186,7 @@ let assert_operation_liveness block_hash live_blocks operations = return (List.iter (List.iter (fun op -> - if - not - (Tezos_crypto.Block_hash.Set.mem - op.Operation.shell.branch - live_blocks) + if not (Block_hash.Set.mem op.Operation.shell.branch live_blocks) then let error = Outdated_operation @@ -1218,19 +1203,18 @@ let assert_operation_liveness block_hash live_blocks operations = once by [Block_validator_process] *) let check_liveness ~live_blocks ~live_operations block_hash operations = let open Result_syntax in - let* (_ : Tezos_crypto.Operation_hash.Set.t) = + let* (_ : Operation_hash.Set.t) = assert_no_duplicate_operations block_hash live_operations operations in assert_operation_liveness block_hash live_blocks operations type apply_environment = { max_operations_ttl : int; - chain_id : Tezos_crypto.Chain_id.t; + chain_id : Chain_id.t; predecessor_block_header : Block_header.t; predecessor_context : Tezos_protocol_environment.Context.t; - predecessor_block_metadata_hash : Tezos_crypto.Block_metadata_hash.t option; - predecessor_ops_metadata_hash : - Tezos_crypto.Operation_metadata_list_list_hash.t option; + predecessor_block_metadata_hash : Block_metadata_hash.t option; + predecessor_ops_metadata_hash : Operation_metadata_list_list_hash.t option; user_activated_upgrades : User_activated.upgrades; user_activated_protocol_overrides : User_activated.protocol_overrides; operation_metadata_size_limit : Shell_limits.operation_metadata_size_limit; @@ -1407,7 +1391,7 @@ let preapply ~chain_id ~user_activated_upgrades (* This should not happen: it should be handled in the validator. *) failwith "Prevalidation: missing protocol '%a' for the current block." - Tezos_crypto.Protocol_hash.pp_short + Protocol_hash.pp_short protocol | Some protocol -> return protocol in diff --git a/src/lib_validation/block_validation.mli b/src/lib_validation/block_validation.mli index 0cc29f3d98dc..d440d5bfd8ed 100644 --- a/src/lib_validation/block_validation.mli +++ b/src/lib_validation/block_validation.mli @@ -30,7 +30,7 @@ the protocol. *) type validation_store = { - context_hash : Tezos_crypto.Context_hash.t; + context_hash : Context_hash.t; timestamp : Time.Protocol.t; message : string option; max_operations_ttl : int; @@ -47,7 +47,7 @@ val may_patch_protocol : val update_testchain_status : Tezos_protocol_environment.Context.t -> - predecessor_hash:Tezos_crypto.Block_hash.t -> + predecessor_hash:Block_hash.t -> Time.Protocol.t -> Tezos_protocol_environment.Context.t Lwt.t @@ -56,14 +56,11 @@ val update_testchain_status : increases from [before] to [after]. Otherwise, an [Invalid_protocol_environment_transition] error is returned. *) val check_proto_environment_version_increasing : - Tezos_crypto.Block_hash.t -> - Protocol.env_version -> - Protocol.env_version -> - unit tzresult + Block_hash.t -> Protocol.env_version -> Protocol.env_version -> unit tzresult (** [init_test_chain] must only be called on a forking block. *) val init_test_chain : - Tezos_crypto.Chain_id.t -> + Chain_id.t -> Tezos_protocol_environment.Context.t -> Block_header.t -> Block_header.t tzresult Lwt.t @@ -74,12 +71,11 @@ val operation_metadata_encoding : operation_metadata Data_encoding.t type ops_metadata = | No_metadata_hash of operation_metadata list list - | Metadata_hash of - (operation_metadata * Tezos_crypto.Operation_metadata_hash.t) list list + | Metadata_hash of (operation_metadata * Operation_metadata_hash.t) list list type result = { validation_store : validation_store; - block_metadata : bytes * Tezos_crypto.Block_metadata_hash.t option; + block_metadata : bytes * Block_metadata_hash.t option; ops_metadata : ops_metadata; } @@ -96,23 +92,22 @@ val preapply_result_encoding : (** [check_liveness live_blocks live_operations hash ops] checks there is no duplicate operation and that is not out-of-date *) val check_liveness : - live_blocks:Tezos_crypto.Block_hash.Set.t -> - live_operations:Tezos_crypto.Operation_hash.Set.t -> - Tezos_crypto.Block_hash.t -> + live_blocks:Block_hash.Set.t -> + live_operations:Operation_hash.Set.t -> + Block_hash.t -> Operation.t list list -> unit tzresult type apply_environment = { max_operations_ttl : int; (** time to live of an operation *) - chain_id : Tezos_crypto.Chain_id.t; (** chain_id of the current branch *) + chain_id : Chain_id.t; (** chain_id of the current branch *) predecessor_block_header : Block_header.t; (** header of the predecessor block being validated *) predecessor_context : Tezos_protocol_environment.Context.t; (** context associated to the predecessor block *) - predecessor_block_metadata_hash : Tezos_crypto.Block_metadata_hash.t option; + predecessor_block_metadata_hash : Block_metadata_hash.t option; (** hash of block header metadata of the predecessor block *) - predecessor_ops_metadata_hash : - Tezos_crypto.Operation_metadata_list_list_hash.t option; + predecessor_ops_metadata_hash : Operation_metadata_list_list_hash.t option; (** hash of operation metadata of the predecessor block *) user_activated_upgrades : User_activated.upgrades; (** user activated upgrades *) @@ -153,9 +148,9 @@ val apply : 2. [P.validate_operation] 3. [P.finalize_validation] *) val precheck : - chain_id:Tezos_crypto.Chain_id.t -> + chain_id:Chain_id.t -> predecessor_block_header:Block_header.t -> - predecessor_block_hash:Tezos_crypto.Block_hash.t -> + predecessor_block_hash:Block_hash.t -> predecessor_context:Tezos_protocol_environment.Context.t -> cache:Tezos_protocol_environment.Context.source_of_cache -> Block_header.t -> @@ -163,21 +158,20 @@ val precheck : unit tzresult Lwt.t val preapply : - chain_id:Tezos_crypto.Chain_id.t -> + chain_id:Chain_id.t -> user_activated_upgrades:Tezos_base.User_activated.upgrades -> user_activated_protocol_overrides:Tezos_base.User_activated.protocol_overrides -> operation_metadata_size_limit:Shell_limits.operation_metadata_size_limit -> timestamp:Time.Protocol.t -> protocol_data:bytes -> - live_blocks:Tezos_crypto.Block_hash.Set.t -> - live_operations:Tezos_crypto.Operation_hash.Set.t -> + live_blocks:Block_hash.Set.t -> + live_operations:Operation_hash.Set.t -> predecessor_context:Tezos_protocol_environment.Context.t -> predecessor_shell_header:Block_header.shell_header -> - predecessor_hash:Tezos_crypto.Block_hash.t -> + predecessor_hash:Block_hash.t -> predecessor_max_operations_ttl:int -> - predecessor_block_metadata_hash:Tezos_crypto.Block_metadata_hash.t option -> - predecessor_ops_metadata_hash: - Tezos_crypto.Operation_metadata_list_list_hash.t option -> + predecessor_block_metadata_hash:Block_metadata_hash.t option -> + predecessor_ops_metadata_hash:Operation_metadata_list_list_hash.t option -> Operation.t list list -> ((Block_header.shell_header * error Preapply_result.t list) * (apply_result * Tezos_protocol_environment.Context.t)) @@ -187,14 +181,12 @@ val preapply : (** Hypothesis: we assume that the given block has already been validated -- E.g. by calling [precheck]. *) val recompute_metadata : - chain_id:Tezos_crypto.Chain_id.t -> + chain_id:Chain_id.t -> predecessor_block_header:Block_header.t -> predecessor_context:Tezos_protocol_environment.Context.t -> - predecessor_block_metadata_hash:Tezos_crypto.Block_metadata_hash.t option -> - predecessor_ops_metadata_hash: - Tezos_crypto.Operation_metadata_list_list_hash.t option -> + predecessor_block_metadata_hash:Block_metadata_hash.t option -> + predecessor_ops_metadata_hash:Operation_metadata_list_list_hash.t option -> block_header:Block_header.t -> operations:Operation.t trace trace -> cache:Tezos_protocol_environment.Context.source_of_cache -> - ((bytes * Tezos_crypto.Block_metadata_hash.t option) * ops_metadata) tzresult - Lwt.t + ((bytes * Block_metadata_hash.t option) * ops_metadata) tzresult Lwt.t diff --git a/src/lib_validation/dune b/src/lib_validation/dune index c736cd9c4d2c..e4c5cebcc231 100644 --- a/src/lib_validation/dune +++ b/src/lib_validation/dune @@ -8,6 +8,7 @@ (instrumentation (backend tezos-time-measurement)) (libraries tezos-base + tezos-crypto tezos-rpc tezos-context tezos-context-ops @@ -18,6 +19,7 @@ (flags (:standard) -open Tezos_base.TzPervasives + -open Tezos_crypto -open Tezos_context -open Tezos_context_ops -open Tezos_shell_context diff --git a/src/lib_validation/external_validation.ml b/src/lib_validation/external_validation.ml index f5e8c595c06e..668dd7b856be 100644 --- a/src/lib_validation/external_validation.ml +++ b/src/lib_validation/external_validation.ml @@ -37,48 +37,46 @@ type parameters = { type request = | Init | Validate of { - chain_id : Tezos_crypto.Chain_id.t; + chain_id : Chain_id.t; block_header : Block_header.t; predecessor_block_header : Block_header.t; - predecessor_block_metadata_hash : - Tezos_crypto.Block_metadata_hash.t option; + predecessor_block_metadata_hash : Block_metadata_hash.t option; predecessor_ops_metadata_hash : - Tezos_crypto.Operation_metadata_list_list_hash.t option; + Operation_metadata_list_list_hash.t option; operations : Operation.t list list; max_operations_ttl : int; should_precheck : bool; simulate : bool; } | Preapply of { - chain_id : Tezos_crypto.Chain_id.t; + chain_id : Chain_id.t; timestamp : Time.Protocol.t; protocol_data : bytes; - live_blocks : Tezos_crypto.Block_hash.Set.t; - live_operations : Tezos_crypto.Operation_hash.Set.t; + live_blocks : Block_hash.Set.t; + live_operations : Operation_hash.Set.t; predecessor_shell_header : Block_header.shell_header; - predecessor_hash : Tezos_crypto.Block_hash.t; + predecessor_hash : Block_hash.t; predecessor_max_operations_ttl : int; - predecessor_block_metadata_hash : - Tezos_crypto.Block_metadata_hash.t option; + predecessor_block_metadata_hash : Block_metadata_hash.t option; predecessor_ops_metadata_hash : - Tezos_crypto.Operation_metadata_list_list_hash.t option; + Operation_metadata_list_list_hash.t option; operations : Operation.t list list; } | Precheck of { - chain_id : Tezos_crypto.Chain_id.t; + chain_id : Chain_id.t; predecessor_block_header : Block_header.t; - predecessor_block_hash : Tezos_crypto.Block_hash.t; + predecessor_block_hash : Block_hash.t; header : Block_header.t; operations : Operation.t list list; - hash : Tezos_crypto.Block_hash.t; + hash : Block_hash.t; } - | Commit_genesis of {chain_id : Tezos_crypto.Chain_id.t} + | Commit_genesis of {chain_id : Chain_id.t} | Fork_test_chain of { - chain_id : Tezos_crypto.Chain_id.t; - context_hash : Tezos_crypto.Context_hash.t; + chain_id : Chain_id.t; + context_hash : Context_hash.t; forked_header : Block_header.t; } - | Context_garbage_collection of {context_hash : Tezos_crypto.Context_hash.t} + | Context_garbage_collection of {context_hash : Context_hash.t} | Terminate | Reconfigure_event_logging of Tezos_base_unix.Internal_event_unix.Configuration.t @@ -89,42 +87,38 @@ let request_pp ppf = function Format.fprintf ppf "block validation %a for chain %a" - Tezos_crypto.Block_hash.pp_short + Block_hash.pp_short (Block_header.hash block_header) - Tezos_crypto.Chain_id.pp_short + Chain_id.pp_short chain_id | Preapply {predecessor_hash; chain_id; _} -> Format.fprintf ppf "preapply block ontop of %a for chain %a" - Tezos_crypto.Block_hash.pp_short + Block_hash.pp_short predecessor_hash - Tezos_crypto.Chain_id.pp_short + Chain_id.pp_short chain_id | Precheck {hash; _} -> - Format.fprintf - ppf - "precheck block %a" - Tezos_crypto.Block_hash.pp_short - hash + Format.fprintf ppf "precheck block %a" Block_hash.pp_short hash | Commit_genesis {chain_id} -> Format.fprintf ppf "commit genesis block for chain %a" - Tezos_crypto.Chain_id.pp_short + Chain_id.pp_short chain_id | Fork_test_chain {forked_header; _} -> Format.fprintf ppf "test chain fork on block %a" - Tezos_crypto.Block_hash.pp_short + Block_hash.pp_short (Block_header.hash forked_header) | Terminate -> Format.fprintf ppf "terminate validation process" | Context_garbage_collection {context_hash} -> Format.fprintf ppf "garbage collecting context below %a" - Tezos_crypto.Context_hash.pp + Context_hash.pp context_hash | Reconfigure_event_logging _ -> Format.fprintf ppf "reconfigure event logging" @@ -185,15 +179,11 @@ let case_validate tag = tag ~title:"validate" (obj9 - (req "chain_id" Tezos_crypto.Chain_id.encoding) + (req "chain_id" Chain_id.encoding) (req "block_header" (dynamic_size Block_header.encoding)) (req "pred_header" (dynamic_size Block_header.encoding)) - (opt - "pred_block_metadata_hash" - Tezos_crypto.Block_metadata_hash.encoding) - (opt - "pred_ops_metadata_hash" - Tezos_crypto.Operation_metadata_list_list_hash.encoding) + (opt "pred_block_metadata_hash" Block_metadata_hash.encoding) + (opt "pred_ops_metadata_hash" Operation_metadata_list_list_hash.encoding) (req "max_operations_ttl" int31) (req "operations" (list (list (dynamic_size Operation.encoding)))) (req "should_precheck" bool) @@ -251,20 +241,18 @@ let case_preapply tag = ~title:"preapply" (merge_objs (obj10 - (req "chain_id" Tezos_crypto.Chain_id.encoding) + (req "chain_id" Chain_id.encoding) (req "timestamp" Time.Protocol.encoding) (req "protocol_data" bytes) - (req "live_blocks" Tezos_crypto.Block_hash.Set.encoding) - (req "live_operations" Tezos_crypto.Operation_hash.Set.encoding) + (req "live_blocks" Block_hash.Set.encoding) + (req "live_operations" Operation_hash.Set.encoding) (req "predecessor_shell_header" Block_header.shell_header_encoding) - (req "predecessor_hash" Tezos_crypto.Block_hash.encoding) + (req "predecessor_hash" Block_hash.encoding) (req "predecessor_max_operations_ttl" int31) - (opt - "predecessor_block_metadata_hash" - Tezos_crypto.Block_metadata_hash.encoding) + (opt "predecessor_block_metadata_hash" Block_metadata_hash.encoding) (opt "predecessor_ops_metadata_hash" - Tezos_crypto.Operation_metadata_list_list_hash.encoding)) + Operation_metadata_list_list_hash.encoding)) (obj1 (req "operations" (list (list (dynamic_size Operation.encoding)))))) (function | Preapply @@ -326,11 +314,11 @@ let case_precheck tag = tag ~title:"precheck" (obj6 - (req "chain_id" Tezos_crypto.Chain_id.encoding) + (req "chain_id" Chain_id.encoding) (req "predecessor_block_header" (dynamic_size Block_header.encoding)) - (req "predecessor_block_hash" Tezos_crypto.Block_hash.encoding) + (req "predecessor_block_hash" Block_hash.encoding) (req "header" (dynamic_size Block_header.encoding)) - (req "hash" Tezos_crypto.Block_hash.encoding) + (req "hash" Block_hash.encoding) (req "operations" (list (list (dynamic_size Operation.encoding))))) (function | Precheck @@ -371,7 +359,7 @@ let case_context_gc tag = case tag ~title:"context_gc" - (obj1 (req "context_hash" Tezos_crypto.Context_hash.encoding)) + (obj1 (req "context_hash" Context_hash.encoding)) (function | Context_garbage_collection {context_hash} -> Some context_hash | _ -> None) @@ -391,15 +379,15 @@ let request_encoding = case (Tag 2) ~title:"commit_genesis" - (obj1 (req "chain_id" Tezos_crypto.Chain_id.encoding)) + (obj1 (req "chain_id" Chain_id.encoding)) (function Commit_genesis {chain_id} -> Some chain_id | _ -> None) (fun chain_id -> Commit_genesis {chain_id}); case (Tag 3) ~title:"fork_test_chain" (obj3 - (req "chain_id" Tezos_crypto.Chain_id.encoding) - (req "context_hash" Tezos_crypto.Context_hash.encoding) + (req "chain_id" Chain_id.encoding) + (req "context_hash" Context_hash.encoding) (req "forked_header" Block_header.encoding)) (function | Fork_test_chain {chain_id; context_hash; forked_header} -> diff --git a/src/lib_validation/external_validation.mli b/src/lib_validation/external_validation.mli index d89f6c33c0e8..d1bb6afe0016 100644 --- a/src/lib_validation/external_validation.mli +++ b/src/lib_validation/external_validation.mli @@ -37,48 +37,46 @@ type parameters = { type request = | Init | Validate of { - chain_id : Tezos_crypto.Chain_id.t; + chain_id : Chain_id.t; block_header : Block_header.t; predecessor_block_header : Block_header.t; - predecessor_block_metadata_hash : - Tezos_crypto.Block_metadata_hash.t option; + predecessor_block_metadata_hash : Block_metadata_hash.t option; predecessor_ops_metadata_hash : - Tezos_crypto.Operation_metadata_list_list_hash.t option; + Operation_metadata_list_list_hash.t option; operations : Operation.t list list; max_operations_ttl : int; should_precheck : bool; simulate : bool; } | Preapply of { - chain_id : Tezos_crypto.Chain_id.t; + chain_id : Chain_id.t; timestamp : Time.Protocol.t; protocol_data : bytes; - live_blocks : Tezos_crypto.Block_hash.Set.t; - live_operations : Tezos_crypto.Operation_hash.Set.t; + live_blocks : Block_hash.Set.t; + live_operations : Operation_hash.Set.t; predecessor_shell_header : Block_header.shell_header; - predecessor_hash : Tezos_crypto.Block_hash.t; + predecessor_hash : Block_hash.t; predecessor_max_operations_ttl : int; - predecessor_block_metadata_hash : - Tezos_crypto.Block_metadata_hash.t option; + predecessor_block_metadata_hash : Block_metadata_hash.t option; predecessor_ops_metadata_hash : - Tezos_crypto.Operation_metadata_list_list_hash.t option; + Operation_metadata_list_list_hash.t option; operations : Operation.t list list; } | Precheck of { - chain_id : Tezos_crypto.Chain_id.t; + chain_id : Chain_id.t; predecessor_block_header : Block_header.t; - predecessor_block_hash : Tezos_crypto.Block_hash.t; + predecessor_block_hash : Block_hash.t; header : Block_header.t; operations : Operation.t list list; - hash : Tezos_crypto.Block_hash.t; + hash : Block_hash.t; } - | Commit_genesis of {chain_id : Tezos_crypto.Chain_id.t} + | Commit_genesis of {chain_id : Chain_id.t} | Fork_test_chain of { - chain_id : Tezos_crypto.Chain_id.t; - context_hash : Tezos_crypto.Context_hash.t; + chain_id : Chain_id.t; + context_hash : Context_hash.t; forked_header : Block_header.t; } - | Context_garbage_collection of {context_hash : Tezos_crypto.Context_hash.t} + | Context_garbage_collection of {context_hash : Context_hash.t} | Terminate | Reconfigure_event_logging of Tezos_base_unix.Internal_event_unix.Configuration.t diff --git a/src/lib_validation/external_validator.ml b/src/lib_validation/external_validator.ml index 21727db52ffb..de6c6988645f 100644 --- a/src/lib_validation/external_validator.ml +++ b/src/lib_validation/external_validator.ml @@ -51,8 +51,8 @@ module Events = struct ~level:Debug ~name:"dynload_protocol" ~msg:"dynamic loading of protocol {protocol}" - ~pp1:Tezos_crypto.Protocol_hash.pp - ("protocol", Tezos_crypto.Protocol_hash.encoding) + ~pp1:Protocol_hash.pp + ("protocol", Protocol_hash.encoding) let validation_request = declare_1 @@ -60,8 +60,7 @@ module Events = struct ~level:Debug ~name:"validation_request" ~msg:"validating block {block}" - ~pp1:(fun fmt header -> - Tezos_crypto.Block_hash.pp fmt (Block_header.hash header)) + ~pp1:(fun fmt header -> Block_hash.pp fmt (Block_header.hash header)) ("block", Block_header.encoding) let precheck_request = @@ -70,8 +69,8 @@ module Events = struct ~level:Debug ~name:"precheck_request" ~msg:"prechecking block {hash}" - ~pp1:Tezos_crypto.Block_hash.pp - ("hash", Tezos_crypto.Block_hash.encoding) + ~pp1:Block_hash.pp + ("hash", Block_hash.encoding) let commit_genesis_request = declare_1 @@ -79,8 +78,8 @@ module Events = struct ~level:Debug ~name:"commit_genesis_request" ~msg:"committing genesis block {genesis}" - ~pp1:Tezos_crypto.Block_hash.pp - ("genesis", Tezos_crypto.Block_hash.encoding) + ~pp1:Block_hash.pp + ("genesis", Block_hash.encoding) let initialization_request = declare_0 @@ -105,8 +104,8 @@ module Events = struct ~level:Debug ~name:"context_gc_request" ~msg:"garbage collecting context below {context_hash}" - ~pp1:Tezos_crypto.Context_hash.pp - ("context_hash", Tezos_crypto.Context_hash.encoding) + ~pp1:Context_hash.pp + ("context_hash", Context_hash.encoding) let termination_request = declare_0 @@ -127,8 +126,8 @@ let load_protocol proto protocol_root = else let cmxs_file = protocol_root - // Tezos_crypto.Protocol_hash.to_short_b58check proto - // Format.asprintf "protocol_%a.cmxs" Tezos_crypto.Protocol_hash.pp proto + // Protocol_hash.to_short_b58check proto + // Format.asprintf "protocol_%a.cmxs" Protocol_hash.pp proto in let*! () = Events.(emit dynload_protocol proto) in match Dynlink.loadfile_private cmxs_file with @@ -247,7 +246,7 @@ let run ~readonly input output = let*! () = External_validation.send output - (Error_monad.result_encoding Tezos_crypto.Context_hash.encoding) + (Error_monad.result_encoding Context_hash.encoding) commit in loop cache None diff --git a/src/lib_validation/patch_context.ml b/src/lib_validation/patch_context.ml index 0c3ba522117e..6c788e361f37 100644 --- a/src/lib_validation/patch_context.ml +++ b/src/lib_validation/patch_context.ml @@ -39,7 +39,7 @@ let patch_context (genesis : Genesis.t) key_json ctxt = let module Proto = (val proto) in let* {context; _} = Proto.init - (Tezos_crypto.Chain_id.of_block_hash genesis.block) + (Chain_id.of_block_hash genesis.block) ctxt { level = 0l; @@ -47,9 +47,9 @@ let patch_context (genesis : Genesis.t) key_json ctxt = predecessor = genesis.block; timestamp = genesis.time; validation_passes = 0; - operations_hash = Tezos_crypto.Operation_list_list_hash.empty; + operations_hash = Operation_list_list_hash.empty; fitness = []; - context = Tezos_crypto.Context_hash.zero; + context = Context_hash.zero; } in return context -- GitLab From ec90b553a618bd697155527a347c5b30f6305258 Mon Sep 17 00:00:00 2001 From: vbot Date: Thu, 10 Nov 2022 09:52:09 +0100 Subject: [PATCH 09/19] Shell/Validation: adapt validation to the new semantics --- src/lib_validation/block_validation.ml | 210 ++++++++++++++++----- src/lib_validation/block_validation.mli | 18 +- src/lib_validation/external_validation.ml | 27 ++- src/lib_validation/external_validation.mli | 3 + src/lib_validation/external_validator.ml | 38 ++-- 5 files changed, 228 insertions(+), 68 deletions(-) diff --git a/src/lib_validation/block_validation.ml b/src/lib_validation/block_validation.ml index ea12d1a82b60..266cc22ee20e 100644 --- a/src/lib_validation/block_validation.ml +++ b/src/lib_validation/block_validation.ml @@ -42,7 +42,7 @@ module Event = struct end type validation_store = { - context_hash : Context_hash.t; + resulting_context_hash : Context_hash.t; timestamp : Time.Protocol.t; message : string option; max_operations_ttl : int; @@ -53,31 +53,31 @@ let validation_store_encoding = let open Data_encoding in conv (fun { - context_hash; + resulting_context_hash; timestamp; message; max_operations_ttl; last_allowed_fork_level; } -> - ( context_hash, + ( resulting_context_hash, timestamp, message, max_operations_ttl, last_allowed_fork_level )) - (fun ( context_hash, + (fun ( resulting_context_hash, timestamp, message, max_operations_ttl, last_allowed_fork_level ) -> { - context_hash; + resulting_context_hash; timestamp; message; max_operations_ttl; last_allowed_fork_level; }) (obj5 - (req "context_hash" Context_hash.encoding) + (req "resulting_context_hash" Context_hash.encoding) (req "timestamp" Time.Protocol.encoding) (req "message" (option string)) (req "max_operations_ttl" int31) @@ -116,7 +116,25 @@ type ops_metadata = | No_metadata_hash of operation_metadata list list | Metadata_hash of (operation_metadata * Operation_metadata_hash.t) list list +module Shell_header_hash = + Blake2B.Make + (Base58) + (struct + let name = "shell_header_hash" + + let title = "A shell header identifier" + + let b58check_prefix = Base58.Prefix.block_hash + + let size = None + end) + +let hash_shell_header shell = + Shell_header_hash.hash_bytes + [Data_encoding.Binary.to_bytes_exn Block_header.shell_header_encoding shell] + type result = { + shell_header_hash : Shell_header_hash.t; validation_store : validation_store; block_metadata : Bytes.t * Block_metadata_hash.t option; ops_metadata : ops_metadata; @@ -205,11 +223,12 @@ let result_encoding = ] in conv - (fun {validation_store; block_metadata; ops_metadata} -> - (validation_store, block_metadata, ops_metadata)) - (fun (validation_store, block_metadata, ops_metadata) -> - {validation_store; block_metadata; ops_metadata}) - (obj3 + (fun {shell_header_hash; validation_store; block_metadata; ops_metadata} -> + (shell_header_hash, validation_store, block_metadata, ops_metadata)) + (fun (shell_header_hash, validation_store, block_metadata, ops_metadata) -> + {shell_header_hash; validation_store; block_metadata; ops_metadata}) + (obj4 + (req "shell_header_hash" Shell_header_hash.encoding) (req "validation_store" validation_store_encoding) (req "block_metadata" (tup2 bytes (option Block_metadata_hash.encoding))) (req "ops_metadata" ops_metadata_encoding)) @@ -287,8 +306,8 @@ module Make (Proto : Registered_protocol.T) = struct | Refused of error list | Outdated - let check_block_header ~(predecessor_block_header : Block_header.t) hash - (block_header : Block_header.t) = + let check_block_header ~(predecessor_block_header : Block_header.t) + ~predecessor_resulting_context_hash hash (block_header : Block_header.t) = let open Lwt_result_syntax in let* () = fail_unless @@ -323,6 +342,24 @@ module Make (Proto : Registered_protocol.T) = struct (Unexpected_number_of_validation_passes block_header.shell.validation_passes)) in + let is_consistent = + match Proto.expected_context_hash with + | Predecessor_resulting_context -> + Context_hash.equal + block_header.shell.context + predecessor_resulting_context_hash + | Resulting_context -> + (* The check that a header's context is the resulting context + will be performed post-application (when the resulting + context is known). *) + true + in + let* () = + fail_unless + is_consistent + (Validation_errors.Inconsistent_hash + (predecessor_resulting_context_hash, block_header.shell.context)) + in return_unit let parse_block_header block_hash (block_header : Block_header.t) = @@ -560,7 +597,10 @@ module Make (Proto : Registered_protocol.T) = struct (validation_result : Tezos_protocol_environment.validation_result) = let open Lwt_result_syntax in if Protocol_hash.equal new_protocol Proto.hash then - return (validation_result, Proto.environment_version) + return + ( validation_result, + Proto.environment_version, + Proto.expected_context_hash ) else match Registered_protocol.get new_protocol with | None -> @@ -579,24 +619,30 @@ module Make (Proto : Registered_protocol.T) = struct let* validation_result = NewProto.init chain_id validation_result.context block_header.shell in - return (validation_result, NewProto.environment_version) + return + ( validation_result, + NewProto.environment_version, + NewProto.expected_context_hash ) let apply ?(simulate = false) ?cached_result chain_id ~cache ~user_activated_upgrades ~user_activated_protocol_overrides ~operation_metadata_size_limit ~max_operations_ttl ~(predecessor_block_header : Block_header.t) ~predecessor_block_metadata_hash ~predecessor_ops_metadata_hash - ~predecessor_context ~(block_header : Block_header.t) operations = + ~predecessor_context ~predecessor_resulting_context_hash + ~(block_header : Block_header.t) operations = let open Lwt_result_syntax in let block_hash = Block_header.hash block_header in + let shell_header_hash = hash_shell_header block_header.shell in match cached_result with | Some (({result; _} as cached_result), context) - when Context_hash.equal - result.validation_store.context_hash - block_header.shell.context - && Time.Protocol.equal - result.validation_store.timestamp - block_header.shell.timestamp -> + when Shell_header_hash.equal result.shell_header_hash shell_header_hash -> + (* In order to implement the preapply's cache mechanism, we + need to differentiate blocks. Their hash cannot be used as + a resulting preapply block does not contain correct + protocol data (e.g. signature). Therefore, their hashes + won't be the same. However, shell headers will remain the + same thus we use those to discriminate blocks. *) let*! () = Validation_events.(emit using_preapply_result block_hash) in let*! context_hash = if simulate then @@ -612,11 +658,17 @@ module Make (Proto : Registered_protocol.T) = struct context in assert ( - Context_hash.equal context_hash result.validation_store.context_hash) ; + Context_hash.equal + context_hash + result.validation_store.resulting_context_hash) ; return cached_result | Some _ | None -> let* () = - check_block_header ~predecessor_block_header block_hash block_header + check_block_header + ~predecessor_block_header + ~predecessor_resulting_context_hash + block_hash + block_header in let* block_header = parse_block_header block_hash block_header in let* () = check_operation_quota block_hash operations in @@ -680,7 +732,8 @@ module Make (Proto : Registered_protocol.T) = struct found = validation_result.fitness; })) in - let* validation_result, new_protocol_env_version = + let* validation_result, new_protocol_env_version, expected_context_hash + = may_init_new_protocol chain_id new_protocol @@ -703,7 +756,7 @@ module Make (Proto : Registered_protocol.T) = struct in let (Context {cache; _}) = validation_result.context in let context = validation_result.context in - let*! context_hash = + let*! resulting_context_hash = if simulate then Lwt.return @@ Context_ops.hash @@ -717,14 +770,26 @@ module Make (Proto : Registered_protocol.T) = struct context [@time.duration_lwt context_commitment] [@time.flush] in let* () = + let is_context_consistent = + match expected_context_hash with + | Predecessor_resulting_context -> + (* The check that the header's context is the + predecessor's resulting context has already been + performed in the [check_block_header] call above. *) + true + | Resulting_context -> + Context_hash.equal + resulting_context_hash + block_header.shell.context + in fail_unless - (Context_hash.equal context_hash block_header.shell.context) + is_context_consistent (Validation_errors.Inconsistent_hash - (context_hash, block_header.shell.context)) + (resulting_context_hash, block_header.shell.context)) in let validation_store = { - context_hash; + resulting_context_hash; timestamp = block_header.shell.timestamp; message = validation_result.message; max_operations_ttl = validation_result.max_operations_ttl; @@ -732,7 +797,16 @@ module Make (Proto : Registered_protocol.T) = struct } in return - {result = {validation_store; block_metadata; ops_metadata}; cache} + { + result = + { + shell_header_hash = hash_shell_header block_header.shell; + validation_store; + block_metadata; + ops_metadata; + }; + cache; + } let recompute_metadata chain_id ~cache ~(predecessor_block_header : Block_header.t) @@ -765,7 +839,7 @@ module Make (Proto : Registered_protocol.T) = struct in let context = validation_result.context in let*! new_protocol = Context_ops.get_protocol context in - let* _validation_result, new_protocol_env_version = + let* _validation_result, new_protocol_env_version, _expected_context_hash = may_init_new_protocol chain_id new_protocol @@ -851,7 +925,7 @@ module Make (Proto : Registered_protocol.T) = struct let preapply ~chain_id ~cache ~user_activated_upgrades ~user_activated_protocol_overrides ~operation_metadata_size_limit ~protocol_data ~live_blocks ~live_operations ~timestamp - ~predecessor_context + ~predecessor_context ~predecessor_resulting_context_hash ~(predecessor_shell_header : Block_header.shell_header) ~predecessor_hash ~predecessor_max_operations_ttl ~predecessor_block_metadata_hash ~predecessor_ops_metadata_hash ~operations = @@ -1040,12 +1114,19 @@ module Make (Proto : Registered_protocol.T) = struct let shell_header : Block_header.shell_header = {shell_header with proto_level; fitness = validation_result.fitness} in - let* validation_result, cache, new_protocol_env_version = + let* ( validation_result, + cache, + new_protocol_env_version, + proto_expected_context_hash ) = if Protocol_hash.equal protocol Proto.hash then let (Tezos_protocol_environment.Context.Context {cache; _}) = validation_result.context in - return (validation_result, cache, Proto.environment_version) + return + ( validation_result, + cache, + Proto.environment_version, + Proto.expected_context_hash ) else match Registered_protocol.get protocol with | None -> @@ -1068,17 +1149,26 @@ module Make (Proto : Registered_protocol.T) = struct let*! () = Validation_events.(emit new_protocol_initialisation NewProto.hash) in - return (validation_result, cache, NewProto.environment_version) + return + ( validation_result, + cache, + NewProto.environment_version, + NewProto.expected_context_hash ) in let context = validation_result.context in - let context_hash = + let resulting_context_hash = Context_ops.hash ?message:validation_result.message ~time:timestamp context in + let header_context_hash = + match proto_expected_context_hash with + | Resulting_context -> resulting_context_hash + | Predecessor_resulting_context -> predecessor_resulting_context_hash + in let preapply_result = - ({shell_header with context = context_hash}, validation_result_list) + ({shell_header with context = header_context_hash}, validation_result_list) in let* block_metadata, ops_metadata = compute_metadata @@ -1097,24 +1187,36 @@ module Make (Proto : Registered_protocol.T) = struct let result = let validation_store = { - context_hash; + resulting_context_hash; timestamp; message = validation_result.message; max_operations_ttl; last_allowed_fork_level = validation_result.last_allowed_fork_level; } in - let result = {validation_store; block_metadata; ops_metadata} in + let result = + { + shell_header_hash = hash_shell_header (fst preapply_result); + validation_store; + block_metadata; + ops_metadata; + } + in {result; cache} in return (preapply_result, (result, context)) let precheck block_hash chain_id ~(predecessor_block_header : Block_header.t) - ~predecessor_block_hash ~predecessor_context ~cache + ~predecessor_block_hash ~predecessor_context + ~predecessor_resulting_context_hash ~cache ~(block_header : Block_header.t) operations = let open Lwt_result_syntax in let* () = - check_block_header ~predecessor_block_header block_hash block_header + check_block_header + ~predecessor_block_header + ~predecessor_resulting_context_hash + block_hash + block_header in let* block_header = parse_block_header block_hash block_header in let* () = check_operation_quota block_hash operations in @@ -1149,7 +1251,8 @@ module Make (Proto : Registered_protocol.T) = struct return_unit let precheck chain_id ~(predecessor_block_header : Block_header.t) - ~predecessor_block_hash ~predecessor_context ~cache + ~predecessor_block_hash ~predecessor_context + ~predecessor_resulting_context_hash ~cache ~(block_header : Block_header.t) operations = let block_hash = Block_header.hash block_header in trace (invalid_block block_hash Economic_protocol_error) @@ -1159,6 +1262,7 @@ module Make (Proto : Registered_protocol.T) = struct ~predecessor_block_header ~predecessor_block_hash ~predecessor_context + ~predecessor_resulting_context_hash ~cache ~block_header operations @@ -1213,6 +1317,7 @@ type apply_environment = { chain_id : Chain_id.t; predecessor_block_header : Block_header.t; predecessor_context : Tezos_protocol_environment.Context.t; + predecessor_resulting_context_hash : Context_hash.t; predecessor_block_metadata_hash : Block_metadata_hash.t option; predecessor_ops_metadata_hash : Operation_metadata_list_list_hash.t option; user_activated_upgrades : User_activated.upgrades; @@ -1267,7 +1372,8 @@ let recompute_metadata ~chain_id ~predecessor_block_header ~predecessor_context | (Ok _ | Error _) as res -> Lwt.return res let precheck ~chain_id ~predecessor_block_header ~predecessor_block_hash - ~predecessor_context ~cache block_header operations = + ~predecessor_context ~predecessor_resulting_context_hash ~cache block_header + operations = let open Lwt_result_syntax in let block_hash = Block_header.hash block_header in let*! pred_protocol_hash = Context_ops.get_protocol predecessor_context in @@ -1285,6 +1391,7 @@ let precheck ~chain_id ~predecessor_block_header ~predecessor_block_hash ~predecessor_block_header ~predecessor_block_hash ~predecessor_context + ~predecessor_resulting_context_hash ~cache ~block_header operations @@ -1300,6 +1407,7 @@ let apply ?simulate ?cached_result ?(should_precheck = true) predecessor_block_metadata_hash; predecessor_ops_metadata_hash; predecessor_context; + predecessor_resulting_context_hash; } ~cache block_hash block_header operations = let open Lwt_result_syntax in let*! pred_protocol_hash = Context_ops.get_protocol predecessor_context in @@ -1318,6 +1426,7 @@ let apply ?simulate ?cached_result ?(should_precheck = true) ~predecessor_block_header ~predecessor_block_hash:block_header.Block_header.shell.predecessor ~predecessor_context + ~predecessor_resulting_context_hash ~cache block_header operations @@ -1336,6 +1445,7 @@ let apply ?simulate ?cached_result ?(should_precheck = true) ~predecessor_block_metadata_hash ~predecessor_ops_metadata_hash ~predecessor_context + ~predecessor_resulting_context_hash ~cache ~block_header operations @@ -1360,7 +1470,11 @@ let apply ?simulate ?cached_result ?should_precheck apply_environment ~cache in match r with | Error (Validation_errors.Inconsistent_hash _ :: _) -> - (* The shell makes an assumption over the protocol concerning the cache which may be broken. In that case, the application fails with an [Inconsistency_hash] error. To make the node resilient to such problem, when such an error occurs, we retry the application using a fresh cache. *) + (* The shell makes an assumption over the protocol concerning + the cache which may be broken. In that case, the application + fails with an [Inconsistency_hash] error. To make the node + resilient to such problem, when such an error occurs, we retry + the application using a fresh cache. *) let*! () = Event.(emit inherited_inconsistent_cache) block_hash in apply ?cached_result @@ -1380,7 +1494,8 @@ let apply ?simulate ?cached_result ?should_precheck apply_environment ~cache let preapply ~chain_id ~user_activated_upgrades ~user_activated_protocol_overrides ~operation_metadata_size_limit ~timestamp ~protocol_data ~live_blocks ~live_operations ~predecessor_context - ~predecessor_shell_header ~predecessor_hash ~predecessor_max_operations_ttl + ~predecessor_resulting_context_hash ~predecessor_shell_header + ~predecessor_hash ~predecessor_max_operations_ttl ~predecessor_block_metadata_hash ~predecessor_ops_metadata_hash operations = let open Lwt_result_syntax in let*! protocol = Context_ops.get_protocol predecessor_context in @@ -1418,6 +1533,7 @@ let preapply ~chain_id ~user_activated_upgrades ~live_operations ~timestamp ~predecessor_context + ~predecessor_resulting_context_hash ~predecessor_shell_header ~predecessor_hash ~predecessor_max_operations_ttl @@ -1428,7 +1544,8 @@ let preapply ~chain_id ~user_activated_upgrades let preapply ~chain_id ~user_activated_upgrades ~user_activated_protocol_overrides ~operation_metadata_size_limit ~timestamp ~protocol_data ~live_blocks ~live_operations ~predecessor_context - ~predecessor_shell_header ~predecessor_hash ~predecessor_max_operations_ttl + ~predecessor_resulting_context_hash ~predecessor_shell_header + ~predecessor_hash ~predecessor_max_operations_ttl ~predecessor_block_metadata_hash ~predecessor_ops_metadata_hash operations = let open Lwt_result_syntax in let*! r = @@ -1442,6 +1559,7 @@ let preapply ~chain_id ~user_activated_upgrades ~live_blocks ~live_operations ~predecessor_context + ~predecessor_resulting_context_hash ~predecessor_shell_header ~predecessor_hash ~predecessor_max_operations_ttl diff --git a/src/lib_validation/block_validation.mli b/src/lib_validation/block_validation.mli index d440d5bfd8ed..d65c64bf9f13 100644 --- a/src/lib_validation/block_validation.mli +++ b/src/lib_validation/block_validation.mli @@ -30,7 +30,7 @@ the protocol. *) type validation_store = { - context_hash : Context_hash.t; + resulting_context_hash : Context_hash.t; timestamp : Time.Protocol.t; message : string option; max_operations_ttl : int; @@ -73,7 +73,12 @@ type ops_metadata = | No_metadata_hash of operation_metadata list list | Metadata_hash of (operation_metadata * Operation_metadata_hash.t) list list +module Shell_header_hash : S.HASH + type result = { + shell_header_hash : Shell_header_hash.t; + (** This field is used as a (local) unique identifier for blocks + in order to implement the preapply cache mechanism. *) validation_store : validation_store; block_metadata : bytes * Block_metadata_hash.t option; ops_metadata : ops_metadata; @@ -105,6 +110,8 @@ type apply_environment = { (** header of the predecessor block being validated *) predecessor_context : Tezos_protocol_environment.Context.t; (** context associated to the predecessor block *) + predecessor_resulting_context_hash : Context_hash.t; + (** predecessor block resulting context hash *) predecessor_block_metadata_hash : Block_metadata_hash.t option; (** hash of block header metadata of the predecessor block *) predecessor_ops_metadata_hash : Operation_metadata_list_list_hash.t option; @@ -141,9 +148,10 @@ val apply : apply_result tzresult Lwt.t (** [precheck chain_id ~predecessor_block_header - ~predecessor_block_hash ~predecessor_context ~cache header ops] - gets the protocol [P] of the context of the predecessor block and - calls successively: + ~predecessor_block_hash ~predecessor_context + ~predecessor_resulting_context_hash ~cache header ops] gets the + protocol [P] of the context of the predecessor block and calls + successively: 1. [P.begin_validate] 2. [P.validate_operation] 3. [P.finalize_validation] *) @@ -152,6 +160,7 @@ val precheck : predecessor_block_header:Block_header.t -> predecessor_block_hash:Block_hash.t -> predecessor_context:Tezos_protocol_environment.Context.t -> + predecessor_resulting_context_hash:Context_hash.t -> cache:Tezos_protocol_environment.Context.source_of_cache -> Block_header.t -> Operation.t list list -> @@ -167,6 +176,7 @@ val preapply : live_blocks:Block_hash.Set.t -> live_operations:Operation_hash.Set.t -> predecessor_context:Tezos_protocol_environment.Context.t -> + predecessor_resulting_context_hash:Context_hash.t -> predecessor_shell_header:Block_header.shell_header -> predecessor_hash:Block_hash.t -> predecessor_max_operations_ttl:int -> diff --git a/src/lib_validation/external_validation.ml b/src/lib_validation/external_validation.ml index 668dd7b856be..d531d05bfef1 100644 --- a/src/lib_validation/external_validation.ml +++ b/src/lib_validation/external_validation.ml @@ -43,6 +43,7 @@ type request = predecessor_block_metadata_hash : Block_metadata_hash.t option; predecessor_ops_metadata_hash : Operation_metadata_list_list_hash.t option; + predecessor_resulting_context_hash : Context_hash.t; operations : Operation.t list list; max_operations_ttl : int; should_precheck : bool; @@ -60,12 +61,14 @@ type request = predecessor_block_metadata_hash : Block_metadata_hash.t option; predecessor_ops_metadata_hash : Operation_metadata_list_list_hash.t option; + predecessor_resulting_context_hash : Context_hash.t; operations : Operation.t list list; } | Precheck of { chain_id : Chain_id.t; predecessor_block_header : Block_header.t; predecessor_block_hash : Block_hash.t; + predecessor_resulting_context_hash : Context_hash.t; header : Block_header.t; operations : Operation.t list list; hash : Block_hash.t; @@ -178,12 +181,13 @@ let case_validate tag = case tag ~title:"validate" - (obj9 + (obj10 (req "chain_id" Chain_id.encoding) (req "block_header" (dynamic_size Block_header.encoding)) (req "pred_header" (dynamic_size Block_header.encoding)) (opt "pred_block_metadata_hash" Block_metadata_hash.encoding) (opt "pred_ops_metadata_hash" Operation_metadata_list_list_hash.encoding) + (req "predecessor_resulting_context_hash" Context_hash.encoding) (req "max_operations_ttl" int31) (req "operations" (list (list (dynamic_size Operation.encoding)))) (req "should_precheck" bool) @@ -196,6 +200,7 @@ let case_validate tag = predecessor_block_header; predecessor_block_metadata_hash; predecessor_ops_metadata_hash; + predecessor_resulting_context_hash; max_operations_ttl; operations; should_precheck; @@ -207,6 +212,7 @@ let case_validate tag = predecessor_block_header, predecessor_block_metadata_hash, predecessor_ops_metadata_hash, + predecessor_resulting_context_hash, max_operations_ttl, operations, should_precheck, @@ -217,6 +223,7 @@ let case_validate tag = predecessor_block_header, predecessor_block_metadata_hash, predecessor_ops_metadata_hash, + predecessor_resulting_context_hash, max_operations_ttl, operations, should_precheck, @@ -228,6 +235,7 @@ let case_validate tag = predecessor_block_header; predecessor_block_metadata_hash; predecessor_ops_metadata_hash; + predecessor_resulting_context_hash; max_operations_ttl; operations; should_precheck; @@ -253,7 +261,9 @@ let case_preapply tag = (opt "predecessor_ops_metadata_hash" Operation_metadata_list_list_hash.encoding)) - (obj1 (req "operations" (list (list (dynamic_size Operation.encoding)))))) + (obj2 + (req "predecessor_resulting_context_hash" Context_hash.encoding) + (req "operations" (list (list (dynamic_size Operation.encoding)))))) (function | Preapply { @@ -267,6 +277,7 @@ let case_preapply tag = predecessor_max_operations_ttl; predecessor_block_metadata_hash; predecessor_ops_metadata_hash; + predecessor_resulting_context_hash; operations; } -> Some @@ -280,7 +291,7 @@ let case_preapply tag = predecessor_max_operations_ttl, predecessor_block_metadata_hash, predecessor_ops_metadata_hash ), - operations ) + (predecessor_resulting_context_hash, operations) ) | _ -> None) (fun ( ( chain_id, timestamp, @@ -292,7 +303,7 @@ let case_preapply tag = predecessor_max_operations_ttl, predecessor_block_metadata_hash, predecessor_ops_metadata_hash ), - operations ) -> + (predecessor_resulting_context_hash, operations) ) -> Preapply { chain_id; @@ -305,6 +316,7 @@ let case_preapply tag = predecessor_max_operations_ttl; predecessor_block_metadata_hash; predecessor_ops_metadata_hash; + predecessor_resulting_context_hash; operations; }) @@ -313,10 +325,11 @@ let case_precheck tag = case tag ~title:"precheck" - (obj6 + (obj7 (req "chain_id" Chain_id.encoding) (req "predecessor_block_header" (dynamic_size Block_header.encoding)) (req "predecessor_block_hash" Block_hash.encoding) + (req "predecessor_resulting_context_hash" Context_hash.encoding) (req "header" (dynamic_size Block_header.encoding)) (req "hash" Block_hash.encoding) (req "operations" (list (list (dynamic_size Operation.encoding))))) @@ -326,6 +339,7 @@ let case_precheck tag = chain_id; predecessor_block_header; predecessor_block_hash; + predecessor_resulting_context_hash; header; operations; hash; @@ -334,6 +348,7 @@ let case_precheck tag = ( chain_id, predecessor_block_header, predecessor_block_hash, + predecessor_resulting_context_hash, header, hash, operations ) @@ -341,6 +356,7 @@ let case_precheck tag = (fun ( chain_id, predecessor_block_header, predecessor_block_hash, + predecessor_resulting_context_hash, header, hash, operations ) -> @@ -349,6 +365,7 @@ let case_precheck tag = chain_id; predecessor_block_header; predecessor_block_hash; + predecessor_resulting_context_hash; header; operations; hash; diff --git a/src/lib_validation/external_validation.mli b/src/lib_validation/external_validation.mli index d1bb6afe0016..982219d38c3a 100644 --- a/src/lib_validation/external_validation.mli +++ b/src/lib_validation/external_validation.mli @@ -43,6 +43,7 @@ type request = predecessor_block_metadata_hash : Block_metadata_hash.t option; predecessor_ops_metadata_hash : Operation_metadata_list_list_hash.t option; + predecessor_resulting_context_hash : Context_hash.t; operations : Operation.t list list; max_operations_ttl : int; should_precheck : bool; @@ -60,12 +61,14 @@ type request = predecessor_block_metadata_hash : Block_metadata_hash.t option; predecessor_ops_metadata_hash : Operation_metadata_list_list_hash.t option; + predecessor_resulting_context_hash : Context_hash.t; operations : Operation.t list list; } | Precheck of { chain_id : Chain_id.t; predecessor_block_header : Block_header.t; predecessor_block_hash : Block_hash.t; + predecessor_resulting_context_hash : Context_hash.t; header : Block_header.t; operations : Operation.t list list; hash : Block_hash.t; diff --git a/src/lib_validation/external_validator.ml b/src/lib_validation/external_validator.ml index de6c6988645f..c7bdd5e64943 100644 --- a/src/lib_validation/external_validator.ml +++ b/src/lib_validation/external_validator.ml @@ -257,6 +257,7 @@ let run ~readonly input output = predecessor_block_header; predecessor_block_metadata_hash; predecessor_ops_metadata_hash; + predecessor_resulting_context_hash; operations; max_operations_ttl; should_precheck; @@ -266,16 +267,17 @@ let run ~readonly input output = let*! block_application_result = let* predecessor_context = Error_monad.catch_es (fun () -> - let pred_context_hash = - predecessor_block_header.shell.context + let*! o = + Context.checkout + context_index + predecessor_resulting_context_hash in - let*! o = Context.checkout context_index pred_context_hash in match o with | Some c -> return (Shell_context.wrap_disk_context c) | None -> tzfail (Block_validator_errors.Failed_to_checkout_context - pred_context_hash)) + predecessor_resulting_context_hash)) in let*! protocol_hash = Context_ops.get_protocol predecessor_context in let* () = load_protocol protocol_hash protocol_root in @@ -290,13 +292,14 @@ let run ~readonly input output = predecessor_block_metadata_hash; predecessor_ops_metadata_hash; predecessor_context; + predecessor_resulting_context_hash; } in - let predecessor_context = predecessor_block_header.shell.context in let cache = match cache with | None -> `Load - | Some cache -> `Inherited (cache, predecessor_context) + | Some cache -> + `Inherited (cache, predecessor_resulting_context_hash) in with_retry_to_load_protocol protocol_root (fun () -> Block_validation.apply @@ -320,7 +323,11 @@ let run ~readonly input output = | Error _ as err -> (err, cache) | Ok {result; cache} -> ( Ok result, - Some {context_hash = block_header.shell.context; cache} ) + Some + { + context_hash = result.validation_store.resulting_context_hash; + cache; + } ) in let*! () = External_validation.send @@ -341,14 +348,16 @@ let run ~readonly input output = predecessor_max_operations_ttl; predecessor_block_metadata_hash; predecessor_ops_metadata_hash; + predecessor_resulting_context_hash; operations; } -> let*! block_preapplication_result = let* predecessor_context = Error_monad.catch_es (fun () -> - let pred_context_hash = predecessor_shell_header.context in let*! context = - Context.checkout context_index pred_context_hash + Context.checkout + context_index + predecessor_resulting_context_hash in match context with | Some context -> @@ -356,7 +365,7 @@ let run ~readonly input output = | None -> tzfail (Block_validator_errors.Failed_to_checkout_context - pred_context_hash)) + predecessor_resulting_context_hash)) in let*! protocol_hash = Context_ops.get_protocol predecessor_context in let* () = load_protocol protocol_hash protocol_root in @@ -376,6 +385,7 @@ let run ~readonly input output = ~predecessor_max_operations_ttl ~predecessor_block_metadata_hash ~predecessor_ops_metadata_hash + ~predecessor_resulting_context_hash operations) in let*! cachable_result = @@ -405,6 +415,7 @@ let run ~readonly input output = chain_id; predecessor_block_header; predecessor_block_hash; + predecessor_resulting_context_hash; header; operations; hash; @@ -416,7 +427,7 @@ let run ~readonly input output = let*! o = Context.checkout context_index - predecessor_block_header.shell.context + predecessor_resulting_context_hash in match o with | Some context -> @@ -424,19 +435,20 @@ let run ~readonly input output = | None -> tzfail (Block_validator_errors.Failed_to_checkout_context - predecessor_block_header.shell.context)) + predecessor_resulting_context_hash)) in let cache = match cache with | None -> `Lazy | Some cache -> - `Inherited (cache, predecessor_block_header.shell.context) + `Inherited (cache, predecessor_resulting_context_hash) in Block_validation.precheck ~chain_id ~predecessor_block_header ~predecessor_block_hash ~predecessor_context + ~predecessor_resulting_context_hash ~cache header operations -- GitLab From e2e4b861efbad6b83fb6a5edc77a99f61f40d7ea Mon Sep 17 00:00:00 2001 From: vbot Date: Thu, 10 Nov 2022 10:00:18 +0100 Subject: [PATCH 10/19] Shell: adapt to new validation semantics --- src/lib_shell/block_validator.ml | 38 +++++++++----- src/lib_shell/block_validator.mli | 12 ++++- src/lib_shell/block_validator_process.ml | 67 ++++++++++++++++++------ src/lib_shell/bootstrap_pipeline.ml | 2 +- src/lib_shell/bootstrap_pipeline.mli | 2 +- src/lib_shell/chain_validator.ml | 34 +++++++++--- src/lib_shell/peer_validator.ml | 6 +-- src/lib_shell/peer_validator.mli | 2 +- src/lib_shell/protocol_validator.ml | 13 ++++- 9 files changed, 130 insertions(+), 46 deletions(-) diff --git a/src/lib_shell/block_validator.ml b/src/lib_shell/block_validator.ml index 90e6f69a93e2..3386dbbf1c2f 100644 --- a/src/lib_shell/block_validator.ml +++ b/src/lib_shell/block_validator.ml @@ -29,7 +29,7 @@ open Block_validator_worker_state open Block_validator_errors type validation_result = - | Already_commited + | Already_committed | Outdated_block | Validated | Validation_error of error trace @@ -38,6 +38,11 @@ type validation_result = | Validation_error_after_precheck of error trace | Precheck_failed of error trace +type new_block = { + block : Store.Block.t; + resulting_context_hash : Tezos_crypto.Context_hash.t; +} + module Block_hash_ring = Aches.Vache.Map (Aches.Vache.FIFO_Precise) (Aches.Vache.Strong) (Tezos_crypto.Block_hash) @@ -75,7 +80,7 @@ module Request = struct type validation_request = { chain_db : Distributed_db.chain_db; - notify_new_block : Store.Block.t -> unit; + notify_new_block : new_block -> unit; canceler : Lwt_canceler.t option; peer : P2p_peer.Id.t option; hash : Tezos_crypto.Block_hash.t; @@ -175,7 +180,7 @@ let on_validation_request w let chain_store = Distributed_db.chain_store chain_db in let*! b = Store.Block.is_known_valid chain_store hash in match b with - | true -> return Already_commited + | true -> return Already_committed | false -> ( let*! r = match @@ -266,9 +271,14 @@ let on_validation_request w in match o with | Some block -> - notify_new_block block ; + notify_new_block + { + block; + resulting_context_hash = + result.validation_store.context_hash; + } ; return Validated - | None -> return Already_commited))) + | None -> return Already_committed))) in match r with | Ok r -> return r @@ -363,7 +373,7 @@ let on_completion : let open Lwt_syntax in Prometheus.Counter.inc_one metrics.worker_counters.worker_completion_count ; match (request, v) with - | Request.Request_validation {hash; _}, Already_commited -> + | Request.Request_validation {hash; _}, Already_committed -> Prometheus.Counter.inc_one metrics.already_commited_blocks_count ; let* () = Events.(emit previously_validated) hash in Lwt.return_unit @@ -452,14 +462,14 @@ type block_validity = let validate w ?canceler ?peer ?(notify_new_block = fun _ -> ()) ?(precheck_and_notify = false) chain_db hash (header : Block_header.t) - operations : block_validity Lwt.t = + operations = let open Lwt_syntax in let chain_store = Distributed_db.chain_store chain_db in let* b = Store.Block.is_known_valid chain_store hash in match b with | true -> let* () = Events.(emit previously_validated) hash in - Lwt.return Valid + return Valid | false -> ( let* r = let open Lwt_result_syntax in @@ -501,16 +511,16 @@ let validate w ?canceler ?peer ?(notify_new_block = fun _ -> ()) }) in match r with - | Ok (Validated | Already_commited | Outdated_block) -> Lwt.return Valid + | Ok (Validated | Already_committed | Outdated_block) -> return Valid | Ok (Validation_error_after_precheck errs) -> - Lwt.return (Invalid_after_precheck errs) + return (Invalid_after_precheck errs) | Ok (Precheck_failed errs) | Ok (Validation_error errs) | Error (Request_error errs) -> - Lwt.return (Invalid errs) - | Error (Closed None) -> Lwt.return (Invalid [Worker_types.Terminated]) - | Error (Closed (Some errs)) -> Lwt.return (Invalid errs) - | Error (Any exn) -> Lwt.return (Invalid [Exn exn]) + return (Invalid errs) + | Error (Closed None) -> return (Invalid [Worker_types.Terminated]) + | Error (Closed (Some errs)) -> return (Invalid errs) + | Error (Any exn) -> return (Invalid [Exn exn]) | _ -> (* preapplication cases *) assert false) diff --git a/src/lib_shell/block_validator.mli b/src/lib_shell/block_validator.mli index 6f308f91e339..9e22318c9d2e 100644 --- a/src/lib_shell/block_validator.mli +++ b/src/lib_shell/block_validator.mli @@ -28,6 +28,16 @@ type t +(** Type of a validated block *) +type new_block = { + block : Store.Block.t; (** The block itself. *) + resulting_context_hash : Tezos_crypto.Context_hash.t; + (** The context hash resulting of [block]'s application. + + It may be the same one as contained in its header depending + on the protocol expected semantics. *) +} + (** [create limits ddb bvp start_testchain] creates a [Block_validator]. @@ -95,7 +105,7 @@ val validate : t -> ?canceler:Lwt_canceler.t -> ?peer:P2p_peer.Id.t -> - ?notify_new_block:(Store.Block.t -> unit) -> + ?notify_new_block:(new_block -> unit) -> ?precheck_and_notify:bool -> Distributed_db.chain_db -> Tezos_crypto.Block_hash.t -> diff --git a/src/lib_shell/block_validator_process.ml b/src/lib_shell/block_validator_process.ml index 3c54d4552824..d8760910dab6 100644 --- a/src/lib_shell/block_validator_process.ml +++ b/src/lib_shell/block_validator_process.ml @@ -74,6 +74,7 @@ module type S = sig predecessor_block_metadata_hash:Tezos_crypto.Block_metadata_hash.t option -> predecessor_ops_metadata_hash: Tezos_crypto.Operation_metadata_list_list_hash.t option -> + predecessor_resulting_context_hash:Tezos_crypto.Context_hash.t -> Operation.t list list -> (Block_header.shell_header * error Preapply_result.t list) tzresult Lwt.t @@ -205,14 +206,19 @@ module Internal_validator_process = struct let open Lwt_result_syntax in let chain_id = Store.Chain.chain_id chain_store in let predecessor_block_header = Store.Block.header predecessor in - let context_hash = predecessor_block_header.shell.context in + let* predecessor_resulting_context_hash = + Store.Block.resulting_context_hash chain_store predecessor + in let context_index = get_context_index chain_store in let* predecessor_context = - let*! o = Context_ops.checkout context_index context_hash in + let*! o = + Context_ops.checkout context_index predecessor_resulting_context_hash + in match o with | None -> tzfail - (Block_validator_errors.Failed_to_checkout_context context_hash) + (Block_validator_errors.Failed_to_checkout_context + predecessor_resulting_context_hash) | Some ctx -> return ctx in let predecessor_block_metadata_hash = @@ -229,6 +235,7 @@ module Internal_validator_process = struct predecessor_block_metadata_hash; predecessor_ops_metadata_hash; predecessor_context; + predecessor_resulting_context_hash; user_activated_upgrades; user_activated_protocol_overrides; operation_metadata_size_limit; @@ -246,12 +253,15 @@ module Internal_validator_process = struct in let now = Time.System.now () in let block_hash = Block_header.hash block_header in - let predecessor_context_hash = Store.Block.context_hash predecessor in + let predecessor_resulting_context_hash = + env.predecessor_resulting_context_hash + in let*! () = Events.(emit validation_request (block_hash, env.chain_id)) in let cache = match validator.cache with | None -> `Load - | Some block_cache -> `Inherited (block_cache, predecessor_context_hash) + | Some block_cache -> + `Inherited (block_cache, predecessor_resulting_context_hash) in let* {result; cache} = Block_validation.apply @@ -267,7 +277,8 @@ module Internal_validator_process = struct let then_ = Time.System.now () in Ptime.diff then_ now in - validator.cache <- Some {context_hash = block_header.shell.context; cache} ; + validator.cache <- + Some {context_hash = predecessor_resulting_context_hash; cache} ; validator.preapply_result <- None ; let*! () = Events.(emit validation_success (block_hash, timespan)) in return result @@ -275,18 +286,21 @@ module Internal_validator_process = struct let preapply_block validator ~chain_id ~timestamp ~protocol_data ~live_blocks ~live_operations ~predecessor_shell_header ~predecessor_hash ~predecessor_max_operations_ttl ~predecessor_block_metadata_hash - ~predecessor_ops_metadata_hash operations = + ~predecessor_ops_metadata_hash ~predecessor_resulting_context_hash + operations = let open Lwt_result_syntax in let context_index = Store.context_index (Store.Chain.global_store validator.chain_store) in - let context_hash = predecessor_shell_header.Block_header.context in let* predecessor_context = - let*! o = Context_ops.checkout context_index context_hash in + let*! o = + Context_ops.checkout context_index predecessor_resulting_context_hash + in match o with | None -> tzfail - (Block_validator_errors.Failed_to_checkout_context context_hash) + (Block_validator_errors.Failed_to_checkout_context + predecessor_resulting_context_hash) | Some ctx -> return ctx in let user_activated_upgrades = validator.user_activated_upgrades in @@ -307,6 +321,7 @@ module Internal_validator_process = struct ~live_blocks ~live_operations ~predecessor_context + ~predecessor_resulting_context_hash ~predecessor_shell_header ~predecessor_hash ~predecessor_max_operations_ttl @@ -325,20 +340,25 @@ module Internal_validator_process = struct Store.context_index (Store.Chain.global_store validator.chain_store) in let predecessor_block_header = Store.Block.header predecessor in - let context_hash = predecessor_block_header.Block_header.shell.context in + let* predecessor_resulting_context_hash = + Store.Block.resulting_context_hash chain_store predecessor + in let* predecessor_context = - let*! o = Context_ops.checkout context_index context_hash in + let*! o = + Context_ops.checkout context_index predecessor_resulting_context_hash + in match o with | None -> tzfail - (Block_validator_errors.Failed_to_checkout_context context_hash) + (Block_validator_errors.Failed_to_checkout_context + predecessor_resulting_context_hash) | Some ctx -> return ctx in let cache = match validator.cache with | None -> `Lazy | Some block_cache -> - `Inherited (block_cache, predecessor_block_header.shell.context) + `Inherited (block_cache, predecessor_resulting_context_hash) in let predecessor_block_hash = Store.Block.hash predecessor in Block_validation.precheck @@ -346,6 +366,7 @@ module Internal_validator_process = struct ~predecessor_block_header ~predecessor_block_hash ~predecessor_context + ~predecessor_resulting_context_hash ~cache header operations @@ -764,6 +785,7 @@ module External_validator_process = struct let apply_block ~simulate ?(should_precheck = true) validator chain_store ~predecessor ~max_operations_ttl block_header operations = + let open Lwt_result_syntax in let chain_id = Store.Chain.chain_id chain_store in let predecessor_block_header = Store.Block.header predecessor in let predecessor_block_metadata_hash = @@ -772,6 +794,9 @@ module External_validator_process = struct let predecessor_ops_metadata_hash = Store.Block.all_operations_metadata_hash predecessor in + let* predecessor_resulting_context_hash = + Store.Block.resulting_context_hash chain_store predecessor + in let request = External_validation.Validate { @@ -780,6 +805,7 @@ module External_validator_process = struct predecessor_block_header; predecessor_block_metadata_hash; predecessor_ops_metadata_hash; + predecessor_resulting_context_hash; operations; max_operations_ttl; should_precheck; @@ -791,7 +817,8 @@ module External_validator_process = struct let preapply_block validator ~chain_id ~timestamp ~protocol_data ~live_blocks ~live_operations ~predecessor_shell_header ~predecessor_hash ~predecessor_max_operations_ttl ~predecessor_block_metadata_hash - ~predecessor_ops_metadata_hash operations = + ~predecessor_ops_metadata_hash ~predecessor_resulting_context_hash + operations = let request = External_validation.Preapply { @@ -805,21 +832,27 @@ module External_validator_process = struct predecessor_max_operations_ttl; predecessor_block_metadata_hash; predecessor_ops_metadata_hash; + predecessor_resulting_context_hash; operations; } in send_request validator request Block_validation.preapply_result_encoding let precheck_block validator chain_store ~predecessor header hash operations = + let open Lwt_result_syntax in let chain_id = Store.Chain.chain_id chain_store in let predecessor_block_header = Store.Block.header predecessor in let predecessor_block_hash = Store.Block.hash predecessor in + let* predecessor_resulting_context_hash = + Store.Block.resulting_context_hash chain_store predecessor + in let request = External_validation.Precheck { chain_id; predecessor_block_header; predecessor_block_hash; + predecessor_resulting_context_hash; header; operations; hash; @@ -1015,6 +1048,9 @@ let preapply_block (E {validator_process = (module VP); validator} : t) let predecessor_max_operations_ttl = Store.Block.max_operations_ttl metadata in + let* predecessor_resulting_context_hash = + Store.Block.resulting_context_hash chain_store predecessor + in VP.preapply_block validator ~chain_id @@ -1027,4 +1063,5 @@ let preapply_block (E {validator_process = (module VP); validator} : t) ~predecessor_max_operations_ttl ~predecessor_block_metadata_hash ~predecessor_ops_metadata_hash + ~predecessor_resulting_context_hash operations diff --git a/src/lib_shell/bootstrap_pipeline.ml b/src/lib_shell/bootstrap_pipeline.ml index ca15e74d3be5..79d4d3fa8247 100644 --- a/src/lib_shell/bootstrap_pipeline.ml +++ b/src/lib_shell/bootstrap_pipeline.ml @@ -126,7 +126,7 @@ type t = { chain_db : Distributed_db.chain_db; locator : Block_locator.t; block_validator : Block_validator.t; - notify_new_block : Store.Block.t -> unit; + notify_new_block : Block_validator.new_block -> unit; fetched_headers : (Tezos_crypto.Block_hash.t * Block_header.t) list Lwt_pipe.Bounded.t; fetched_blocks : diff --git a/src/lib_shell/bootstrap_pipeline.mli b/src/lib_shell/bootstrap_pipeline.mli index 4002286a8648..d1db889f712c 100644 --- a/src/lib_shell/bootstrap_pipeline.mli +++ b/src/lib_shell/bootstrap_pipeline.mli @@ -55,7 +55,7 @@ type t If a timeout is triggered, the whole [bootstrap_pipeline] is canceled.*) val create : - ?notify_new_block:(Store.Block.t -> unit) -> + ?notify_new_block:(Block_validator.new_block -> unit) -> block_header_timeout:Time.System.Span.t -> block_operations_timeout:Time.System.Span.t -> Block_validator.t -> diff --git a/src/lib_shell/chain_validator.ml b/src/lib_shell/chain_validator.ml index 5468186aaf80..5eab392f14fb 100644 --- a/src/lib_shell/chain_validator.ml +++ b/src/lib_shell/chain_validator.ml @@ -46,6 +46,7 @@ module Request = struct peer : P2p_peer_id.t option; (* The peer who sent the block if it was not injected locally. *) block : Store.Block.t; + resulting_context_hash : Tezos_crypto.Context_hash.t; } -> (update, error trace) t | Notify_branch : P2p_peer.Id.t * Block_locator.t -> (unit, Empty.t) t @@ -154,7 +155,7 @@ let check_and_update_synchronisation_state w (hash, block) peer_id : unit Lwt.t else Lwt.return_unit (* Called for every validated block. *) -let notify_new_block w peer block = +let notify_new_block w peer {Block_validator.block; resulting_context_hash} = let nv = Worker.state w in Option.iter (fun id -> @@ -165,7 +166,9 @@ let notify_new_block w peer block = nv.parameters.parent ; Lwt_watcher.notify nv.valid_block_input block ; Lwt_watcher.notify nv.parameters.global_valid_block_input block ; - Worker.Queue.push_request_now w (Validated {peer; block}) + Worker.Queue.push_request_now + w + (Validated {peer; block; resulting_context_hash}) let with_activated_peer_validator w peer_id f = let open Lwt_syntax in @@ -194,18 +197,30 @@ let with_activated_peer_validator w peer_id f = | Worker_types.Launching _ -> return_ok_unit) -let may_update_protocol_level chain_store ~block = +let may_update_protocol_level chain_store block resulting_context_hash = let open Lwt_result_syntax in let* pred = Store.Block.read_predecessor chain_store block in let prev_proto_level = Store.Block.proto_level pred in let new_proto_level = Store.Block.proto_level block in if Compare.Int.(prev_proto_level < new_proto_level) then - let* context = Store.Block.context chain_store block in - let*! new_protocol = Context_ops.get_protocol context in + let context_index = + Store.context_index (Store.Chain.global_store chain_store) + in + let* resulting_context = + protect (fun () -> + let*! c = + Context_ops.checkout_exn context_index resulting_context_hash + in + return c) + in + let*! new_protocol = Context_ops.get_protocol resulting_context in + let* (module NewProto) = Registered_protocol.get_result new_protocol in Store.Chain.may_update_protocol_level chain_store ~pred ~protocol_level:new_proto_level + ~expect_predecessor_context: + (NewProto.expected_context_hash = Predecessor_resulting_context) (block, new_protocol) else return_unit @@ -449,7 +464,7 @@ let may_synchronise_context synchronisation_state chain_store = else Lwt.return_unit let on_validation_request w peer start_testchain active_chains spawn_child block - = + resulting_context_hash = let open Lwt_result_syntax in let*! () = Option.iter_s @@ -475,7 +490,9 @@ let on_validation_request w peer start_testchain active_chains spawn_child block return Ignored_head | Some previous -> let*! () = broadcast_head w ~previous block in - let* () = may_update_protocol_level chain_store ~block in + let* () = + may_update_protocol_level chain_store block resulting_context_hash + in let*! () = if start_testchain then may_switch_test_chain w active_chains spawn_child block @@ -555,7 +572,7 @@ let on_request (type a b) w start_testchain active_chains spawn_child Prometheus.Counter.inc_one nv.parameters.metrics.worker_counters.worker_request_count ; match req with - | Request.Validated {peer; block} -> + | Request.Validated {peer; block; resulting_context_hash} -> on_validation_request w peer @@ -563,6 +580,7 @@ let on_request (type a b) w start_testchain active_chains spawn_child active_chains spawn_child block + resulting_context_hash | Request.Notify_branch (peer_id, locator) -> on_notify_branch w peer_id locator | Request.Notify_head (peer_id, hash, header, mempool) -> diff --git a/src/lib_shell/peer_validator.ml b/src/lib_shell/peer_validator.ml index c365749590a6..19667230c907 100644 --- a/src/lib_shell/peer_validator.ml +++ b/src/lib_shell/peer_validator.ml @@ -73,7 +73,7 @@ module Types = struct chain_db : Distributed_db.chain_db; block_validator : Block_validator.t; (* callback to chain_validator *) - notify_new_block : Store.Block.t -> unit; + notify_new_block : Block_validator.new_block -> unit; notify_termination : unit -> unit; limits : Shell_limits.peer_validator_limits; } @@ -482,9 +482,9 @@ let on_launch _ name parameters : (_, launch_error) result Lwt.t = last_validated_head = Store.Block.header genesis; last_advertised_head = Store.Block.header genesis; } - and notify_new_block block = + and notify_new_block ({block; _} as new_block) = pv.last_validated_head <- Store.Block.header block ; - parameters.notify_new_block block + parameters.notify_new_block new_block in Prometheus.Counter.inc_one metrics.connections ; Lwt.return (Ok pv) diff --git a/src/lib_shell/peer_validator.mli b/src/lib_shell/peer_validator.mli index 95e97943d8fb..409d2d93c0ed 100644 --- a/src/lib_shell/peer_validator.mli +++ b/src/lib_shell/peer_validator.mli @@ -29,7 +29,7 @@ type t val peer_id : t -> P2p_peer.Id.t val create : - ?notify_new_block:(Store.Block.t -> unit) -> + ?notify_new_block:(Block_validator.new_block -> unit) -> ?notify_termination:(unit -> unit) -> Shell_limits.peer_validator_limits -> Block_validator.t -> diff --git a/src/lib_shell/protocol_validator.ml b/src/lib_shell/protocol_validator.ml index 28974fdfaa57..a2c08b701d8d 100644 --- a/src/lib_shell/protocol_validator.ml +++ b/src/lib_shell/protocol_validator.ml @@ -167,17 +167,23 @@ let fetch_and_compile_protocols pv ?peer ?timeout (block : Store.Block.t) = let* context = Store.Block.context chain_store block in let protocol = let*! protocol_hash = Context_ops.get_protocol context in - let* _p = fetch_and_compile_protocol pv ?peer ?timeout protocol_hash in + let* (module Proto) = + fetch_and_compile_protocol pv ?peer ?timeout protocol_hash + in Store.Chain.may_update_protocol_level chain_store ~protocol_level + ~expect_predecessor_context: + (Proto.expected_context_hash = Predecessor_resulting_context) (block, protocol_hash) and test_protocol = let*! v = Context_ops.get_test_chain context in match v with | Not_running -> return_unit | Forking {protocol; _} | Running {protocol; _} -> ( - let* _ = fetch_and_compile_protocol pv ?peer ?timeout protocol in + let* (module Proto) = + fetch_and_compile_protocol pv ?peer ?timeout protocol + in let*! o = Store.Chain.testchain chain_store in match o with | None -> return_unit @@ -187,6 +193,9 @@ let fetch_and_compile_protocols pv ?peer ?timeout (block : Store.Block.t) = Store.Chain.may_update_protocol_level test_chain_store ~protocol_level + ~expect_predecessor_context: + (Proto.expected_context_hash + = Predecessor_resulting_context) (block, protocol) in return_unit) -- GitLab From bc8c15b73c27bcac701f3b5a4029cdfd1e153f87 Mon Sep 17 00:00:00 2001 From: vbot Date: Thu, 10 Nov 2022 10:05:27 +0100 Subject: [PATCH 11/19] Node: plug the automatic store upgrade --- manifest/main.ml | 1 + opam/octez-node-config.opam | 1 + src/bin_node/node_identity_file.ml | 12 +++++- src/bin_node/node_replay_command.ml | 8 +++- src/bin_node/node_run_command.ml | 7 +++- src/bin_node/node_snapshot_command.ml | 6 +-- src/lib_node_config/config_file.ml | 14 ++++++- src/lib_node_config/data_version.ml | 53 +++++++++++++++++++-------- src/lib_node_config/data_version.mli | 10 +++-- src/lib_node_config/dune | 2 + 10 files changed, 85 insertions(+), 29 deletions(-) diff --git a/manifest/main.ml b/manifest/main.ml index 6e23382c241a..d51eb9aa49dd 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -3281,6 +3281,7 @@ let octez_node_config = octez_rpc_http |> open_; octez_rpc_http_server |> open_; octez_context |> open_; + octez_store |> open_; octez_validation |> open_; ] diff --git a/opam/octez-node-config.opam b/opam/octez-node-config.opam index 4372a79c4761..09667a51908a 100644 --- a/opam/octez-node-config.opam +++ b/opam/octez-node-config.opam @@ -15,6 +15,7 @@ depends: [ "tezos-rpc-http" "tezos-rpc-http-server" "tezos-context" + "tezos-store" "tezos-validation" ] build: [ diff --git a/src/bin_node/node_identity_file.ml b/src/bin_node/node_identity_file.ml index 9e87ae5597ba..84303040b8de 100644 --- a/src/bin_node/node_identity_file.ml +++ b/src/bin_node/node_identity_file.ml @@ -183,8 +183,18 @@ let write file identity = let open Lwt_result_syntax in if Sys.file_exists file then tzfail (Existent_identity_file file) else + let dummy_genesis = + { + Genesis.time = Time.Protocol.epoch; + block = Tezos_crypto.Block_hash.zero; + protocol = Tezos_crypto.Protocol_hash.zero; + } + in let* () = - Data_version.ensure_data_dir ~mode:Exists (Filename.dirname file) + Data_version.ensure_data_dir + ~mode:Exists + dummy_genesis + (Filename.dirname file) in Lwt_utils_unix.Json.write_file file diff --git a/src/bin_node/node_replay_command.ml b/src/bin_node/node_replay_command.ml index 49fc80837817..e3ee947466cc 100644 --- a/src/bin_node/node_replay_command.ml +++ b/src/bin_node/node_replay_command.ml @@ -230,7 +230,7 @@ let replay_one_block strict main_chain_store validator_process block = let* protocol = Store.Block.protocol_hash main_chain_store block in let* (module Proto) = Registered_protocol.get_result protocol in let* expected_context_hash = - if Proto.expected_context_hash = `Current then + if Proto.expected_context_hash = Resulting_context then return (Store.Block.context_hash block) else Store.Block.resulting_context_hash main_chain_store block in @@ -440,7 +440,11 @@ let replay ~singleprocess ~strict (config : Config_file.t) blocks = let run ?verbosity ~singleprocess ~strict (config : Config_file.t) blocks = let open Lwt_result_syntax in - let* () = Data_version.ensure_data_dir config.data_dir in + let* () = + Data_version.ensure_data_dir + config.blockchain_network.genesis + config.data_dir + in Lwt_lock_file.try_with_lock ~when_locked:(fun () -> failwith "Data directory is locked by another process") diff --git a/src/bin_node/node_run_command.ml b/src/bin_node/node_run_command.ml index 3a8f7167cb89..6b1ef98e884e 100644 --- a/src/bin_node/node_run_command.ml +++ b/src/bin_node/node_run_command.ml @@ -520,7 +520,12 @@ let run ?verbosity ?sandbox ?target ?(cli_warnings = []) let*! () = Lwt_list.iter_s (fun evt -> Internal_event.Simple.emit evt ()) cli_warnings in - let* () = Data_version.ensure_data_dir ~mode:Is_compatible config.data_dir in + let* () = + Data_version.ensure_data_dir + ~mode:Is_compatible + config.blockchain_network.genesis + config.data_dir + in let* () = Config_validation.check ?ignore_testchain_warning config in let* identity = init_identity_file config in Updater.init (Data_version.protocol_dir config.data_dir) ; diff --git a/src/bin_node/node_snapshot_command.ml b/src/bin_node/node_snapshot_command.ml index 3ed876e3c0c9..be3cf7a6b1c7 100644 --- a/src/bin_node/node_snapshot_command.ml +++ b/src/bin_node/node_snapshot_command.ml @@ -161,7 +161,7 @@ module Term = struct Lwt.return data_dir | None -> Lwt.return node_config.data_dir in - let* () = Data_version.ensure_data_dir data_dir in + let* () = Data_version.ensure_data_dir genesis data_dir in let context_dir = Data_version.context_dir data_dir in let store_dir = Data_version.store_dir data_dir in let* block = @@ -228,7 +228,7 @@ module Term = struct let*! () = if existing_data_dir then (* Remove only context and store if the import directory - was previously existing. *) + was previously existing. *) let*! () = Lwt_utils_unix.remove_dir (Data_version.store_dir data_dir) in @@ -239,7 +239,7 @@ module Term = struct let*! lock_file_exists = Lwt_unix.file_exists lock_file in if lock_file_exists then Lwt_unix.unlink lock_file else Lwt.return_unit in - let* () = Data_version.ensure_data_dir ~mode:Is_bare data_dir in + let* () = Data_version.ensure_data_dir ~mode:Is_bare genesis data_dir in (* Lock only on snapshot import *) Lwt_lock_file.try_with_lock ~when_locked:(fun () -> diff --git a/src/lib_node_config/config_file.ml b/src/lib_node_config/config_file.ml index 95328af4aa3f..908361755329 100644 --- a/src/lib_node_config/config_file.ml +++ b/src/lib_node_config/config_file.ml @@ -851,7 +851,12 @@ let read fp = let write fp cfg = let open Lwt_result_syntax in - let* () = Data_version.ensure_data_dir ~mode:Exists (Filename.dirname fp) in + let* () = + Data_version.ensure_data_dir + ~mode:Exists + cfg.blockchain_network.genesis + (Filename.dirname fp) + in Lwt_utils_unix.Json.write_file fp (Data_encoding.Json.construct encoding cfg) let to_string cfg = @@ -879,7 +884,12 @@ let update ?(disable_config_validation = false) ?data_dir ?min_connections Event.(emit all_rpc_allowed allow_all_rpc) else Lwt.return_unit in - let* () = Data_version.ensure_data_dir ~mode:Exists data_dir in + let* () = + Data_version.ensure_data_dir + ~mode:Exists + cfg.blockchain_network.genesis + data_dir + in let peer_table_size = Option.map (fun i -> (i, i / 4 * 3)) peer_table_size in let unopt_list ~default = function [] -> default | l -> l in let limits : Tezos_p2p_services.P2p_limits.t = diff --git a/src/lib_node_config/data_version.ml b/src/lib_node_config/data_version.ml index 3a4b43f50dc2..6a62bb5bb8b1 100644 --- a/src/lib_node_config/data_version.ml +++ b/src/lib_node_config/data_version.ml @@ -49,10 +49,12 @@ module Version = struct let c = Int.compare v1.major v2.major in if c <> 0 then c else Int.compare v1.minor v2.minor + let ( < ) v1 v2 = compare v1 v2 < 0 + let equal v1 v2 = compare v1 v2 = 0 let make ~major ~minor = - if major < 0 || minor < 0 then + if Compare.Int.(major < 0 || minor < 0) then invalid_arg (Printf.sprintf "Version.make: version number cannot be negative: %d.%d" @@ -97,13 +99,12 @@ end * - (0.)0.7 : new store metadata representation * - (0.)0.8 : context upgrade (upgrade to irmin.3.0) * - 1.0 : context upgrade (upgrade to irmin.3.3) - * - 2.0 : introduce context GC (upgrade to irmin.3.4) *) + * - 2.0 : introduce context GC (upgrade to irmin.3.4) + * - 3.0 : change blocks' context hash semantics *) (* FIXME https://gitlab.com/tezos/tezos/-/issues/2861 We should enable the semantic versioning instead of applying hardcoded rules.*) -let current_version = Version.make ~major:2 ~minor:0 - let v_0_6 = Version.make ~major:0 ~minor:6 let v_0_7 = Version.make ~major:0 ~minor:7 @@ -112,6 +113,12 @@ let v_0_8 = Version.make ~major:0 ~minor:8 let v_1_0 = Version.make ~major:1 ~minor:0 +let v_2_0 = Version.make ~major:2 ~minor:0 + +let v_3_0 = Version.make ~major:3 ~minor:0 + +let current_version = v_3_0 + (* List of upgrade functions from each still supported previous version to the current [data_version] above. If this list grows too much, an idea would be to have triples (version, version, @@ -127,17 +134,29 @@ let upgradable_data_version = let*! () = Context.close ctxt in return_unit in + let v_3_0_upgrade ~data_dir genesis = + let store_dir = store_dir data_dir in + Store.v_3_0_upgrade ~store_dir genesis + in [ ( v_0_6, - fun ~data_dir _ ~chain_name:_ ~sandbox_parameters:_ -> - v_1_0_upgrade ~data_dir ); + fun ~data_dir genesis ~chain_name:_ ~sandbox_parameters:_ -> + let* () = v_1_0_upgrade ~data_dir in + v_3_0_upgrade ~data_dir genesis ); ( v_0_7, - fun ~data_dir _ ~chain_name:_ ~sandbox_parameters:_ -> - v_1_0_upgrade ~data_dir ); + fun ~data_dir genesis ~chain_name:_ ~sandbox_parameters:_ -> + let* () = v_1_0_upgrade ~data_dir in + v_3_0_upgrade ~data_dir genesis ); ( v_0_8, - fun ~data_dir _ ~chain_name:_ ~sandbox_parameters:_ -> - v_1_0_upgrade ~data_dir ); - (v_1_0, fun ~data_dir:_ _ ~chain_name:_ ~sandbox_parameters:_ -> return_unit); + fun ~data_dir genesis ~chain_name:_ ~sandbox_parameters:_ -> + let* () = v_1_0_upgrade ~data_dir in + v_3_0_upgrade ~data_dir genesis ); + ( v_1_0, + fun ~data_dir genesis ~chain_name:_ ~sandbox_parameters:_ -> + v_3_0_upgrade ~data_dir genesis ); + ( v_2_0, + fun ~data_dir genesis ~chain_name:_ ~sandbox_parameters:_ -> + v_3_0_upgrade ~data_dir genesis ); ] type error += Invalid_data_dir_version of Version.t * Version.t @@ -423,7 +442,7 @@ let upgrade_data_dir ~data_dir genesis ~chain_name ~sandbox_parameters = let*! () = Events.(emit aborting_upgrade e) in Lwt.return (Error e)) -let ensure_data_dir ?(mode = Is_compatible) data_dir = +let ensure_data_dir ?(mode = Is_compatible) genesis data_dir = let open Lwt_result_syntax in let* o = ensure_data_dir ~mode data_dir in match o with @@ -432,18 +451,20 @@ let ensure_data_dir ?(mode = Is_compatible) data_dir = (* Enable automatic upgrade to avoid users to manually upgrade. *) | Some (version, _) when Version.( - equal version v_1_0 || equal version v_0_6 || equal version v_0_7 - || equal version v_0_8) -> + equal version v_2_0 || equal version v_1_0 || equal version v_0_6 + || equal version v_0_7 || equal version v_0_8) -> let* () = - upgrade_data_dir ~data_dir () ~chain_name:() ~sandbox_parameters:() + upgrade_data_dir ~data_dir genesis ~chain_name:() ~sandbox_parameters:() in let*! () = Events.(emit finished_upgrading_node (version, current_version)) in return_unit - | Some (version, _) -> + | Some (version, _) when Version.(version < current_version) -> tzfail (Data_dir_needs_upgrade {expected = current_version; actual = version}) + | Some (version, _) -> + tzfail (Invalid_data_dir_version (current_version, version)) let upgrade_status data_dir = let open Lwt_result_syntax in diff --git a/src/lib_node_config/data_version.mli b/src/lib_node_config/data_version.mli index b06d7540c780..61e2d6521647 100644 --- a/src/lib_node_config/data_version.mli +++ b/src/lib_node_config/data_version.mli @@ -56,10 +56,12 @@ val default_peers_file_name : string *) type ensure_mode = Exists | Is_bare | Is_compatible -(** [ensure_data_dir ~mode] ensures that a directory is valid with - regards to the given [mode]. See [ensure_mode] for the mode's - properties. By default, the mode is set to [Is_compatible]. *) -val ensure_data_dir : ?mode:ensure_mode -> string -> unit tzresult Lwt.t +(** [ensure_data_dir ~mode genesis data_dir] ensures that a directory + is valid with regards to the given [mode]. See [ensure_mode] for + the mode's properties. By default, the mode is set to + [Is_compatible]. *) +val ensure_data_dir : + ?mode:ensure_mode -> Genesis.t -> string -> unit tzresult Lwt.t (** Upgrade data directory from an older version. diff --git a/src/lib_node_config/dune b/src/lib_node_config/dune index 1a152fe80a78..df6565210035 100644 --- a/src/lib_node_config/dune +++ b/src/lib_node_config/dune @@ -12,6 +12,7 @@ tezos-rpc-http tezos-rpc-http-server tezos-context + tezos-store tezos-validation) (flags (:standard) @@ -22,4 +23,5 @@ -open Tezos_rpc_http -open Tezos_rpc_http_server -open Tezos_context + -open Tezos_store -open Tezos_validation)) -- GitLab From 85161bc9eb2003c2f54e9ee4a2d7fb1c29b6ebae Mon Sep 17 00:00:00 2001 From: vbot Date: Mon, 14 Nov 2022 16:12:32 +0100 Subject: [PATCH 12/19] Shell/RPCs: add resulting_context_hash RPC --- src/lib_proxy/proxy_services.ml | 9 ++++++++- src/lib_shell/block_directory.ml | 9 ++++++++- src/lib_shell_services/block_services.ml | 11 +++++++++++ src/lib_shell_services/block_services.mli | 16 ++++++++++++++++ 4 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/lib_proxy/proxy_services.ml b/src/lib_proxy/proxy_services.ml index f46160d6fe69..21536ea420ac 100644 --- a/src/lib_proxy/proxy_services.ml +++ b/src/lib_proxy/proxy_services.ml @@ -179,6 +179,13 @@ let build_directory (printer : Tezos_client_base.Client_context.printer) ~block:block_key () in + let* resulting_context_hash = + Tezos_shell_services.Block_services.Empty.resulting_context_hash + rpc_context + ~chain + ~block:block_key + () + in let* protocols = get_protocols ?expected_protocol rpc_context chain block_key in @@ -197,7 +204,7 @@ let build_directory (printer : Tezos_client_base.Client_context.printer) } in let* initial_context = - Proxy_environment.initial_context ctx block_header.context + Proxy_environment.initial_context ctx resulting_context_hash in let mapped_directory = Tezos_rpc.Directory.map diff --git a/src/lib_shell/block_directory.ml b/src/lib_shell/block_directory.ml index deb14000afe1..3a733510788e 100644 --- a/src/lib_shell/block_directory.ml +++ b/src/lib_shell/block_directory.ml @@ -102,6 +102,10 @@ let build_raw_header_rpc_directory (module Proto : Block_services.PROTO) = header.protocol_data)) ; register0 S.Header.raw_protocol_data (fun (_, _, header) () () -> return header.protocol_data) ; + register0 S.resulting_context_hash (fun (chain_store, hash, _) () () -> + let*! block_opt = Store.Block.read_block_opt chain_store hash in + let block = WithExceptions.Option.to_exn ~none:Not_found block_opt in + Store.Block.resulting_context_hash chain_store block) ; (* helpers *) register0 S.Helpers.Forge.block_header (fun _block () header -> return (Data_encoding.Binary.to_bytes_exn Block_header.encoding header)) ; @@ -292,11 +296,14 @@ let build_raw_rpc_directory (module Proto : Block_services.PROTO) in let predecessor_header = Store.Block.header predecessor_block in let* context = Store.Block.context chain_store predecessor_block in + let* predecessor_resulting_context = + Store.Block.resulting_context_hash chain_store predecessor_block + in let* predecessor_context = let*! ctxt = Context_ops.checkout (Context_ops.index context) - (Store.Block.context_hash predecessor_block) + predecessor_resulting_context in match ctxt with Some c -> return c | None -> fail_with_exn Not_found in diff --git a/src/lib_shell_services/block_services.ml b/src/lib_shell_services/block_services.ml index 3d3b680603e5..e781a19997bb 100644 --- a/src/lib_shell_services/block_services.ml +++ b/src/lib_shell_services/block_services.ml @@ -618,6 +618,13 @@ module Make (Proto : PROTO) (Next_proto : PROTO) = struct ~output:raw_protocol_encoding Tezos_rpc.Path.(path / "protocols") + let resulting_context_hash = + Tezos_rpc.Service.get_service + ~description:"Context hash resulting of the block application." + ~query:Tezos_rpc.Query.empty + ~output:Tezos_crypto.Context_hash.encoding + Tezos_rpc.Path.(path / "resulting_context_hash") + module Header = struct let path = Tezos_rpc.Path.(path / "header") @@ -1439,6 +1446,10 @@ module Make (Proto : PROTO) (Next_proto : PROTO) = struct let f = make_call0 S.protocols ctxt in fun ?(chain = `Main) ?(block = `Head 0) () -> f chain block () () + let resulting_context_hash ctxt = + let f = make_call0 S.resulting_context_hash ctxt in + fun ?(chain = `Main) ?(block = `Head 0) () -> f chain block () () + module Header = struct module S = S.Header diff --git a/src/lib_shell_services/block_services.mli b/src/lib_shell_services/block_services.mli index f06a70c8f13b..ef27dccd1c54 100644 --- a/src/lib_shell_services/block_services.mli +++ b/src/lib_shell_services/block_services.mli @@ -219,6 +219,13 @@ module Make (Proto : PROTO) (Next_proto : PROTO) : sig unit -> Tezos_crypto.Block_metadata_hash.t tzresult Lwt.t + val resulting_context_hash : + #simple -> + ?chain:chain -> + ?block:block -> + unit -> + Tezos_crypto.Context_hash.t tzresult Lwt.t + module Header : sig val shell_header : #simple -> @@ -514,6 +521,15 @@ module Make (Proto : PROTO) (Next_proto : PROTO) : sig val protocols : ([`GET], prefix, prefix, unit, unit, protocols) Tezos_rpc.Service.t + val resulting_context_hash : + ( [`GET], + prefix, + prefix, + unit, + unit, + Tezos_crypto.Context_hash.t ) + Tezos_rpc.Service.t + module Header : sig val shell_header : ( [`GET], -- GitLab From 0cddcfb2f313216f31ace47f77ace7208bf06e0d Mon Sep 17 00:00:00 2001 From: vbot Date: Thu, 10 Nov 2022 10:07:34 +0100 Subject: [PATCH 13/19] Baker/Alpha: make baker forge header with the pred resulting context --- .../lib_delegate/baking_simulator.ml | 9 ++++++-- src/proto_alpha/lib_delegate/baking_state.ml | 23 +++++++++++-------- src/proto_alpha/lib_delegate/baking_state.mli | 1 + src/proto_alpha/lib_delegate/block_forge.ml | 11 +++++++-- src/proto_alpha/lib_delegate/node_rpc.ml | 13 +++++++---- 5 files changed, 40 insertions(+), 17 deletions(-) diff --git a/src/proto_alpha/lib_delegate/baking_simulator.ml b/src/proto_alpha/lib_delegate/baking_simulator.ml index 39b641175d45..c4bcd1aa80dc 100644 --- a/src/proto_alpha/lib_delegate/baking_simulator.ml +++ b/src/proto_alpha/lib_delegate/baking_simulator.ml @@ -79,10 +79,15 @@ let check_context_consistency (abstract_index : Abstract_context_index.t) let begin_construction ~timestamp ~protocol_data (abstract_index : Abstract_context_index.t) predecessor chain_id = protect (fun () -> - let {Baking_state.shell = pred_shell; hash = pred_hash; _} = + let { + Baking_state.shell = pred_shell; + hash = pred_hash; + resulting_context_hash; + _; + } = predecessor in - abstract_index.checkout_fun pred_shell.context >>= function + abstract_index.checkout_fun resulting_context_hash >>= function | None -> fail Failed_to_checkout_context | Some context -> let header : Tezos_base.Block_header.shell_header = diff --git a/src/proto_alpha/lib_delegate/baking_state.ml b/src/proto_alpha/lib_delegate/baking_state.ml index 2902469662c7..1d934dc2d87d 100644 --- a/src/proto_alpha/lib_delegate/baking_state.ml +++ b/src/proto_alpha/lib_delegate/baking_state.ml @@ -112,6 +112,7 @@ type prequorum = { type block_info = { hash : Tezos_crypto.Block_hash.t; shell : Block_header.shell_header; + resulting_context_hash : Tezos_crypto.Context_hash.t; payload_hash : Block_payload_hash.t; payload_round : Round.t; round : Round.t; @@ -175,6 +176,7 @@ let block_info_encoding = (fun { hash; shell; + resulting_context_hash; payload_hash; payload_round; round; @@ -187,29 +189,30 @@ let block_info_encoding = } -> ( ( hash, shell, + resulting_context_hash, payload_hash, payload_round, round, protocol, next_protocol, prequorum, - List.map Operation.pack quorum, - payload ), - live_blocks )) + List.map Operation.pack quorum ), + (payload, live_blocks) )) (fun ( ( hash, shell, + resulting_context_hash, payload_hash, payload_round, round, protocol, next_protocol, prequorum, - quorum, - payload ), - live_blocks ) -> + quorum ), + (payload, live_blocks) ) -> { hash; shell; + resulting_context_hash; payload_hash; payload_round; round; @@ -224,15 +227,17 @@ let block_info_encoding = (obj10 (req "hash" Tezos_crypto.Block_hash.encoding) (req "shell" Block_header.shell_header_encoding) + (req "resulting_context_hash" Tezos_crypto.Context_hash.encoding) (req "payload_hash" Block_payload_hash.encoding) (req "payload_round" Round.encoding) (req "round" Round.encoding) (req "protocol" Tezos_crypto.Protocol_hash.encoding) (req "next_protocol" Tezos_crypto.Protocol_hash.encoding) (req "prequorum" (option prequorum_encoding)) - (req "quorum" (list (dynamic_size Operation.encoding))) - (req "payload" Operation_pool.payload_encoding)) - (obj1 (req "live_blocks" Tezos_crypto.Block_hash.Set.encoding))) + (req "quorum" (list (dynamic_size Operation.encoding)))) + (obj2 + (req "payload" Operation_pool.payload_encoding) + (req "live_blocks" Tezos_crypto.Block_hash.Set.encoding))) let round_of_shell_header shell_header = Environment.wrap_tzresult diff --git a/src/proto_alpha/lib_delegate/baking_state.mli b/src/proto_alpha/lib_delegate/baking_state.mli index 3936193bb346..bf7caa0ff186 100644 --- a/src/proto_alpha/lib_delegate/baking_state.mli +++ b/src/proto_alpha/lib_delegate/baking_state.mli @@ -58,6 +58,7 @@ type prequorum = { type block_info = { hash : Tezos_crypto.Block_hash.t; shell : Block_header.shell_header; + resulting_context_hash : Tezos_crypto.Context_hash.t; payload_hash : Block_payload_hash.t; payload_round : Round.t; round : Round.t; diff --git a/src/proto_alpha/lib_delegate/block_forge.ml b/src/proto_alpha/lib_delegate/block_forge.ml index 73fc1bcfa3e3..0df28bfea484 100644 --- a/src/proto_alpha/lib_delegate/block_forge.ml +++ b/src/proto_alpha/lib_delegate/block_forge.ml @@ -67,7 +67,7 @@ let convert_operation (op : packed_operation) : Tezos_base.Operation.t = (* Build the block header : mimics node prevalidation *) let finalize_block_header shell_header timestamp validation_result operations_hash predecessor_block_metadata_hash - predecessor_ops_metadata_hash = + predecessor_ops_metadata_hash predecessor_resulting_context = let {Tezos_protocol_environment.context; fitness; message; _} = validation_result in @@ -96,6 +96,11 @@ let finalize_block_header shell_header timestamp validation_result | None -> Lwt.return context) >>= fun context -> let context = Context_ops.hash ~time:timestamp ?message context in + (* For the time being, we still fully build the block while we build + confidence to fully unplug the baker validation. The resulting + context hash is ignored as it is not necessary to craft a block. + See: https://gitlab.com/tezos/tezos/-/issues/4285 *) + ignore context ; let header = Tezos_base.Block_header. { @@ -104,7 +109,7 @@ let finalize_block_header shell_header timestamp validation_result validation_passes; operations_hash; fitness; - context; + context = predecessor_resulting_context; } in return header @@ -235,6 +240,7 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info operations_hash pred_block_metadata_hash pred_op_metadata_hash + predecessor_block.resulting_context_hash >>=? fun shell_header -> let operations = List.map (List.map convert_operation) operations in let payload_hash = @@ -342,6 +348,7 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info operations_hash pred_block_metadata_hash pred_op_metadata_hash + predecessor_block.resulting_context_hash >>=? fun shell_header -> let operations = List.map (List.map convert_operation) operations in return (shell_header, operations, payload_hash) diff --git a/src/proto_alpha/lib_delegate/node_rpc.ml b/src/proto_alpha/lib_delegate/node_rpc.ml index 96749ae7c4de..94be468c039b 100644 --- a/src/proto_alpha/lib_delegate/node_rpc.ml +++ b/src/proto_alpha/lib_delegate/node_rpc.ml @@ -64,8 +64,8 @@ let extract_prequorum preendorsements = } | _ -> None -let raw_info cctxt ~chain ~block_hash shell payload_hash payload_round - current_protocol next_protocol live_blocks = +let raw_info cctxt ~chain ~block_hash shell resulting_context_hash payload_hash + payload_round current_protocol next_protocol live_blocks = Events.(emit raw_info (block_hash, shell.Tezos_base.Block_header.level)) >>= fun () -> let open Protocol_client_context in @@ -105,6 +105,7 @@ let raw_info cctxt ~chain ~block_hash shell payload_hash payload_round { Baking_state.hash = block_hash; shell; + resulting_context_hash; payload_hash; payload_round; round; @@ -123,6 +124,8 @@ let info cctxt ~chain ~block () = (* Fails if the block's protocol is not the current one *) Shell_services.Blocks.protocols cctxt ~chain ~block () >>=? fun {current_protocol; next_protocol} -> + Shell_services.Blocks.resulting_context_hash cctxt ~chain ~block () + >>=? fun resulting_context_hash -> (if Tezos_crypto.Protocol_hash.(current_protocol <> Protocol.hash) then Block_services.Header.shell_header cctxt ~chain ~block () >>=? fun shell -> Chain_services.Blocks.Header.raw_protocol_data cctxt ~chain ~block () @@ -144,16 +147,17 @@ let info cctxt ~chain ~block () = (payload_hash, payload_round) | None -> (dummy_payload_hash, Round.zero) in - return (hash, shell, payload_hash, payload_round) + return (hash, shell, resulting_context_hash, payload_hash, payload_round) else Alpha_block_services.header cctxt ~chain ~block () >>=? fun {hash; shell; protocol_data; _} -> return ( hash, shell, + resulting_context_hash, protocol_data.contents.payload_hash, protocol_data.contents.payload_round )) - >>=? fun (hash, shell, payload_hash, payload_round) -> + >>=? fun (hash, shell, resulting_context_hash, payload_hash, payload_round) -> (Chain_services.Blocks.live_blocks cctxt ~chain ~block () >>= function | Error _ -> (* The RPC might fail when a block's metadata is not available *) @@ -165,6 +169,7 @@ let info cctxt ~chain ~block () = ~chain ~block_hash:hash shell + resulting_context_hash payload_hash payload_round current_protocol -- GitLab From 94c1652e989980d9c71602d349ad5f0968588e51 Mon Sep 17 00:00:00 2001 From: vbot Date: Mon, 14 Nov 2022 14:59:26 +0100 Subject: [PATCH 14/19] Baker: handle legacy disk state format --- src/proto_alpha/lib_delegate/baking_state.ml | 163 ++++++++++++++++++- 1 file changed, 157 insertions(+), 6 deletions(-) diff --git a/src/proto_alpha/lib_delegate/baking_state.ml b/src/proto_alpha/lib_delegate/baking_state.ml index 1d934dc2d87d..a5cb4ae67159 100644 --- a/src/proto_alpha/lib_delegate/baking_state.ml +++ b/src/proto_alpha/lib_delegate/baking_state.ml @@ -442,7 +442,138 @@ let event_encoding = (fun tk -> Timeout tk); ] -(* Disk state *) +(* Legacy disk state support: this will only be used when migrating + from a baker using the previous format to the new one and will + happen once. *) + +type legacy_block_info = { + legacy_hash : Tezos_crypto.Block_hash.t; + legacy_shell : Block_header.shell_header; + legacy_payload_hash : Block_payload_hash.t; + legacy_payload_round : Round.t; + legacy_round : Round.t; + legacy_protocol : Tezos_crypto.Protocol_hash.t; + legacy_next_protocol : Tezos_crypto.Protocol_hash.t; + legacy_prequorum : prequorum option; + legacy_quorum : Kind.endorsement operation list; + legacy_payload : Operation_pool.payload; + legacy_live_blocks : Tezos_crypto.Block_hash.Set.t; +} + +let legacy_block_info_encoding : legacy_block_info Data_encoding.t = + let open Data_encoding in + conv + (fun ({ + legacy_hash; + legacy_shell; + legacy_payload_hash; + legacy_payload_round; + legacy_round; + legacy_protocol; + legacy_next_protocol; + legacy_prequorum; + legacy_quorum; + legacy_payload; + legacy_live_blocks; + } : + legacy_block_info) -> + ( ( legacy_hash, + legacy_shell, + legacy_payload_hash, + legacy_payload_round, + legacy_round, + legacy_protocol, + legacy_next_protocol, + legacy_prequorum, + List.map Operation.pack legacy_quorum, + legacy_payload ), + legacy_live_blocks )) + (fun ( ( legacy_hash, + legacy_shell, + legacy_payload_hash, + legacy_payload_round, + legacy_round, + legacy_protocol, + legacy_next_protocol, + legacy_prequorum, + legacy_quorum, + legacy_payload ), + legacy_live_blocks ) -> + { + legacy_hash; + legacy_shell; + legacy_payload_hash; + legacy_payload_round; + legacy_round; + legacy_protocol; + legacy_next_protocol; + legacy_prequorum; + legacy_quorum = + List.filter_map Operation_pool.unpack_endorsement legacy_quorum; + legacy_payload; + legacy_live_blocks; + }) + (merge_objs + (obj10 + (req "hash" Tezos_crypto.Block_hash.encoding) + (req "shell" Block_header.shell_header_encoding) + (req "payload_hash" Block_payload_hash.encoding) + (req "payload_round" Round.encoding) + (req "round" Round.encoding) + (req "protocol" Tezos_crypto.Protocol_hash.encoding) + (req "next_protocol" Tezos_crypto.Protocol_hash.encoding) + (req "prequorum" (option prequorum_encoding)) + (req "quorum" (list (dynamic_size Operation.encoding))) + (req "payload" Operation_pool.payload_encoding)) + (obj1 (req "live_blocks" Tezos_crypto.Block_hash.Set.encoding))) + +type legacy_proposal = { + legacy_block : legacy_block_info; + legacy_predecessor : legacy_block_info; +} + +let legacy_proposal_encoding = + let open Data_encoding in + conv + (fun {legacy_block; legacy_predecessor} -> + (legacy_block, legacy_predecessor)) + (fun (legacy_block, legacy_predecessor) -> + {legacy_block; legacy_predecessor}) + (obj2 + (req "block" legacy_block_info_encoding) + (req "predecessor" legacy_block_info_encoding)) + +type legacy_endorsable_payload = { + legacy_proposal : legacy_proposal; + prequorum : prequorum; +} + +let legacy_endorsable_payload_encoding = + let open Data_encoding in + conv + (fun {legacy_proposal; prequorum} -> (legacy_proposal, prequorum)) + (fun (legacy_proposal, prequorum) -> {legacy_proposal; prequorum}) + (obj2 + (req "proposal" legacy_proposal_encoding) + (req "prequorum" prequorum_encoding)) + +type legacy_state_data = { + level_data : int32; + locked_round_data : locked_round option; + legacy_endorsable_payload_data : legacy_endorsable_payload option; +} + +let legacy_state_data_encoding = + let open Data_encoding in + conv + (fun {level_data; locked_round_data; legacy_endorsable_payload_data} -> + (level_data, locked_round_data, legacy_endorsable_payload_data)) + (fun (level_data, locked_round_data, legacy_endorsable_payload_data) -> + {level_data; locked_round_data; legacy_endorsable_payload_data}) + (obj3 + (req "level" int32) + (req "locked_round" (option locked_round_encoding)) + (req "endorsable_payload" (option legacy_endorsable_payload_encoding))) type state_data = { level_data : int32; @@ -450,6 +581,25 @@ type state_data = { endorsable_payload_data : endorsable_payload option; } +let update_legacy_state_data raw_data = + (* "Upgrade" the file format by dumping the legacy + endorsable data. It is sound as the new baker + cannot start (i.e. load the preexisting state) + until the new protocol activates. + + Note: the new encoding is not backward compatible + (unless the endorsable_payload_data is [None]). *) + let legacy_state_data = + Data_encoding.Binary.of_string_exn legacy_state_data_encoding raw_data + in + { + level_data = legacy_state_data.level_data; + locked_round_data = None; + endorsable_payload_data = None; + } + +(* Disk state *) + let state_data_encoding = let open Data_encoding in conv @@ -575,11 +725,12 @@ let load_endorsable_data cctxt location = ~mode:Input filename (fun channel -> - Lwt_io.read channel >>= fun bytes -> - Lwt.return - (Data_encoding.Binary.of_bytes_exn - state_data_encoding - (Bytes.unsafe_of_string bytes))) + Lwt_io.read channel >>= fun str -> + match + Data_encoding.Binary.of_string_opt state_data_encoding str + with + | Some state_data -> Lwt.return state_data + | None -> Lwt.return (update_legacy_state_data str)) >>= return_some) let may_load_endorsable_data state = -- GitLab From 53c3dd9df2a01eda2ef2aad869bc24a6be8e03df Mon Sep 17 00:00:00 2001 From: vbot Date: Tue, 15 Nov 2022 10:53:34 +0100 Subject: [PATCH 15/19] Doc/RPC: link genesis protocol --- docs/doc_gen/dune | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/doc_gen/dune b/docs/doc_gen/dune index 1baefad6a1a9..04ed2cf21836 100644 --- a/docs/doc_gen/dune +++ b/docs/doc_gen/dune @@ -8,6 +8,7 @@ tezos-rpc-http-server tezos-protocol-updater ; TODO tezos/tezos#2170: adapt next line(s) + tezos-embedded-protocol-genesis tezos-embedded-protocol-014-PtKathma tezos-embedded-protocol-015-PtLimaPt tezos-embedded-protocol-alpha -- GitLab From 418de845464137139d95964a78a0e03720ffe9d2 Mon Sep 17 00:00:00 2001 From: vbot Date: Tue, 15 Nov 2022 11:43:42 +0100 Subject: [PATCH 16/19] Mockup: add resulting_context_hash RPC and adapt baking --- src/lib_mockup/local_services.ml | 13 +++++++++++++ .../test/mockup_simulator/faked_services.ml | 13 +++++++++++++ .../test/mockup_simulator/mockup_simulator.ml | 18 ++++++++++++++++-- .../test/mockup_simulator/mockup_simulator.mli | 1 + 4 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/lib_mockup/local_services.ml b/src/lib_mockup/local_services.ml index bccc41c9b5d1..bdd3ac4ad092 100644 --- a/src/lib_mockup/local_services.ml +++ b/src/lib_mockup/local_services.ml @@ -905,6 +905,18 @@ module Make (E : MENV) = struct in Tezos_rpc.Answer.return block_header))) + let resulting_context_hash () = + Directory.prefix + (Tezos_rpc.Path.prefix Chain_services.path Block_services.path) + (Directory.register + Directory.empty + E.Block_services.S.resulting_context_hash + (fun (((), chain), _block) () () -> + with_chain ~caller_name:"resulting_context_hash" chain (fun () -> + (* This is not sufficient but this library doesn't have + what's necessary to determine the correct value. *) + Tezos_rpc.Answer.return E.rpc_context.block_header.context))) + let protocol_data_raw () = Directory.prefix (Tezos_rpc.Path.prefix Chain_services.path Block_services.path) @@ -965,6 +977,7 @@ module Make (E : MENV) = struct |> merge (monitor_heads ()) |> merge (header ()) |> merge (operations ()) + |> merge (resulting_context_hash ()) |> merge (protocol_data_raw ()) |> merge (monitor_operations ()) end diff --git a/src/proto_alpha/lib_delegate/test/mockup_simulator/faked_services.ml b/src/proto_alpha/lib_delegate/test/mockup_simulator/faked_services.ml index ee21b96b07c2..ff2b7871d752 100644 --- a/src/proto_alpha/lib_delegate/test/mockup_simulator/faked_services.ml +++ b/src/proto_alpha/lib_delegate/test/mockup_simulator/faked_services.ml @@ -21,6 +21,10 @@ module type Mocked_services_hooks = sig val header : Block_services.block -> Mockup.M.Block_services.block_header tzresult Lwt.t + (** [resulting_context_hash] returns the context resulting hash of the given block. *) + val resulting_context_hash : + Block_services.block -> Tezos_crypto.Context_hash.t tzresult Lwt.t + (** [operations] returns all operations included in the block. *) val operations : Block_services.block -> @@ -146,6 +150,14 @@ module Make (Hooks : Mocked_services_hooks) = struct Mockup.M.Block_services.S.header (fun (((), _chain), block) _ _ -> Hooks.header block) + let resulting_context_hash = + Directory.prefix + (Tezos_rpc.Path.prefix Chain_services.path Block_services.path) + @@ Directory.register + Directory.empty + Mockup.M.Block_services.S.resulting_context_hash + (fun (((), _chain), block) _ _ -> Hooks.resulting_context_hash block) + let operations = Directory.prefix (Tezos_rpc.Path.prefix Chain_services.path Block_services.path) @@ -277,6 +289,7 @@ module Make (Hooks : Mocked_services_hooks) = struct let merge = Directory.merge in Directory.empty |> merge monitor_heads |> merge protocols |> merge header |> merge operations |> merge hash |> merge shell_header + |> merge resulting_context_hash |> merge (chain chain_id) |> merge inject_block |> merge inject_operation |> merge monitor_operations |> merge list_blocks |> merge live_blocks |> merge raw_protocol_data diff --git a/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.ml b/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.ml index 65c8ad1d7c6c..671df850a7df 100644 --- a/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.ml +++ b/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.ml @@ -28,6 +28,7 @@ type block = { protocol_data : Protocol.Alpha_context.Block_header.protocol_data; raw_protocol_data : Bytes.t; operations : Mockup.M.Block_services.operation list list; + resulting_context_hash : Tezos_crypto.Context_hash.t; } type chain = block list @@ -322,6 +323,11 @@ let make_mocked_services_hooks (state : state) (user_hooks : (module Hooks)) : protocol_data = x.protocol_data; } + let resulting_context_hash + (block : Tezos_shell_services.Block_services.block) : + Tezos_crypto.Context_hash.t tzresult Lwt.t = + locate_block state block >>=? fun x -> return x.resulting_context_hash + let operations block = locate_block state block >>=? fun x -> return x.operations @@ -628,7 +634,13 @@ let rec process_block state block_hash (block_header : Block_header.t) get_predecessor () >>=? fun predecessor -> head state >>=? fun head -> reconstruct_context predecessor.rpc_context operations block_header - >>=? fun ({context; _}, _) -> + >>=? fun ({context; message; _}, _) -> + let resulting_context_hash = + Tezos_context_ops.Context_ops.hash + ~time:block_header.shell.timestamp + ?message + context + in let rpc_context = Tezos_protocol_environment. {context; block_hash; block_header = block_header.shell} @@ -661,6 +673,7 @@ let rec process_block state block_hash (block_header : Block_header.t) protocol_data; raw_protocol_data = block_header.protocol_data; operations; + resulting_context_hash; } in let predecessor_hash = block_header.Block_header.shell.predecessor in @@ -679,7 +692,7 @@ let rec process_block state block_hash (block_header : Block_header.t) new_block ; Tezos_crypto.Context_hash.Table.replace state.ctxt_table - rpc_context.Tezos_protocol_environment.block_header.context + resulting_context_hash rpc_context ; if Fitness.( @@ -746,6 +759,7 @@ let create_fake_node_state ~i ~live_depth protocol_data; raw_protocol_data = block_header0.protocol_data; operations = [[]; []; []; []]; + resulting_context_hash = block_header0.shell.context; } in let chain0 = [genesis0] in diff --git a/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.mli b/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.mli index 42a13cae3af4..ef4db4d8b7cb 100644 --- a/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.mli +++ b/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.mli @@ -29,6 +29,7 @@ type block = { protocol_data : Protocol.Alpha_context.Block_header.protocol_data; raw_protocol_data : Bytes.t; operations : Mockup.M.Block_services.operation list list; + resulting_context_hash : Tezos_crypto.Context_hash.t; } (** Chain is a list of blocks. *) -- GitLab From 5a7922a2a79da643f7e6d465171a5fb0d1472fd5 Mon Sep 17 00:00:00 2001 From: vbot Date: Wed, 23 Nov 2022 11:02:37 +0100 Subject: [PATCH 17/19] Tezt: adapt bad block propagation test --- tezt/tests/precheck.ml | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tezt/tests/precheck.ml b/tezt/tests/precheck.ml index 15574700297c..de072ac06ab6 100644 --- a/tezt/tests/precheck.ml +++ b/tezt/tests/precheck.ml @@ -171,14 +171,14 @@ let propagate_precheckable_bad_block = N4 *) Log.info "Setting up the node topology" ; - let n1 = Node.create [] in + let node_client = Node.create [] in let ring = Cluster.create ~name:"ring" 4 [Private_mode; Synchronisation_threshold 0] in let n2 = List.hd ring in Cluster.ring ring ; - Cluster.connect [n1] [n2] ; - let cluster = n1 :: ring in + Cluster.connect [node_client] [n2] ; + let cluster = node_client :: ring in Log.info "Starting up cluster" ; let* () = Cluster.start @@ -187,7 +187,7 @@ let propagate_precheckable_bad_block = cluster in Log.info "Cluster initialized" ; - let* client = Client.(init ~endpoint:(Node n1) ()) in + let* client = Client.(init ~endpoint:(Node node_client) ()) in let* () = Client.activate_protocol ~protocol client in let bootstrap1 = Constant.bootstrap1.alias in let* () = @@ -197,7 +197,9 @@ let propagate_precheckable_bad_block = let* () = Client.bake_for_and_wait ~keys:[bootstrap1] client in wait_for_cluster_at_level cluster i) in - let* block_header = forge_block ~client n1 ~key:bootstrap1 ~with_op:false in + let* block_header = + forge_block ~client node_client ~key:bootstrap1 ~with_op:false + in (* Put a bad context *) Log.info "Crafting a block header with a bad context hash" ; let dummy_context_hash = @@ -248,8 +250,18 @@ let propagate_precheckable_bad_block = ] in (* Wait all nodes to precheck the block but fail on validation *) + let expect_precheck_failure node = + Node.wait_for node "precheck_failure.v0" (fun _ -> Some ()) + in let precheck_waiter = - Lwt_list.iter_p wait_precheck_but_validation_fail cluster + if Protocol.(protocol <= Lima) then + (* On Lima and below: wait all nodes to precheck the block + but fail on validation *) + Lwt_list.iter_p wait_precheck_but_validation_fail cluster + else + (* Post Lima: the precheck is not an over-approximation + anymore and cannot even be considered precheckable. *) + expect_precheck_failure node_client in let p = Client.spawn_rpc ~data:injection_json POST ["injection"; "block"] client -- GitLab From 19f3b38eeeea142a8b1bb303d3db36d588fe4c1a Mon Sep 17 00:00:00 2001 From: vbot Date: Thu, 1 Dec 2022 17:33:26 +0100 Subject: [PATCH 18/19] Everywhere: validation_store.context_hash to resulting_context_hash --- src/bin_node/node_replay_command.ml | 5 +++-- src/lib_shell/block_validator.ml | 2 +- src/lib_store/mocked/store.ml | 7 +++++-- src/lib_store/unix/snapshots.ml | 10 +++++++--- src/lib_store/unix/store.ml | 2 +- src/lib_store/unix/test/alpha_utils.ml | 2 +- src/lib_store/unix/test/test_locator.ml | 6 +++--- src/lib_store/unix/test/test_utils.ml | 2 +- 8 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/bin_node/node_replay_command.ml b/src/bin_node/node_replay_command.ml index e3ee947466cc..2fe4c2c1657c 100644 --- a/src/bin_node/node_replay_command.ml +++ b/src/bin_node/node_replay_command.ml @@ -262,11 +262,12 @@ let replay_one_block strict main_chain_store validator_process block = (not (Tezos_crypto.Context_hash.equal expected_context_hash - result.validation_store.context_hash)) + result.validation_store.resulting_context_hash)) (fun () -> let*! () = Event.(strict_emit ~strict inconsistent_context_hash) - (expected_context_hash, result.validation_store.context_hash) + ( expected_context_hash, + result.validation_store.resulting_context_hash ) in return_unit) in diff --git a/src/lib_shell/block_validator.ml b/src/lib_shell/block_validator.ml index 3386dbbf1c2f..b0aa21957b63 100644 --- a/src/lib_shell/block_validator.ml +++ b/src/lib_shell/block_validator.ml @@ -275,7 +275,7 @@ let on_validation_request w { block; resulting_context_hash = - result.validation_store.context_hash; + result.validation_store.resulting_context_hash; } ; return Validated | None -> return Already_committed))) diff --git a/src/lib_store/mocked/store.ml b/src/lib_store/mocked/store.ml index a5e07f9c3daf..943766c361a1 100644 --- a/src/lib_store/mocked/store.ml +++ b/src/lib_store/mocked/store.ml @@ -394,7 +394,7 @@ module Block = struct let { Block_validation.validation_store = { - context_hash; + resulting_context_hash; timestamp = _; message; max_operations_ttl; @@ -519,7 +519,10 @@ module Block = struct in let block = {Block_repr.hash; contents; metadata} in let* () = - Block_store.store_block chain_store.block_store block context_hash + Block_store.store_block + chain_store.block_store + block + resulting_context_hash in let*! () = Store_events.(emit store_block) (hash, block_header.shell.level) diff --git a/src/lib_store/unix/snapshots.ml b/src/lib_store/unix/snapshots.ml index 4ed4c12c3603..fc324c4e693b 100644 --- a/src/lib_store/unix/snapshots.ml +++ b/src/lib_store/unix/snapshots.ml @@ -3555,12 +3555,15 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct fail_unless (expect_predecessor_context || Context_hash.equal - validation_store.Tezos_validation.Block_validation.context_hash + validation_store + .Tezos_validation.Block_validation.resulting_context_hash block_header.Block_header.shell.context) (Inconsistent_context_hash { expected = block_header.Block_header.shell.context; - got = validation_store.Tezos_validation.Block_validation.context_hash; + got = + validation_store + .Tezos_validation.Block_validation.resulting_context_hash; }) let restore_and_apply_context snapshot_importer protocol_levels @@ -3874,7 +3877,8 @@ module Make_snapshot_importer (Importer : IMPORTER) : Snapshot_importer = struct ~genesis_context_hash ~floating_blocks_stream ~new_head_with_metadata - ~new_head_resulting_context_hash:validation_store.context_hash + ~new_head_resulting_context_hash: + validation_store.resulting_context_hash ~protocol_levels ~history_mode) in diff --git a/src/lib_store/unix/store.ml b/src/lib_store/unix/store.ml index 62c47ae4e11f..b8577713600a 100644 --- a/src/lib_store/unix/store.ml +++ b/src/lib_store/unix/store.ml @@ -447,7 +447,7 @@ module Block = struct let { Block_validation.validation_store = { - context_hash = resulting_context_hash; + resulting_context_hash; timestamp = _; message; max_operations_ttl; diff --git a/src/lib_store/unix/test/alpha_utils.ml b/src/lib_store/unix/test/alpha_utils.ml index a179eeb96281..ba239842d54e 100644 --- a/src/lib_store/unix/test/alpha_utils.ml +++ b/src/lib_store/unix/test/alpha_utils.ml @@ -675,7 +675,7 @@ let apply_and_store chain_store ?(synchronous_merge = true) ?policy { Tezos_validation.Block_validation.validation_store = { - context_hash = resulting_context_hash; + resulting_context_hash; timestamp = block_header.shell.timestamp; message = validation.Tezos_protocol_environment.message; max_operations_ttl = validation.max_operations_ttl; diff --git a/src/lib_store/unix/test/test_locator.ml b/src/lib_store/unix/test/test_locator.ml index 765ece5f7e67..a0dc885f93ce 100644 --- a/src/lib_store/unix/test/test_locator.ml +++ b/src/lib_store/unix/test/test_locator.ml @@ -101,7 +101,7 @@ let make_empty_chain chain_store n : Block_hash.t Lwt.t = Context_ops.commit ~time:header.shell.timestamp empty_context in let header = {header with shell = {header.shell with context}} in - let context_hash = empty_context_hash in + let resulting_context_hash = empty_context_hash in let message = None in let max_operations_ttl = 0 in let rec loop lvl pred = @@ -118,7 +118,7 @@ let make_empty_chain chain_store n : Block_hash.t Lwt.t = { Block_validation.validation_store = { - context_hash; + resulting_context_hash; timestamp = header.shell.timestamp; message; max_operations_ttl; @@ -160,7 +160,7 @@ let make_multiple_protocol_chain (chain_store : Store.Chain.t) let genesis_header = {header with shell = {header.shell with context}} in let empty_result = { - Block_validation.context_hash = context; + Block_validation.resulting_context_hash = context; timestamp = Time.Protocol.epoch; message = None; max_operations_ttl = 0; diff --git a/src/lib_store/unix/test/test_utils.ml b/src/lib_store/unix/test/test_utils.ml index 0dc264e95482..336ea3e3beeb 100644 --- a/src/lib_store/unix/test/test_utils.ml +++ b/src/lib_store/unix/test/test_utils.ml @@ -463,7 +463,7 @@ let store_raw_block chain_store ?resulting_context (raw_block : Block_repr.t) = { Tezos_validation.Block_validation.validation_store = { - context_hash = + resulting_context_hash = Option.value ~default:(Block_repr.context raw_block) resulting_context; -- GitLab From 3a9fd0eb04d4d665796cc14e8e8d15ef20d4127b Mon Sep 17 00:00:00 2001 From: vbot Date: Wed, 23 Nov 2022 12:02:13 +0100 Subject: [PATCH 19/19] Changelog: add entries --- CHANGES.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index f508e723b241..268650df159a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -73,6 +73,17 @@ Node and later) where the validation of a block is dissociated from its application. (MR :gl:`!7014`) +- **Breaking change**: bumped the node's storage version to + ``3.0``. This new version changes the store's representation + required by the new protocol's semantics. Upgrading to this new + version is automatic and irreversible. + +- **Breaking change**: bumped the snapshot version to ``5``. This + version changes internal snapshot file representation to include + more information required by the new protocol's semantics. Snapshots + of version ``4`` exported with previous versions of Octez can still + be imported. Snapshots of version ``5`` are not backward compatible. + Client ------ @@ -85,6 +96,15 @@ Client Baker ----- +- **Breaking change**: modified the baker's persistent state. Once the + protocol "M" activates, the new baker will automatically overwrite + the existing persistent state to the new format. This implies that + previous bakers will fail to load this new state from disk unless + the user directly removes the file + ``/_baker_state``. On mainnet, this will have + no effect as when the new protocol activates, previous bakers will + be permanently idle. + Accuser ------- -- GitLab