From 25aab37ec397ae6313dac9426f76b8b87fa982c1 Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Tue, 24 Jan 2023 10:55:56 +0100 Subject: [PATCH 01/46] Proto/Baker: use the defined wrapper create_round_durations --- src/proto_alpha/lib_delegate/baking_scheduling.ml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/proto_alpha/lib_delegate/baking_scheduling.ml b/src/proto_alpha/lib_delegate/baking_scheduling.ml index 38d42a06cc3d..04a717b6a99a 100644 --- a/src/proto_alpha/lib_delegate/baking_scheduling.ml +++ b/src/proto_alpha/lib_delegate/baking_scheduling.ml @@ -652,13 +652,7 @@ let create_initial_state cctxt ?(synchronize = true) ~chain config } in (if synchronize then - let round_durations = - Stdlib.Option.get - @@ Round.Durations.create_opt - ~first_round_duration:constants.parametric.minimal_block_delay - ~delay_increment_per_round: - constants.parametric.delay_increment_per_round - in + create_round_durations constants >>? fun round_durations -> Baking_actions.compute_round current_proposal round_durations >>? fun current_round -> ok {current_round; current_phase = Idle} else ok {Baking_state.current_round = Round.zero; current_phase = Idle}) -- GitLab From d65f140019aa1dd6eae780c3de8a839b8fff2761 Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Tue, 24 Jan 2023 10:57:40 +0100 Subject: [PATCH 02/46] Proto/Baker: use the round_durations field cached in state --- .../lib_delegate/baking_scheduling.ml | 36 +++++++------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/src/proto_alpha/lib_delegate/baking_scheduling.ml b/src/proto_alpha/lib_delegate/baking_scheduling.ml index 04a717b6a99a..16e79ccce712 100644 --- a/src/proto_alpha/lib_delegate/baking_scheduling.ml +++ b/src/proto_alpha/lib_delegate/baking_scheduling.ml @@ -220,32 +220,20 @@ let compute_next_round_time state = repropose a block at next level. *) None | None -> ( - let first_round_duration = - state.global_state.constants.parametric.minimal_block_delay - in - let delay_increment_per_round = - state.global_state.constants.parametric.delay_increment_per_round - in + let round_durations = state.global_state.round_durations in + let predecessor_timestamp = proposal.predecessor.shell.timestamp in + let predecessor_round = proposal.predecessor.round in + let next_round = Round.succ state.round_state.current_round in match - Round.Durations.create_opt - ~first_round_duration - ~delay_increment_per_round + timestamp_of_round + state.global_state.cache.known_timestamps + round_durations + ~predecessor_timestamp + ~predecessor_round + ~round:next_round with - | Some round_durations -> ( - let predecessor_timestamp = proposal.predecessor.shell.timestamp in - let predecessor_round = proposal.predecessor.round in - let next_round = Round.succ state.round_state.current_round in - match - timestamp_of_round - state.global_state.cache.known_timestamps - round_durations - ~predecessor_timestamp - ~predecessor_round - ~round:next_round - with - | Ok timestamp -> Some (timestamp, next_round) - | _ -> assert false) - | None -> assert false) + | Ok timestamp -> Some (timestamp, next_round) + | _ -> assert false) (** [first_potential_round_at_next_level state ~earliest_round] yields an optional pair of the earliest possible round (at or after -- GitLab From 92b1dbd74625848e1773b3f0a8e08afc24341f30 Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Tue, 24 Jan 2023 11:03:48 +0100 Subject: [PATCH 03/46] Proto/Baker: simplify implementation of timestamp_of_round --- src/proto_alpha/lib_delegate/baking_scheduling.ml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/proto_alpha/lib_delegate/baking_scheduling.ml b/src/proto_alpha/lib_delegate/baking_scheduling.ml index 16e79ccce712..fc61460acbae 100644 --- a/src/proto_alpha/lib_delegate/baking_scheduling.ml +++ b/src/proto_alpha/lib_delegate/baking_scheduling.ml @@ -68,9 +68,9 @@ let find_in_known_round_intervals known_round_intervals ~predecessor_timestamp {predecessor_timestamp; predecessor_round; time_interval = (now, now)}) (** Memoization wrapper for [Round.timestamp_of_round]. *) -let timestamp_of_round known_timestamps round_durations ~predecessor_timestamp - ~predecessor_round ~round = +let timestamp_of_round state ~predecessor_timestamp ~predecessor_round ~round = let open Baking_cache in + let known_timestamps = state.global_state.cache.known_timestamps in match Timestamp_of_round_cache.find_opt known_timestamps @@ -79,7 +79,7 @@ let timestamp_of_round known_timestamps round_durations ~predecessor_timestamp (* Compute and register the timestamp if not already existing. *) | None -> Protocol.Alpha_context.Round.timestamp_of_round - round_durations + state.global_state.round_durations ~predecessor_timestamp ~predecessor_round ~round @@ -220,14 +220,12 @@ let compute_next_round_time state = repropose a block at next level. *) None | None -> ( - let round_durations = state.global_state.round_durations in let predecessor_timestamp = proposal.predecessor.shell.timestamp in let predecessor_round = proposal.predecessor.round in let next_round = Round.succ state.round_state.current_round in match timestamp_of_round - state.global_state.cache.known_timestamps - round_durations + state ~predecessor_timestamp ~predecessor_round ~round:next_round @@ -375,8 +373,7 @@ let compute_next_potential_baking_time_at_next_level state = >>= fun () -> Lwt.return_none | None | Some _ -> ( timestamp_of_round - state.global_state.cache.known_timestamps - round_durations + state ~predecessor_timestamp ~predecessor_round ~round:first_potential_round -- GitLab From a4a56298622da22b08571e611db3a6f809bb873d Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Mon, 23 Jan 2023 15:29:17 +0100 Subject: [PATCH 04/46] Tezt/Baking: add two tests to play with minimal_timestamp flag --- tezt/tests/baking.ml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tezt/tests/baking.ml b/tezt/tests/baking.ml index 7d099b7af055..d37c44cead41 100644 --- a/tezt/tests/baking.ml +++ b/tezt/tests/baking.ml @@ -712,10 +712,31 @@ let test_operation_pool_ordering check_hashes op_hashes hashes ; Lwt.return_unit +(** This test activates a protocol with a timestamps [Now]. It then bakes a + block with the given [minimal_timestamp] flag. *) +let baking_with_given_minimal_timestamp ~minimal_timestamp = + Protocol.register_test + ~supports:Protocol.(From_protocol (number Mumbai + 1)) + ~__FILE__ + ~title:(sf "Baking minimal timestamp (%b)" minimal_timestamp) + ~tags:["baking"; "timestamp"] + @@ fun protocol -> + let* _node, client = + Client.init_with_protocol + ~timestamp:(Ago (Tezos_base.Time.System.Span.of_seconds_exn 60.)) + ~nodes_args:[Synchronisation_threshold 0] + ~protocol + `Client + () + in + Client.bake_for_and_wait ~minimal_timestamp client + let register ~protocols = test_ordering protocols ; wrong_branch_operation_dismissal protocols ; - baking_operation_exception protocols + baking_operation_exception protocols ; + baking_with_given_minimal_timestamp ~minimal_timestamp:false protocols ; + baking_with_given_minimal_timestamp ~minimal_timestamp:true protocols let register_operations_pool ~protocols = List.iter -- GitLab From 55eaf3d5608eace3ec3a357b7c3b24e863268435 Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Fri, 20 Jan 2023 20:55:07 +0100 Subject: [PATCH 05/46] Baker: move confusing event placement If the round start (of the first potential round at next level) is in the past, the event will show something like ``` waiting -38min15s until it's time to bake at ... ``` which is quite confusing. Actually the baker does not wait. So the baker now emits this event only when it really waits. Fixes https://gitlab.com/tezos/tezos/-/issues/4646 --- src/proto_alpha/lib_delegate/baking_scheduling.ml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/proto_alpha/lib_delegate/baking_scheduling.ml b/src/proto_alpha/lib_delegate/baking_scheduling.ml index fc61460acbae..96350fe3ba3e 100644 --- a/src/proto_alpha/lib_delegate/baking_scheduling.ml +++ b/src/proto_alpha/lib_delegate/baking_scheduling.ml @@ -460,13 +460,14 @@ let compute_next_timeout state : Baking_state.timeout_kind Lwt.t tzresult Lwt.t let wait_baking_time_next_level (next_baking_time, next_baking_round) = let now = Time.System.now () in let delay = Ptime.diff (Time.System.of_protocol_exn next_baking_time) now in - Events.(emit waiting_time_to_bake (delay, next_baking_time)) >>= fun () -> match sleep_until next_baking_time with | None -> Events.(emit no_need_to_wait_for_proposal ()) >>= fun () -> return (Lwt.return (Time_to_bake_next_level {at_round = next_baking_round})) | Some t -> + Events.(emit waiting_time_to_bake (delay, next_baking_time)) + >>= fun () -> return ( t >>= fun () -> Lwt.return (Time_to_bake_next_level {at_round = next_baking_round}) -- GitLab From 0ca6f1e93b704fe2775d4e306dc6fb9b04cd4e19 Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Wed, 4 Jan 2023 15:50:36 +0100 Subject: [PATCH 06/46] Baker: simplify type `endorsing_slot` --- src/proto_alpha/lib_delegate/baking_state.ml | 19 ++++++++++++------- src/proto_alpha/lib_delegate/baking_state.mli | 2 +- .../lib_delegate/state_transitions.ml | 2 +- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/proto_alpha/lib_delegate/baking_state.ml b/src/proto_alpha/lib_delegate/baking_state.ml index 0130ec952817..475306cac7d8 100644 --- a/src/proto_alpha/lib_delegate/baking_state.ml +++ b/src/proto_alpha/lib_delegate/baking_state.ml @@ -241,7 +241,7 @@ module SlotMap : Map.S with type key = Slot.t = Map.Make (Slot) list of slots (i.e., a list of position indexes in the slot map, in other words the list of rounds when it will be the proposer), and its endorsing power. *) -type endorsing_slot = {slots : Slot.t list; endorsing_power : int} +type endorsing_slot = {first_slot : Slot.t; endorsing_power : int} (* FIXME: determine if the slot map should contain all slots or just the first one *) @@ -782,7 +782,12 @@ let compute_delegate_slots (cctxt : Protocol_client_context.full) List.fold_left (fun (own_map, all_map) slot -> let {Plugin.RPC.Validators.consensus_key; delegate; slots; _} = slot in - let endorsing_slot = {endorsing_power = List.length slots; slots} in + let endorsing_slot = + { + endorsing_power = List.length slots; + first_slot = Stdlib.List.hd slots; + } + in let all_map = List.fold_left (fun all_map slot -> SlotMap.add slot endorsing_slot all_map) @@ -916,13 +921,13 @@ let pp_elected_block fmt {proposal; endorsement_qc} = proposal.block (List.length endorsement_qc) -let pp_endorsing_slot fmt (consensus_key_and_delegate, {slots; endorsing_power}) - = +let pp_endorsing_slot fmt + (consensus_key_and_delegate, {first_slot; endorsing_power}) = Format.fprintf fmt - "slots: @[[%a]@],@ delegate: %a,@ endorsing_power: %d" - Format.(pp_print_list ~pp_sep:pp_print_space Slot.pp) - slots + "slots: @[first_slot: %a@],@ delegate: %a,@ endorsing_power: %d" + Slot.pp + first_slot pp_consensus_key_and_delegate consensus_key_and_delegate endorsing_power diff --git a/src/proto_alpha/lib_delegate/baking_state.mli b/src/proto_alpha/lib_delegate/baking_state.mli index 198cde1a9f96..fef6d3ccc336 100644 --- a/src/proto_alpha/lib_delegate/baking_state.mli +++ b/src/proto_alpha/lib_delegate/baking_state.mli @@ -96,7 +96,7 @@ val round_of_shell_header : Block_header.shell_header -> Round.t tzresult module SlotMap : Map.S with type key = Slot.t -type endorsing_slot = {slots : Slot.t list; endorsing_power : int} +type endorsing_slot = {first_slot : Slot.t; endorsing_power : int} type delegate_slots = { own_delegate_slots : (consensus_key_and_delegate * endorsing_slot) SlotMap.t; diff --git a/src/proto_alpha/lib_delegate/state_transitions.ml b/src/proto_alpha/lib_delegate/state_transitions.ml index 196d248f553e..d39c05dbc8cd 100644 --- a/src/proto_alpha/lib_delegate/state_transitions.ml +++ b/src/proto_alpha/lib_delegate/state_transitions.ml @@ -81,7 +81,7 @@ let make_consensus_list state proposal = SlotMap.fold (fun _slot (consensus_key_and_delegate, slots) acc -> ( consensus_key_and_delegate, - {slot = Stdlib.List.hd slots.slots; level; round; block_payload_hash} ) + {slot = slots.first_slot; level; round; block_payload_hash} ) :: acc) state.level_state.delegate_slots.own_delegate_slots [] -- GitLab From b79ba6eeba3994d07a7ac39ee0c930f74ea2799c Mon Sep 17 00:00:00 2001 From: Albin Coquereau Date: Wed, 18 Jan 2023 15:09:19 +0100 Subject: [PATCH 07/46] lib_delegate: use lwt_result_syntax in block_forge --- .../lib_delegate/block_forge.ml | 335 ++++++++++-------- src/proto_alpha/lib_delegate/block_forge.ml | 335 ++++++++++-------- 2 files changed, 360 insertions(+), 310 deletions(-) diff --git a/src/proto_016_PtMumbai/lib_delegate/block_forge.ml b/src/proto_016_PtMumbai/lib_delegate/block_forge.ml index 917133e0e553..83b0d58a4a8b 100644 --- a/src/proto_016_PtMumbai/lib_delegate/block_forge.ml +++ b/src/proto_016_PtMumbai/lib_delegate/block_forge.ml @@ -68,33 +68,38 @@ let convert_operation (op : packed_operation) : Tezos_base.Operation.t = let finalize_block_header shell_header timestamp validation_result operations_hash predecessor_block_metadata_hash predecessor_ops_metadata_hash predecessor_resulting_context = + let open Lwt_result_syntax in let {Tezos_protocol_environment.context; fitness; message; _} = validation_result in let validation_passes = List.length Main.validation_passes in - (Context_ops.get_test_chain context >>= function - | Not_running -> return context - | Running {expiration; _} -> - if Time.Protocol.(expiration <= timestamp) then - Context_ops.add_test_chain context Not_running >>= fun context -> - return context - else return context - | Forking _ -> assert false) - >>=? fun context -> - (match predecessor_block_metadata_hash with - | Some predecessor_block_metadata_hash -> - Context_ops.add_predecessor_block_metadata_hash - context - predecessor_block_metadata_hash - | None -> Lwt.return context) - >>= fun context -> - (match predecessor_ops_metadata_hash with - | Some predecessor_ops_metadata_hash -> - Context_ops.add_predecessor_ops_metadata_hash - context - predecessor_ops_metadata_hash - | None -> Lwt.return context) - >>= fun context -> + let*! test_chain = Context_ops.get_test_chain context in + let* context = + match test_chain with + | Not_running -> return context + | Running {expiration; _} -> + if Time.Protocol.(expiration <= timestamp) then + let*! context = Context_ops.add_test_chain context Not_running in + return context + else return context + | Forking _ -> assert false + in + let*! context = + match predecessor_block_metadata_hash with + | Some predecessor_block_metadata_hash -> + Context_ops.add_predecessor_block_metadata_hash + context + predecessor_block_metadata_hash + | None -> Lwt.return context + in + let*! context = + match predecessor_ops_metadata_hash with + | Some predecessor_ops_metadata_hash -> + Context_ops.add_predecessor_ops_metadata_hash + context + predecessor_ops_metadata_hash + | None -> Lwt.return context + in let context = Context_ops.hash ~time:timestamp ?message context in (* For the time being, we still fully build the block while we build confidence to fully unplug the baker validation. The resulting @@ -124,6 +129,7 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info ~timestamp ~liquidity_baking_toggle_vote ~user_activated_upgrades fees_config ~seed_nonce_hash ~payload_round simulation_mode simulation_kind constants = + let open Lwt_result_syntax in let predecessor_block = (pred_info : Baking_state.block_info) in let hard_gas_limit_per_block = constants.Constants.Parametric.hard_gas_limit_per_block @@ -131,7 +137,7 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info let chain = `Hash chain_id in let check_protocol_changed ~(validation_result : Tezos_protocol_environment.validation_result) = - Context_ops.get_protocol validation_result.context >>= fun next_protocol -> + let*! next_protocol = Context_ops.get_protocol validation_result.context in let next_protocol = match Tezos_base.Block_header.get_forced_protocol_upgrade @@ -157,14 +163,15 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info ~liquidity_baking_toggle_vote () in - Node_rpc.preapply_block - cctxt - ~chain - ~head:predecessor_block.hash - ~timestamp - ~protocol_data:faked_protocol_data - filtered_operations - >>=? fun (shell_header, preapply_result) -> + let* shell_header, preapply_result = + Node_rpc.preapply_block + cctxt + ~chain + ~head:predecessor_block.hash + ~timestamp + ~protocol_data:faked_protocol_data + filtered_operations + in (* only retain valid operations *) let operations = List.map (fun l -> List.map snd l.Preapply_result.applied) preapply_result @@ -189,59 +196,64 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info ~liquidity_baking_toggle_vote () in - Baking_simulator.begin_construction - ~timestamp - ~protocol_data:faked_protocol_data - context_index - predecessor_block - chain_id - >>=? fun incremental -> - Operation_selection.filter_operations_with_simulation - incremental - fees_config - ~hard_gas_limit_per_block - operation_pool - >>=? fun { - Operation_selection.operations; - validation_result; - operations_hash; - _; - } -> - check_protocol_changed ~validation_result >>=? fun changed -> + let* incremental = + Baking_simulator.begin_construction + ~timestamp + ~protocol_data:faked_protocol_data + context_index + predecessor_block + chain_id + in + + let* {Operation_selection.operations; validation_result; operations_hash; _} + = + Operation_selection.filter_operations_with_simulation + incremental + fees_config + ~hard_gas_limit_per_block + operation_pool + in + let* changed = check_protocol_changed ~validation_result in if changed then (* Fallback to processing via node, which knows both old and new protocol. *) filter_via_node ~operation_pool else - protect - ~on_error:(fun _ -> return_none) - (fun () -> - Shell_services.Blocks.metadata_hash - cctxt - ~block:(`Hash (predecessor_block.hash, 0)) - ~chain - () - >>=? fun pred_block_metadata_hash -> - return (Some pred_block_metadata_hash)) - >>=? fun pred_block_metadata_hash -> - protect - ~on_error:(fun _ -> return_none) - (fun () -> - Shell_services.Blocks.Operation_metadata_hashes.root - cctxt - ~block:(`Hash (predecessor_block.hash, 0)) - ~chain - () - >>=? fun pred_op_metadata_hash -> return (Some pred_op_metadata_hash)) - >>=? fun pred_op_metadata_hash -> - finalize_block_header - incremental.header - timestamp - validation_result - operations_hash - pred_block_metadata_hash - pred_op_metadata_hash - predecessor_block.resulting_context_hash - >>=? fun shell_header -> + let* pred_block_metadata_hash = + protect + ~on_error:(fun _ -> return_none) + (fun () -> + let+ pred_block_metadata_hash = + Shell_services.Blocks.metadata_hash + cctxt + ~block:(`Hash (predecessor_block.hash, 0)) + ~chain + () + in + Some pred_block_metadata_hash) + in + let* pred_op_metadata_hash = + protect + ~on_error:(fun _ -> return_none) + (fun () -> + let+ pred_op_metadata_hash = + Shell_services.Blocks.Operation_metadata_hashes.root + cctxt + ~block:(`Hash (predecessor_block.hash, 0)) + ~chain + () + in + Some pred_op_metadata_hash) + in + let* shell_header = + finalize_block_header + incremental.header + timestamp + validation_result + operations_hash + pred_block_metadata_hash + pred_op_metadata_hash + predecessor_block.resulting_context_hash + in let operations = List.map (List.map convert_operation) operations in let payload_hash = let operation_hashes = @@ -265,14 +277,15 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info ~payload_round () in - Node_rpc.preapply_block - cctxt - ~chain - ~head:predecessor_block.hash - ~timestamp - ~protocol_data:faked_protocol_data - operations - >>=? fun (shell_header, _preapply_result) -> + let* shell_header, _preapply_result = + Node_rpc.preapply_block + cctxt + ~chain + ~head:predecessor_block.hash + ~timestamp + ~protocol_data:faked_protocol_data + operations + in let operations = List.map (List.map convert_operation) operations in return (shell_header, operations, payload_hash) in @@ -285,20 +298,22 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info ~payload_round () in - Shell_services.Chain.chain_id cctxt ~chain () >>=? fun chain_id -> - Baking_simulator.begin_construction - ~timestamp - ~protocol_data:faked_protocol_data - context_index - predecessor_block - chain_id - >>=? fun incremental -> + let* chain_id = Shell_services.Chain.chain_id cctxt ~chain () in + let* incremental = + Baking_simulator.begin_construction + ~timestamp + ~protocol_data:faked_protocol_data + context_index + predecessor_block + chain_id + in (* We still need to filter endorsements. Two endorsements could be referring to the same slot. *) - Operation_selection.filter_consensus_operations_only - incremental - ordered_pool - >>=? fun (incremental, ordered_pool) -> + let* incremental, ordered_pool = + Operation_selection.filter_consensus_operations_only + incremental + ordered_pool + in let operations = Operation_pool.ordered_to_list_list ordered_pool in let operations_hash = Operation_list_list_hash.compute @@ -312,43 +327,50 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info let incremental = {incremental with header = {incremental.header with operations_hash}} in - Baking_simulator.finalize_construction incremental - >>=? fun (validation_result, _) -> - check_protocol_changed ~validation_result >>=? fun changed -> + let* validation_result, _ = + Baking_simulator.finalize_construction incremental + in + let* changed = check_protocol_changed ~validation_result in if changed then (* Fallback to processing via node, which knows both old and new protocol. *) apply_via_node ~ordered_pool ~payload_hash else - protect - ~on_error:(fun _ -> return_none) - (fun () -> - Shell_services.Blocks.metadata_hash - cctxt - ~block:(`Hash (predecessor_block.hash, 0)) - ~chain - () - >>=? fun pred_block_metadata_hash -> - return (Some pred_block_metadata_hash)) - >>=? fun pred_block_metadata_hash -> - protect - ~on_error:(fun _ -> return_none) - (fun () -> - Shell_services.Blocks.Operation_metadata_hashes.root - cctxt - ~block:(`Hash (predecessor_block.hash, 0)) - ~chain - () - >>=? fun pred_op_metadata_hash -> return (Some pred_op_metadata_hash)) - >>=? fun pred_op_metadata_hash -> - finalize_block_header - incremental.header - timestamp - validation_result - operations_hash - pred_block_metadata_hash - pred_op_metadata_hash - predecessor_block.resulting_context_hash - >>=? fun shell_header -> + let* pred_block_metadata_hash = + protect + ~on_error:(fun _ -> return_none) + (fun () -> + let+ pred_block_metadata_hash = + Shell_services.Blocks.metadata_hash + cctxt + ~block:(`Hash (predecessor_block.hash, 0)) + ~chain + () + in + Some pred_block_metadata_hash) + in + let* pred_op_metadata_hash = + protect + ~on_error:(fun _ -> return_none) + (fun () -> + let+ pred_op_metadata_hash = + Shell_services.Blocks.Operation_metadata_hashes.root + cctxt + ~block:(`Hash (predecessor_block.hash, 0)) + ~chain + () + in + Some pred_op_metadata_hash) + in + let* shell_header = + finalize_block_header + incremental.header + timestamp + validation_result + operations_hash + pred_block_metadata_hash + pred_op_metadata_hash + predecessor_block.resulting_context_hash + in let operations = List.map (List.map convert_operation) operations in return (shell_header, operations, payload_hash) in @@ -365,27 +387,30 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info Filter filtered_pool | Apply _ as x -> x in - (match (simulation_mode, simulation_kind) with - | Baking_state.Node, Filter operation_pool -> filter_via_node ~operation_pool - | Node, Apply {ordered_pool; payload_hash} -> - apply_via_node ~ordered_pool ~payload_hash - | Local context_index, Filter operation_pool -> - filter_with_context ~context_index ~operation_pool - | Local context_index, Apply {ordered_pool; payload_hash} -> - apply_with_context ~context_index ~ordered_pool ~payload_hash) - >>=? fun (shell_header, operations, payload_hash) -> - Baking_pow.mine - ~proof_of_work_threshold:constants.proof_of_work_threshold - shell_header - (fun proof_of_work_nonce -> - { - Block_header.payload_hash; - payload_round; - seed_nonce_hash; - proof_of_work_nonce; - liquidity_baking_toggle_vote; - }) - >>=? fun contents -> + let* shell_header, operations, payload_hash = + match (simulation_mode, simulation_kind) with + | Baking_state.Node, Filter operation_pool -> + filter_via_node ~operation_pool + | Node, Apply {ordered_pool; payload_hash} -> + apply_via_node ~ordered_pool ~payload_hash + | Local context_index, Filter operation_pool -> + filter_with_context ~context_index ~operation_pool + | Local context_index, Apply {ordered_pool; payload_hash} -> + apply_with_context ~context_index ~ordered_pool ~payload_hash + in + let* contents = + Baking_pow.mine + ~proof_of_work_threshold:constants.proof_of_work_threshold + shell_header + (fun proof_of_work_nonce -> + { + Block_header.payload_hash; + payload_round; + seed_nonce_hash; + proof_of_work_nonce; + liquidity_baking_toggle_vote; + }) + in let unsigned_block_header = { Block_header.shell = shell_header; diff --git a/src/proto_alpha/lib_delegate/block_forge.ml b/src/proto_alpha/lib_delegate/block_forge.ml index 062306d6367d..5b71c801341a 100644 --- a/src/proto_alpha/lib_delegate/block_forge.ml +++ b/src/proto_alpha/lib_delegate/block_forge.ml @@ -68,33 +68,38 @@ let convert_operation (op : packed_operation) : Tezos_base.Operation.t = let finalize_block_header shell_header timestamp validation_result operations_hash predecessor_block_metadata_hash predecessor_ops_metadata_hash predecessor_resulting_context = + let open Lwt_result_syntax in let {Tezos_protocol_environment.context; fitness; message; _} = validation_result in let validation_passes = List.length Main.validation_passes in - (Context_ops.get_test_chain context >>= function - | Not_running -> return context - | Running {expiration; _} -> - if Time.Protocol.(expiration <= timestamp) then - Context_ops.add_test_chain context Not_running >>= fun context -> - return context - else return context - | Forking _ -> assert false) - >>=? fun context -> - (match predecessor_block_metadata_hash with - | Some predecessor_block_metadata_hash -> - Context_ops.add_predecessor_block_metadata_hash - context - predecessor_block_metadata_hash - | None -> Lwt.return context) - >>= fun context -> - (match predecessor_ops_metadata_hash with - | Some predecessor_ops_metadata_hash -> - Context_ops.add_predecessor_ops_metadata_hash - context - predecessor_ops_metadata_hash - | None -> Lwt.return context) - >>= fun context -> + let*! test_chain = Context_ops.get_test_chain context in + let* context = + match test_chain with + | Not_running -> return context + | Running {expiration; _} -> + if Time.Protocol.(expiration <= timestamp) then + let*! context = Context_ops.add_test_chain context Not_running in + return context + else return context + | Forking _ -> assert false + in + let*! context = + match predecessor_block_metadata_hash with + | Some predecessor_block_metadata_hash -> + Context_ops.add_predecessor_block_metadata_hash + context + predecessor_block_metadata_hash + | None -> Lwt.return context + in + let*! context = + match predecessor_ops_metadata_hash with + | Some predecessor_ops_metadata_hash -> + Context_ops.add_predecessor_ops_metadata_hash + context + predecessor_ops_metadata_hash + | None -> Lwt.return context + in let context = Context_ops.hash ~time:timestamp ?message context in (* For the time being, we still fully build the block while we build confidence to fully unplug the baker validation. The resulting @@ -124,6 +129,7 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info ~timestamp ~liquidity_baking_toggle_vote ~user_activated_upgrades fees_config ~seed_nonce_hash ~payload_round simulation_mode simulation_kind constants = + let open Lwt_result_syntax in let predecessor_block = (pred_info : Baking_state.block_info) in let hard_gas_limit_per_block = constants.Constants.Parametric.hard_gas_limit_per_block @@ -131,7 +137,7 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info let chain = `Hash chain_id in let check_protocol_changed ~(validation_result : Tezos_protocol_environment.validation_result) = - Context_ops.get_protocol validation_result.context >>= fun next_protocol -> + let*! next_protocol = Context_ops.get_protocol validation_result.context in let next_protocol = match Tezos_base.Block_header.get_forced_protocol_upgrade @@ -157,14 +163,15 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info ~liquidity_baking_toggle_vote () in - Node_rpc.preapply_block - cctxt - ~chain - ~head:predecessor_block.hash - ~timestamp - ~protocol_data:faked_protocol_data - filtered_operations - >>=? fun (shell_header, preapply_result) -> + let* shell_header, preapply_result = + Node_rpc.preapply_block + cctxt + ~chain + ~head:predecessor_block.hash + ~timestamp + ~protocol_data:faked_protocol_data + filtered_operations + in (* only retain valid operations *) let operations = List.map (fun l -> List.map snd l.Preapply_result.applied) preapply_result @@ -189,59 +196,64 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info ~liquidity_baking_toggle_vote () in - Baking_simulator.begin_construction - ~timestamp - ~protocol_data:faked_protocol_data - context_index - predecessor_block - chain_id - >>=? fun incremental -> - Operation_selection.filter_operations_with_simulation - incremental - fees_config - ~hard_gas_limit_per_block - operation_pool - >>=? fun { - Operation_selection.operations; - validation_result; - operations_hash; - _; - } -> - check_protocol_changed ~validation_result >>=? fun changed -> + let* incremental = + Baking_simulator.begin_construction + ~timestamp + ~protocol_data:faked_protocol_data + context_index + predecessor_block + chain_id + in + + let* {Operation_selection.operations; validation_result; operations_hash; _} + = + Operation_selection.filter_operations_with_simulation + incremental + fees_config + ~hard_gas_limit_per_block + operation_pool + in + let* changed = check_protocol_changed ~validation_result in if changed then (* Fallback to processing via node, which knows both old and new protocol. *) filter_via_node ~operation_pool else - protect - ~on_error:(fun _ -> return_none) - (fun () -> - Shell_services.Blocks.metadata_hash - cctxt - ~block:(`Hash (predecessor_block.hash, 0)) - ~chain - () - >>=? fun pred_block_metadata_hash -> - return (Some pred_block_metadata_hash)) - >>=? fun pred_block_metadata_hash -> - protect - ~on_error:(fun _ -> return_none) - (fun () -> - Shell_services.Blocks.Operation_metadata_hashes.root - cctxt - ~block:(`Hash (predecessor_block.hash, 0)) - ~chain - () - >>=? fun pred_op_metadata_hash -> return (Some pred_op_metadata_hash)) - >>=? fun pred_op_metadata_hash -> - finalize_block_header - incremental.header - timestamp - validation_result - operations_hash - pred_block_metadata_hash - pred_op_metadata_hash - predecessor_block.resulting_context_hash - >>=? fun shell_header -> + let* pred_block_metadata_hash = + protect + ~on_error:(fun _ -> return_none) + (fun () -> + let+ pred_block_metadata_hash = + Shell_services.Blocks.metadata_hash + cctxt + ~block:(`Hash (predecessor_block.hash, 0)) + ~chain + () + in + Some pred_block_metadata_hash) + in + let* pred_op_metadata_hash = + protect + ~on_error:(fun _ -> return_none) + (fun () -> + let+ pred_op_metadata_hash = + Shell_services.Blocks.Operation_metadata_hashes.root + cctxt + ~block:(`Hash (predecessor_block.hash, 0)) + ~chain + () + in + Some pred_op_metadata_hash) + in + let* shell_header = + finalize_block_header + incremental.header + timestamp + validation_result + operations_hash + pred_block_metadata_hash + pred_op_metadata_hash + predecessor_block.resulting_context_hash + in let operations = List.map (List.map convert_operation) operations in let payload_hash = let operation_hashes = @@ -265,14 +277,15 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info ~payload_round () in - Node_rpc.preapply_block - cctxt - ~chain - ~head:predecessor_block.hash - ~timestamp - ~protocol_data:faked_protocol_data - operations - >>=? fun (shell_header, _preapply_result) -> + let* shell_header, _preapply_result = + Node_rpc.preapply_block + cctxt + ~chain + ~head:predecessor_block.hash + ~timestamp + ~protocol_data:faked_protocol_data + operations + in let operations = List.map (List.map convert_operation) operations in return (shell_header, operations, payload_hash) in @@ -285,20 +298,22 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info ~payload_round () in - Shell_services.Chain.chain_id cctxt ~chain () >>=? fun chain_id -> - Baking_simulator.begin_construction - ~timestamp - ~protocol_data:faked_protocol_data - context_index - predecessor_block - chain_id - >>=? fun incremental -> + let* chain_id = Shell_services.Chain.chain_id cctxt ~chain () in + let* incremental = + Baking_simulator.begin_construction + ~timestamp + ~protocol_data:faked_protocol_data + context_index + predecessor_block + chain_id + in (* We still need to filter endorsements. Two endorsements could be referring to the same slot. *) - Operation_selection.filter_consensus_operations_only - incremental - ordered_pool - >>=? fun (incremental, ordered_pool) -> + let* incremental, ordered_pool = + Operation_selection.filter_consensus_operations_only + incremental + ordered_pool + in let operations = Operation_pool.ordered_to_list_list ordered_pool in let operations_hash = Operation_list_list_hash.compute @@ -312,43 +327,50 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info let incremental = {incremental with header = {incremental.header with operations_hash}} in - Baking_simulator.finalize_construction incremental - >>=? fun (validation_result, _) -> - check_protocol_changed ~validation_result >>=? fun changed -> + let* validation_result, _ = + Baking_simulator.finalize_construction incremental + in + let* changed = check_protocol_changed ~validation_result in if changed then (* Fallback to processing via node, which knows both old and new protocol. *) apply_via_node ~ordered_pool ~payload_hash else - protect - ~on_error:(fun _ -> return_none) - (fun () -> - Shell_services.Blocks.metadata_hash - cctxt - ~block:(`Hash (predecessor_block.hash, 0)) - ~chain - () - >>=? fun pred_block_metadata_hash -> - return (Some pred_block_metadata_hash)) - >>=? fun pred_block_metadata_hash -> - protect - ~on_error:(fun _ -> return_none) - (fun () -> - Shell_services.Blocks.Operation_metadata_hashes.root - cctxt - ~block:(`Hash (predecessor_block.hash, 0)) - ~chain - () - >>=? fun pred_op_metadata_hash -> return (Some pred_op_metadata_hash)) - >>=? fun pred_op_metadata_hash -> - finalize_block_header - incremental.header - timestamp - validation_result - operations_hash - pred_block_metadata_hash - pred_op_metadata_hash - predecessor_block.resulting_context_hash - >>=? fun shell_header -> + let* pred_block_metadata_hash = + protect + ~on_error:(fun _ -> return_none) + (fun () -> + let+ pred_block_metadata_hash = + Shell_services.Blocks.metadata_hash + cctxt + ~block:(`Hash (predecessor_block.hash, 0)) + ~chain + () + in + Some pred_block_metadata_hash) + in + let* pred_op_metadata_hash = + protect + ~on_error:(fun _ -> return_none) + (fun () -> + let+ pred_op_metadata_hash = + Shell_services.Blocks.Operation_metadata_hashes.root + cctxt + ~block:(`Hash (predecessor_block.hash, 0)) + ~chain + () + in + Some pred_op_metadata_hash) + in + let* shell_header = + finalize_block_header + incremental.header + timestamp + validation_result + operations_hash + pred_block_metadata_hash + pred_op_metadata_hash + predecessor_block.resulting_context_hash + in let operations = List.map (List.map convert_operation) operations in return (shell_header, operations, payload_hash) in @@ -365,27 +387,30 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info Filter filtered_pool | Apply _ as x -> x in - (match (simulation_mode, simulation_kind) with - | Baking_state.Node, Filter operation_pool -> filter_via_node ~operation_pool - | Node, Apply {ordered_pool; payload_hash} -> - apply_via_node ~ordered_pool ~payload_hash - | Local context_index, Filter operation_pool -> - filter_with_context ~context_index ~operation_pool - | Local context_index, Apply {ordered_pool; payload_hash} -> - apply_with_context ~context_index ~ordered_pool ~payload_hash) - >>=? fun (shell_header, operations, payload_hash) -> - Baking_pow.mine - ~proof_of_work_threshold:constants.proof_of_work_threshold - shell_header - (fun proof_of_work_nonce -> - { - Block_header.payload_hash; - payload_round; - seed_nonce_hash; - proof_of_work_nonce; - liquidity_baking_toggle_vote; - }) - >>=? fun contents -> + let* shell_header, operations, payload_hash = + match (simulation_mode, simulation_kind) with + | Baking_state.Node, Filter operation_pool -> + filter_via_node ~operation_pool + | Node, Apply {ordered_pool; payload_hash} -> + apply_via_node ~ordered_pool ~payload_hash + | Local context_index, Filter operation_pool -> + filter_with_context ~context_index ~operation_pool + | Local context_index, Apply {ordered_pool; payload_hash} -> + apply_with_context ~context_index ~ordered_pool ~payload_hash + in + let* contents = + Baking_pow.mine + ~proof_of_work_threshold:constants.proof_of_work_threshold + shell_header + (fun proof_of_work_nonce -> + { + Block_header.payload_hash; + payload_round; + seed_nonce_hash; + proof_of_work_nonce; + liquidity_baking_toggle_vote; + }) + in let unsigned_block_header = { Block_header.shell = shell_header; -- GitLab From aaaee5dda5ae13b1223f3a044c6a74ed565f5da9 Mon Sep 17 00:00:00 2001 From: Albin Coquereau Date: Wed, 18 Jan 2023 14:44:42 +0100 Subject: [PATCH 08/46] lib_delegate: move check_protocol_changed outside forge function --- .../lib_delegate/block_forge.ml | 50 ++++++++++++------- src/proto_alpha/lib_delegate/block_forge.ml | 50 ++++++++++++------- 2 files changed, 64 insertions(+), 36 deletions(-) diff --git a/src/proto_016_PtMumbai/lib_delegate/block_forge.ml b/src/proto_016_PtMumbai/lib_delegate/block_forge.ml index 83b0d58a4a8b..d9192de45b0f 100644 --- a/src/proto_016_PtMumbai/lib_delegate/block_forge.ml +++ b/src/proto_016_PtMumbai/lib_delegate/block_forge.ml @@ -40,6 +40,11 @@ type simulation_kind = type simulation_mode = Local of Context.index | Node +(* [forge_faked_protocol_data ?payload_hash ~payload_round ~seed_nonce_hash + ~liquidity_baking_toggle_vote] forges a fake [block_header_data] with + [payload_hash] ([zero] by default), [payload_round], [seed_nonce_hash], + [liquidity_baking_toggle_vote] and with an empty [proof_of_work_nonce] and a + dummy [signature]. *) let forge_faked_protocol_data ?(payload_hash = Block_payload_hash.zero) ~payload_round ~seed_nonce_hash ~liquidity_baking_toggle_vote () = Block_header. @@ -64,7 +69,6 @@ let convert_operation (op : packed_operation) : Tezos_base.Operation.t = op.protocol_data; } -(* Build the block header : mimics node prevalidation *) let finalize_block_header shell_header timestamp validation_result operations_hash predecessor_block_metadata_hash predecessor_ops_metadata_hash predecessor_resulting_context = @@ -125,6 +129,21 @@ let retain_live_operations_only ~live_blocks operation_pool = Block_hash.Set.mem shell.branch live_blocks) operation_pool +let check_protocol_changed ~user_activated_upgrades ~level + ~(validation_result : Tezos_protocol_environment.validation_result) = + let open Lwt_result_syntax in + let*! next_protocol = Context_ops.get_protocol validation_result.context in + let next_protocol = + match + Tezos_base.Block_header.get_forced_protocol_upgrade + ~user_activated_upgrades + ~level + with + | None -> next_protocol + | Some hash -> hash + in + return Protocol_hash.(Protocol.hash <> next_protocol) + let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info ~timestamp ~liquidity_baking_toggle_vote ~user_activated_upgrades fees_config ~seed_nonce_hash ~payload_round simulation_mode simulation_kind @@ -135,20 +154,6 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info constants.Constants.Parametric.hard_gas_limit_per_block in let chain = `Hash chain_id in - let check_protocol_changed - ~(validation_result : Tezos_protocol_environment.validation_result) = - let*! next_protocol = Context_ops.get_protocol validation_result.context in - let next_protocol = - match - Tezos_base.Block_header.get_forced_protocol_upgrade - ~user_activated_upgrades - ~level:(Int32.succ predecessor_block.shell.level) - with - | None -> next_protocol - | Some hash -> hash - in - return Protocol_hash.(Protocol.hash <> next_protocol) - in let filter_via_node ~operation_pool = let filtered_operations = Operation_selection.filter_operations_without_simulation @@ -204,7 +209,6 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info predecessor_block chain_id in - let* {Operation_selection.operations; validation_result; operations_hash; _} = Operation_selection.filter_operations_with_simulation @@ -213,7 +217,12 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info ~hard_gas_limit_per_block operation_pool in - let* changed = check_protocol_changed ~validation_result in + let* changed = + check_protocol_changed + ~level:(Int32.succ predecessor_block.shell.level) + ~user_activated_upgrades + ~validation_result + in if changed then (* Fallback to processing via node, which knows both old and new protocol. *) filter_via_node ~operation_pool @@ -330,7 +339,12 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info let* validation_result, _ = Baking_simulator.finalize_construction incremental in - let* changed = check_protocol_changed ~validation_result in + let* changed = + check_protocol_changed + ~level:(Int32.succ predecessor_block.shell.level) + ~user_activated_upgrades + ~validation_result + in if changed then (* Fallback to processing via node, which knows both old and new protocol. *) apply_via_node ~ordered_pool ~payload_hash diff --git a/src/proto_alpha/lib_delegate/block_forge.ml b/src/proto_alpha/lib_delegate/block_forge.ml index 5b71c801341a..0b5bd9db7649 100644 --- a/src/proto_alpha/lib_delegate/block_forge.ml +++ b/src/proto_alpha/lib_delegate/block_forge.ml @@ -40,6 +40,11 @@ type simulation_kind = type simulation_mode = Local of Context.index | Node +(* [forge_faked_protocol_data ?payload_hash ~payload_round ~seed_nonce_hash + ~liquidity_baking_toggle_vote] forges a fake [block_header_data] with + [payload_hash] ([zero] by default), [payload_round], [seed_nonce_hash], + [liquidity_baking_toggle_vote] and with an empty [proof_of_work_nonce] and a + dummy [signature]. *) let forge_faked_protocol_data ?(payload_hash = Block_payload_hash.zero) ~payload_round ~seed_nonce_hash ~liquidity_baking_toggle_vote () = Block_header. @@ -64,7 +69,6 @@ let convert_operation (op : packed_operation) : Tezos_base.Operation.t = op.protocol_data; } -(* Build the block header : mimics node prevalidation *) let finalize_block_header shell_header timestamp validation_result operations_hash predecessor_block_metadata_hash predecessor_ops_metadata_hash predecessor_resulting_context = @@ -125,6 +129,21 @@ let retain_live_operations_only ~live_blocks operation_pool = Block_hash.Set.mem shell.branch live_blocks) operation_pool +let check_protocol_changed ~user_activated_upgrades ~level + ~(validation_result : Tezos_protocol_environment.validation_result) = + let open Lwt_result_syntax in + let*! next_protocol = Context_ops.get_protocol validation_result.context in + let next_protocol = + match + Tezos_base.Block_header.get_forced_protocol_upgrade + ~user_activated_upgrades + ~level + with + | None -> next_protocol + | Some hash -> hash + in + return Protocol_hash.(Protocol.hash <> next_protocol) + let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info ~timestamp ~liquidity_baking_toggle_vote ~user_activated_upgrades fees_config ~seed_nonce_hash ~payload_round simulation_mode simulation_kind @@ -135,20 +154,6 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info constants.Constants.Parametric.hard_gas_limit_per_block in let chain = `Hash chain_id in - let check_protocol_changed - ~(validation_result : Tezos_protocol_environment.validation_result) = - let*! next_protocol = Context_ops.get_protocol validation_result.context in - let next_protocol = - match - Tezos_base.Block_header.get_forced_protocol_upgrade - ~user_activated_upgrades - ~level:(Int32.succ predecessor_block.shell.level) - with - | None -> next_protocol - | Some hash -> hash - in - return Protocol_hash.(Protocol.hash <> next_protocol) - in let filter_via_node ~operation_pool = let filtered_operations = Operation_selection.filter_operations_without_simulation @@ -204,7 +209,6 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info predecessor_block chain_id in - let* {Operation_selection.operations; validation_result; operations_hash; _} = Operation_selection.filter_operations_with_simulation @@ -213,7 +217,12 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info ~hard_gas_limit_per_block operation_pool in - let* changed = check_protocol_changed ~validation_result in + let* changed = + check_protocol_changed + ~level:(Int32.succ predecessor_block.shell.level) + ~user_activated_upgrades + ~validation_result + in if changed then (* Fallback to processing via node, which knows both old and new protocol. *) filter_via_node ~operation_pool @@ -330,7 +339,12 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info let* validation_result, _ = Baking_simulator.finalize_construction incremental in - let* changed = check_protocol_changed ~validation_result in + let* changed = + check_protocol_changed + ~level:(Int32.succ predecessor_block.shell.level) + ~user_activated_upgrades + ~validation_result + in if changed then (* Fallback to processing via node, which knows both old and new protocol. *) apply_via_node ~ordered_pool ~payload_hash -- GitLab From bfa6a2921a7a21084b86480edd005c046f83ff1a Mon Sep 17 00:00:00 2001 From: Albin Coquereau Date: Wed, 18 Jan 2023 17:07:47 +0100 Subject: [PATCH 09/46] lib_delegate: split block_forge.forge --- .../lib_delegate/block_forge.ml | 513 ++++++++++-------- src/proto_alpha/lib_delegate/block_forge.ml | 513 ++++++++++-------- 2 files changed, 578 insertions(+), 448 deletions(-) diff --git a/src/proto_016_PtMumbai/lib_delegate/block_forge.ml b/src/proto_016_PtMumbai/lib_delegate/block_forge.ml index d9192de45b0f..5ca78bd91bba 100644 --- a/src/proto_016_PtMumbai/lib_delegate/block_forge.ml +++ b/src/proto_016_PtMumbai/lib_delegate/block_forge.ml @@ -144,43 +144,119 @@ let check_protocol_changed ~user_activated_upgrades ~level in return Protocol_hash.(Protocol.hash <> next_protocol) -let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info - ~timestamp ~liquidity_baking_toggle_vote ~user_activated_upgrades - fees_config ~seed_nonce_hash ~payload_round simulation_mode simulation_kind - constants = +let filter_via_node ~chain_id ~fees_config ~hard_gas_limit_per_block + ~faked_protocol_data ~timestamp ~(pred_info : Baking_state.block_info) + ~payload_round ~operation_pool cctxt = let open Lwt_result_syntax in - let predecessor_block = (pred_info : Baking_state.block_info) in - let hard_gas_limit_per_block = - constants.Constants.Parametric.hard_gas_limit_per_block - in let chain = `Hash chain_id in - let filter_via_node ~operation_pool = - let filtered_operations = - Operation_selection.filter_operations_without_simulation - fees_config - ~hard_gas_limit_per_block - operation_pool + let filtered_operations = + Operation_selection.filter_operations_without_simulation + fees_config + ~hard_gas_limit_per_block + operation_pool + in + let* shell_header, preapply_result = + Node_rpc.preapply_block + cctxt + ~chain + ~head:pred_info.hash + ~timestamp + ~protocol_data:faked_protocol_data + filtered_operations + in + (* only retain valid operations *) + let operations = + List.map (fun l -> List.map snd l.Preapply_result.applied) preapply_result + in + let payload_hash = + let operation_hashes = + Stdlib.List.tl operations |> List.flatten + |> List.map Tezos_base.Operation.hash in - let faked_protocol_data = - forge_faked_protocol_data - ~payload_round - ~seed_nonce_hash - ~liquidity_baking_toggle_vote - () + Block_payload.hash + ~predecessor_hash:shell_header.predecessor + ~payload_round + operation_hashes + in + return (shell_header, operations, payload_hash) + +let filter_with_context ~chain_id ~fees_config ~hard_gas_limit_per_block + ~faked_protocol_data ~user_activated_upgrades ~timestamp + ~(pred_info : Baking_state.block_info) ~context_index ~payload_round + ~operation_pool cctxt = + let open Lwt_result_syntax in + let chain = `Hash chain_id in + let* incremental = + Baking_simulator.begin_construction + ~timestamp + ~protocol_data:faked_protocol_data + context_index + pred_info + chain_id + in + let* {Operation_selection.operations; validation_result; operations_hash; _} = + Operation_selection.filter_operations_with_simulation + incremental + fees_config + ~hard_gas_limit_per_block + operation_pool + in + let* changed = + check_protocol_changed + ~level:(Int32.succ pred_info.shell.level) + ~user_activated_upgrades + ~validation_result + in + if changed then + (* Fallback to processing via node, which knows both old and new protocol. *) + filter_via_node + ~chain_id + ~fees_config + ~hard_gas_limit_per_block + ~faked_protocol_data + ~timestamp + ~pred_info + ~payload_round + ~operation_pool + cctxt + else + let* pred_block_metadata_hash = + protect + ~on_error:(fun _ -> return_none) + (fun () -> + let+ pred_block_metadata_hash = + Shell_services.Blocks.metadata_hash + cctxt + ~block:(`Hash (pred_info.hash, 0)) + ~chain + () + in + Some pred_block_metadata_hash) in - let* shell_header, preapply_result = - Node_rpc.preapply_block - cctxt - ~chain - ~head:predecessor_block.hash - ~timestamp - ~protocol_data:faked_protocol_data - filtered_operations + let* pred_op_metadata_hash = + protect + ~on_error:(fun _ -> return_none) + (fun () -> + let+ pred_op_metadata_hash = + Shell_services.Blocks.Operation_metadata_hashes.root + cctxt + ~block:(`Hash (pred_info.hash, 0)) + ~chain + () + in + Some pred_op_metadata_hash) in - (* only retain valid operations *) - let operations = - List.map (fun l -> List.map snd l.Preapply_result.applied) preapply_result + let* shell_header = + finalize_block_header + incremental.header + timestamp + validation_result + operations_hash + pred_block_metadata_hash + pred_op_metadata_hash + pred_info.resulting_context_hash in + let operations = List.map (List.map convert_operation) operations in let payload_hash = let operation_hashes = Stdlib.List.tl operations |> List.flatten @@ -192,201 +268,124 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info operation_hashes in return (shell_header, operations, payload_hash) + +let apply_via_node ~chain_id ~faked_protocol_data ~timestamp + ~(pred_info : Baking_state.block_info) ~ordered_pool ~payload_hash cctxt = + let open Lwt_result_syntax in + let chain = `Hash chain_id in + let operations = Operation_pool.ordered_to_list_list ordered_pool in + let* shell_header, _preapply_result = + Node_rpc.preapply_block + cctxt + ~chain + ~head:pred_info.hash + ~timestamp + ~protocol_data:faked_protocol_data + operations in - let filter_with_context ~context_index ~operation_pool = - let faked_protocol_data = - forge_faked_protocol_data - ~payload_round - ~seed_nonce_hash - ~liquidity_baking_toggle_vote - () - in - let* incremental = - Baking_simulator.begin_construction - ~timestamp - ~protocol_data:faked_protocol_data - context_index - predecessor_block - chain_id - in - let* {Operation_selection.operations; validation_result; operations_hash; _} - = - Operation_selection.filter_operations_with_simulation - incremental - fees_config - ~hard_gas_limit_per_block - operation_pool - in - let* changed = - check_protocol_changed - ~level:(Int32.succ predecessor_block.shell.level) - ~user_activated_upgrades - ~validation_result - in - if changed then - (* Fallback to processing via node, which knows both old and new protocol. *) - filter_via_node ~operation_pool - else - let* pred_block_metadata_hash = - protect - ~on_error:(fun _ -> return_none) - (fun () -> - let+ pred_block_metadata_hash = - Shell_services.Blocks.metadata_hash - cctxt - ~block:(`Hash (predecessor_block.hash, 0)) - ~chain - () - in - Some pred_block_metadata_hash) - in - let* pred_op_metadata_hash = - protect - ~on_error:(fun _ -> return_none) - (fun () -> - let+ pred_op_metadata_hash = - Shell_services.Blocks.Operation_metadata_hashes.root - cctxt - ~block:(`Hash (predecessor_block.hash, 0)) - ~chain - () - in - Some pred_op_metadata_hash) - in - let* shell_header = - finalize_block_header - incremental.header - timestamp - validation_result - operations_hash - pred_block_metadata_hash - pred_op_metadata_hash - predecessor_block.resulting_context_hash - in - let operations = List.map (List.map convert_operation) operations in - let payload_hash = - let operation_hashes = - Stdlib.List.tl operations |> List.flatten - |> List.map Tezos_base.Operation.hash - in - Block_payload.hash - ~predecessor_hash:shell_header.predecessor - ~payload_round - operation_hashes - in - return (shell_header, operations, payload_hash) + let operations = List.map (List.map convert_operation) operations in + return (shell_header, operations, payload_hash) + +let apply_with_context ~chain_id ~faked_protocol_data ~user_activated_upgrades + ~timestamp ~(pred_info : Baking_state.block_info) ~ordered_pool + ~context_index ~payload_hash cctxt = + let open Lwt_result_syntax in + let chain = `Hash chain_id in + let* incremental = + Baking_simulator.begin_construction + ~timestamp + ~protocol_data:faked_protocol_data + context_index + pred_info + chain_id in - let apply_via_node ~ordered_pool ~payload_hash = - let operations = Operation_pool.ordered_to_list_list ordered_pool in - let faked_protocol_data = - forge_faked_protocol_data - ~seed_nonce_hash - ~liquidity_baking_toggle_vote - ~payload_hash - ~payload_round - () - in - let* shell_header, _preapply_result = - Node_rpc.preapply_block - cctxt - ~chain - ~head:predecessor_block.hash - ~timestamp - ~protocol_data:faked_protocol_data - operations - in - let operations = List.map (List.map convert_operation) operations in - return (shell_header, operations, payload_hash) + (* We still need to filter endorsements. Two endorsements could be + referring to the same slot. *) + let* incremental, ordered_pool = + Operation_selection.filter_consensus_operations_only + incremental + ordered_pool in - let apply_with_context ~context_index ~ordered_pool ~payload_hash = - let faked_protocol_data = - forge_faked_protocol_data - ~seed_nonce_hash - ~liquidity_baking_toggle_vote - ~payload_hash - ~payload_round - () - in - let* chain_id = Shell_services.Chain.chain_id cctxt ~chain () in - let* incremental = - Baking_simulator.begin_construction - ~timestamp - ~protocol_data:faked_protocol_data - context_index - predecessor_block - chain_id - in - (* We still need to filter endorsements. Two endorsements could be - referring to the same slot. *) - let* incremental, ordered_pool = - Operation_selection.filter_consensus_operations_only - incremental - ordered_pool - in - let operations = Operation_pool.ordered_to_list_list ordered_pool in - let operations_hash = - Operation_list_list_hash.compute - (List.map - (fun sl -> - Operation_list_hash.compute (List.map Operation.hash_packed sl)) - operations) - in - (* We need to compute the final [operations_hash] before - finalizing the block because it will be used in the cache's nonce. *) - let incremental = - {incremental with header = {incremental.header with operations_hash}} + let operations = Operation_pool.ordered_to_list_list ordered_pool in + let operations_hash = + Operation_list_list_hash.compute + (List.map + (fun sl -> + Operation_list_hash.compute (List.map Operation.hash_packed sl)) + operations) + in + (* We need to compute the final [operations_hash] before + finalizing the block because it will be used in the cache's nonce. *) + let incremental = + {incremental with header = {incremental.header with operations_hash}} + in + let* validation_result, _ = + Baking_simulator.finalize_construction incremental + in + let* changed = + check_protocol_changed + ~level:(Int32.succ pred_info.shell.level) + ~user_activated_upgrades + ~validation_result + in + if changed then + (* Fallback to processing via node, which knows both old and new protocol. *) + apply_via_node + ~chain_id + ~faked_protocol_data + ~timestamp + ~pred_info + ~ordered_pool + ~payload_hash + cctxt + else + let* pred_block_metadata_hash = + protect + ~on_error:(fun _ -> return_none) + (fun () -> + let+ pred_block_metadata_hash = + Shell_services.Blocks.metadata_hash + cctxt + ~block:(`Hash (pred_info.hash, 0)) + ~chain + () + in + Some pred_block_metadata_hash) in - let* validation_result, _ = - Baking_simulator.finalize_construction incremental + let* pred_op_metadata_hash = + protect + ~on_error:(fun _ -> return_none) + (fun () -> + let+ pred_op_metadata_hash = + Shell_services.Blocks.Operation_metadata_hashes.root + cctxt + ~block:(`Hash (pred_info.hash, 0)) + ~chain + () + in + Some pred_op_metadata_hash) in - let* changed = - check_protocol_changed - ~level:(Int32.succ predecessor_block.shell.level) - ~user_activated_upgrades - ~validation_result + let* shell_header = + finalize_block_header + incremental.header + timestamp + validation_result + operations_hash + pred_block_metadata_hash + pred_op_metadata_hash + pred_info.resulting_context_hash in - if changed then - (* Fallback to processing via node, which knows both old and new protocol. *) - apply_via_node ~ordered_pool ~payload_hash - else - let* pred_block_metadata_hash = - protect - ~on_error:(fun _ -> return_none) - (fun () -> - let+ pred_block_metadata_hash = - Shell_services.Blocks.metadata_hash - cctxt - ~block:(`Hash (predecessor_block.hash, 0)) - ~chain - () - in - Some pred_block_metadata_hash) - in - let* pred_op_metadata_hash = - protect - ~on_error:(fun _ -> return_none) - (fun () -> - let+ pred_op_metadata_hash = - Shell_services.Blocks.Operation_metadata_hashes.root - cctxt - ~block:(`Hash (predecessor_block.hash, 0)) - ~chain - () - in - Some pred_op_metadata_hash) - in - let* shell_header = - finalize_block_header - incremental.header - timestamp - validation_result - operations_hash - pred_block_metadata_hash - pred_op_metadata_hash - predecessor_block.resulting_context_hash - in - let operations = List.map (List.map convert_operation) operations in - return (shell_header, operations, payload_hash) + let operations = List.map (List.map convert_operation) operations in + return (shell_header, operations, payload_hash) + +(* [forge] a new [unsigned_block] regarding of [simulation_kind] and [simulation_mode] *) +let forge (cctxt : #Protocol_client_context.full) ~chain_id + ~(pred_info : Baking_state.block_info) ~timestamp + ~liquidity_baking_toggle_vote ~user_activated_upgrades fees_config + ~seed_nonce_hash ~payload_round simulation_mode simulation_kind constants = + let open Lwt_result_syntax in + let hard_gas_limit_per_block = + constants.Constants.Parametric.hard_gas_limit_per_block in let simulation_kind = match simulation_kind with @@ -404,13 +403,79 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info let* shell_header, operations, payload_hash = match (simulation_mode, simulation_kind) with | Baking_state.Node, Filter operation_pool -> - filter_via_node ~operation_pool + let faked_protocol_data = + forge_faked_protocol_data + ~payload_round + ~seed_nonce_hash + ~liquidity_baking_toggle_vote + () + in + filter_via_node + ~chain_id + ~faked_protocol_data + ~fees_config + ~hard_gas_limit_per_block + ~timestamp + ~pred_info + ~payload_round + ~operation_pool + cctxt | Node, Apply {ordered_pool; payload_hash} -> - apply_via_node ~ordered_pool ~payload_hash + let faked_protocol_data = + forge_faked_protocol_data + ~payload_hash + ~payload_round + ~seed_nonce_hash + ~liquidity_baking_toggle_vote + () + in + apply_via_node + ~chain_id + ~faked_protocol_data + ~timestamp + ~pred_info + ~ordered_pool + ~payload_hash + cctxt | Local context_index, Filter operation_pool -> - filter_with_context ~context_index ~operation_pool + let faked_protocol_data = + forge_faked_protocol_data + ~payload_round + ~seed_nonce_hash + ~liquidity_baking_toggle_vote + () + in + filter_with_context + ~chain_id + ~faked_protocol_data + ~fees_config + ~hard_gas_limit_per_block + ~user_activated_upgrades + ~timestamp + ~pred_info + ~context_index + ~payload_round + ~operation_pool + cctxt | Local context_index, Apply {ordered_pool; payload_hash} -> - apply_with_context ~context_index ~ordered_pool ~payload_hash + let faked_protocol_data = + forge_faked_protocol_data + ~payload_hash + ~payload_round + ~seed_nonce_hash + ~liquidity_baking_toggle_vote + () + in + apply_with_context + ~chain_id + ~faked_protocol_data + ~user_activated_upgrades + ~timestamp + ~pred_info + ~ordered_pool + ~context_index + ~payload_hash + cctxt in let* contents = Baking_pow.mine diff --git a/src/proto_alpha/lib_delegate/block_forge.ml b/src/proto_alpha/lib_delegate/block_forge.ml index 0b5bd9db7649..36d743ba2a65 100644 --- a/src/proto_alpha/lib_delegate/block_forge.ml +++ b/src/proto_alpha/lib_delegate/block_forge.ml @@ -144,43 +144,119 @@ let check_protocol_changed ~user_activated_upgrades ~level in return Protocol_hash.(Protocol.hash <> next_protocol) -let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info - ~timestamp ~liquidity_baking_toggle_vote ~user_activated_upgrades - fees_config ~seed_nonce_hash ~payload_round simulation_mode simulation_kind - constants = +let filter_via_node ~chain_id ~fees_config ~hard_gas_limit_per_block + ~faked_protocol_data ~timestamp ~(pred_info : Baking_state.block_info) + ~payload_round ~operation_pool cctxt = let open Lwt_result_syntax in - let predecessor_block = (pred_info : Baking_state.block_info) in - let hard_gas_limit_per_block = - constants.Constants.Parametric.hard_gas_limit_per_block - in let chain = `Hash chain_id in - let filter_via_node ~operation_pool = - let filtered_operations = - Operation_selection.filter_operations_without_simulation - fees_config - ~hard_gas_limit_per_block - operation_pool + let filtered_operations = + Operation_selection.filter_operations_without_simulation + fees_config + ~hard_gas_limit_per_block + operation_pool + in + let* shell_header, preapply_result = + Node_rpc.preapply_block + cctxt + ~chain + ~head:pred_info.hash + ~timestamp + ~protocol_data:faked_protocol_data + filtered_operations + in + (* only retain valid operations *) + let operations = + List.map (fun l -> List.map snd l.Preapply_result.applied) preapply_result + in + let payload_hash = + let operation_hashes = + Stdlib.List.tl operations |> List.flatten + |> List.map Tezos_base.Operation.hash in - let faked_protocol_data = - forge_faked_protocol_data - ~payload_round - ~seed_nonce_hash - ~liquidity_baking_toggle_vote - () + Block_payload.hash + ~predecessor_hash:shell_header.predecessor + ~payload_round + operation_hashes + in + return (shell_header, operations, payload_hash) + +let filter_with_context ~chain_id ~fees_config ~hard_gas_limit_per_block + ~faked_protocol_data ~user_activated_upgrades ~timestamp + ~(pred_info : Baking_state.block_info) ~context_index ~payload_round + ~operation_pool cctxt = + let open Lwt_result_syntax in + let chain = `Hash chain_id in + let* incremental = + Baking_simulator.begin_construction + ~timestamp + ~protocol_data:faked_protocol_data + context_index + pred_info + chain_id + in + let* {Operation_selection.operations; validation_result; operations_hash; _} = + Operation_selection.filter_operations_with_simulation + incremental + fees_config + ~hard_gas_limit_per_block + operation_pool + in + let* changed = + check_protocol_changed + ~level:(Int32.succ pred_info.shell.level) + ~user_activated_upgrades + ~validation_result + in + if changed then + (* Fallback to processing via node, which knows both old and new protocol. *) + filter_via_node + ~chain_id + ~fees_config + ~hard_gas_limit_per_block + ~faked_protocol_data + ~timestamp + ~pred_info + ~payload_round + ~operation_pool + cctxt + else + let* pred_block_metadata_hash = + protect + ~on_error:(fun _ -> return_none) + (fun () -> + let+ pred_block_metadata_hash = + Shell_services.Blocks.metadata_hash + cctxt + ~block:(`Hash (pred_info.hash, 0)) + ~chain + () + in + Some pred_block_metadata_hash) in - let* shell_header, preapply_result = - Node_rpc.preapply_block - cctxt - ~chain - ~head:predecessor_block.hash - ~timestamp - ~protocol_data:faked_protocol_data - filtered_operations + let* pred_op_metadata_hash = + protect + ~on_error:(fun _ -> return_none) + (fun () -> + let+ pred_op_metadata_hash = + Shell_services.Blocks.Operation_metadata_hashes.root + cctxt + ~block:(`Hash (pred_info.hash, 0)) + ~chain + () + in + Some pred_op_metadata_hash) in - (* only retain valid operations *) - let operations = - List.map (fun l -> List.map snd l.Preapply_result.applied) preapply_result + let* shell_header = + finalize_block_header + incremental.header + timestamp + validation_result + operations_hash + pred_block_metadata_hash + pred_op_metadata_hash + pred_info.resulting_context_hash in + let operations = List.map (List.map convert_operation) operations in let payload_hash = let operation_hashes = Stdlib.List.tl operations |> List.flatten @@ -192,201 +268,124 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info operation_hashes in return (shell_header, operations, payload_hash) + +let apply_via_node ~chain_id ~faked_protocol_data ~timestamp + ~(pred_info : Baking_state.block_info) ~ordered_pool ~payload_hash cctxt = + let open Lwt_result_syntax in + let chain = `Hash chain_id in + let operations = Operation_pool.ordered_to_list_list ordered_pool in + let* shell_header, _preapply_result = + Node_rpc.preapply_block + cctxt + ~chain + ~head:pred_info.hash + ~timestamp + ~protocol_data:faked_protocol_data + operations in - let filter_with_context ~context_index ~operation_pool = - let faked_protocol_data = - forge_faked_protocol_data - ~payload_round - ~seed_nonce_hash - ~liquidity_baking_toggle_vote - () - in - let* incremental = - Baking_simulator.begin_construction - ~timestamp - ~protocol_data:faked_protocol_data - context_index - predecessor_block - chain_id - in - let* {Operation_selection.operations; validation_result; operations_hash; _} - = - Operation_selection.filter_operations_with_simulation - incremental - fees_config - ~hard_gas_limit_per_block - operation_pool - in - let* changed = - check_protocol_changed - ~level:(Int32.succ predecessor_block.shell.level) - ~user_activated_upgrades - ~validation_result - in - if changed then - (* Fallback to processing via node, which knows both old and new protocol. *) - filter_via_node ~operation_pool - else - let* pred_block_metadata_hash = - protect - ~on_error:(fun _ -> return_none) - (fun () -> - let+ pred_block_metadata_hash = - Shell_services.Blocks.metadata_hash - cctxt - ~block:(`Hash (predecessor_block.hash, 0)) - ~chain - () - in - Some pred_block_metadata_hash) - in - let* pred_op_metadata_hash = - protect - ~on_error:(fun _ -> return_none) - (fun () -> - let+ pred_op_metadata_hash = - Shell_services.Blocks.Operation_metadata_hashes.root - cctxt - ~block:(`Hash (predecessor_block.hash, 0)) - ~chain - () - in - Some pred_op_metadata_hash) - in - let* shell_header = - finalize_block_header - incremental.header - timestamp - validation_result - operations_hash - pred_block_metadata_hash - pred_op_metadata_hash - predecessor_block.resulting_context_hash - in - let operations = List.map (List.map convert_operation) operations in - let payload_hash = - let operation_hashes = - Stdlib.List.tl operations |> List.flatten - |> List.map Tezos_base.Operation.hash - in - Block_payload.hash - ~predecessor_hash:shell_header.predecessor - ~payload_round - operation_hashes - in - return (shell_header, operations, payload_hash) + let operations = List.map (List.map convert_operation) operations in + return (shell_header, operations, payload_hash) + +let apply_with_context ~chain_id ~faked_protocol_data ~user_activated_upgrades + ~timestamp ~(pred_info : Baking_state.block_info) ~ordered_pool + ~context_index ~payload_hash cctxt = + let open Lwt_result_syntax in + let chain = `Hash chain_id in + let* incremental = + Baking_simulator.begin_construction + ~timestamp + ~protocol_data:faked_protocol_data + context_index + pred_info + chain_id in - let apply_via_node ~ordered_pool ~payload_hash = - let operations = Operation_pool.ordered_to_list_list ordered_pool in - let faked_protocol_data = - forge_faked_protocol_data - ~seed_nonce_hash - ~liquidity_baking_toggle_vote - ~payload_hash - ~payload_round - () - in - let* shell_header, _preapply_result = - Node_rpc.preapply_block - cctxt - ~chain - ~head:predecessor_block.hash - ~timestamp - ~protocol_data:faked_protocol_data - operations - in - let operations = List.map (List.map convert_operation) operations in - return (shell_header, operations, payload_hash) + (* We still need to filter endorsements. Two endorsements could be + referring to the same slot. *) + let* incremental, ordered_pool = + Operation_selection.filter_consensus_operations_only + incremental + ordered_pool in - let apply_with_context ~context_index ~ordered_pool ~payload_hash = - let faked_protocol_data = - forge_faked_protocol_data - ~seed_nonce_hash - ~liquidity_baking_toggle_vote - ~payload_hash - ~payload_round - () - in - let* chain_id = Shell_services.Chain.chain_id cctxt ~chain () in - let* incremental = - Baking_simulator.begin_construction - ~timestamp - ~protocol_data:faked_protocol_data - context_index - predecessor_block - chain_id - in - (* We still need to filter endorsements. Two endorsements could be - referring to the same slot. *) - let* incremental, ordered_pool = - Operation_selection.filter_consensus_operations_only - incremental - ordered_pool - in - let operations = Operation_pool.ordered_to_list_list ordered_pool in - let operations_hash = - Operation_list_list_hash.compute - (List.map - (fun sl -> - Operation_list_hash.compute (List.map Operation.hash_packed sl)) - operations) - in - (* We need to compute the final [operations_hash] before - finalizing the block because it will be used in the cache's nonce. *) - let incremental = - {incremental with header = {incremental.header with operations_hash}} + let operations = Operation_pool.ordered_to_list_list ordered_pool in + let operations_hash = + Operation_list_list_hash.compute + (List.map + (fun sl -> + Operation_list_hash.compute (List.map Operation.hash_packed sl)) + operations) + in + (* We need to compute the final [operations_hash] before + finalizing the block because it will be used in the cache's nonce. *) + let incremental = + {incremental with header = {incremental.header with operations_hash}} + in + let* validation_result, _ = + Baking_simulator.finalize_construction incremental + in + let* changed = + check_protocol_changed + ~level:(Int32.succ pred_info.shell.level) + ~user_activated_upgrades + ~validation_result + in + if changed then + (* Fallback to processing via node, which knows both old and new protocol. *) + apply_via_node + ~chain_id + ~faked_protocol_data + ~timestamp + ~pred_info + ~ordered_pool + ~payload_hash + cctxt + else + let* pred_block_metadata_hash = + protect + ~on_error:(fun _ -> return_none) + (fun () -> + let+ pred_block_metadata_hash = + Shell_services.Blocks.metadata_hash + cctxt + ~block:(`Hash (pred_info.hash, 0)) + ~chain + () + in + Some pred_block_metadata_hash) in - let* validation_result, _ = - Baking_simulator.finalize_construction incremental + let* pred_op_metadata_hash = + protect + ~on_error:(fun _ -> return_none) + (fun () -> + let+ pred_op_metadata_hash = + Shell_services.Blocks.Operation_metadata_hashes.root + cctxt + ~block:(`Hash (pred_info.hash, 0)) + ~chain + () + in + Some pred_op_metadata_hash) in - let* changed = - check_protocol_changed - ~level:(Int32.succ predecessor_block.shell.level) - ~user_activated_upgrades - ~validation_result + let* shell_header = + finalize_block_header + incremental.header + timestamp + validation_result + operations_hash + pred_block_metadata_hash + pred_op_metadata_hash + pred_info.resulting_context_hash in - if changed then - (* Fallback to processing via node, which knows both old and new protocol. *) - apply_via_node ~ordered_pool ~payload_hash - else - let* pred_block_metadata_hash = - protect - ~on_error:(fun _ -> return_none) - (fun () -> - let+ pred_block_metadata_hash = - Shell_services.Blocks.metadata_hash - cctxt - ~block:(`Hash (predecessor_block.hash, 0)) - ~chain - () - in - Some pred_block_metadata_hash) - in - let* pred_op_metadata_hash = - protect - ~on_error:(fun _ -> return_none) - (fun () -> - let+ pred_op_metadata_hash = - Shell_services.Blocks.Operation_metadata_hashes.root - cctxt - ~block:(`Hash (predecessor_block.hash, 0)) - ~chain - () - in - Some pred_op_metadata_hash) - in - let* shell_header = - finalize_block_header - incremental.header - timestamp - validation_result - operations_hash - pred_block_metadata_hash - pred_op_metadata_hash - predecessor_block.resulting_context_hash - in - let operations = List.map (List.map convert_operation) operations in - return (shell_header, operations, payload_hash) + let operations = List.map (List.map convert_operation) operations in + return (shell_header, operations, payload_hash) + +(* [forge] a new [unsigned_block] regarding of [simulation_kind] and [simulation_mode] *) +let forge (cctxt : #Protocol_client_context.full) ~chain_id + ~(pred_info : Baking_state.block_info) ~timestamp + ~liquidity_baking_toggle_vote ~user_activated_upgrades fees_config + ~seed_nonce_hash ~payload_round simulation_mode simulation_kind constants = + let open Lwt_result_syntax in + let hard_gas_limit_per_block = + constants.Constants.Parametric.hard_gas_limit_per_block in let simulation_kind = match simulation_kind with @@ -404,13 +403,79 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~pred_info let* shell_header, operations, payload_hash = match (simulation_mode, simulation_kind) with | Baking_state.Node, Filter operation_pool -> - filter_via_node ~operation_pool + let faked_protocol_data = + forge_faked_protocol_data + ~payload_round + ~seed_nonce_hash + ~liquidity_baking_toggle_vote + () + in + filter_via_node + ~chain_id + ~faked_protocol_data + ~fees_config + ~hard_gas_limit_per_block + ~timestamp + ~pred_info + ~payload_round + ~operation_pool + cctxt | Node, Apply {ordered_pool; payload_hash} -> - apply_via_node ~ordered_pool ~payload_hash + let faked_protocol_data = + forge_faked_protocol_data + ~payload_hash + ~payload_round + ~seed_nonce_hash + ~liquidity_baking_toggle_vote + () + in + apply_via_node + ~chain_id + ~faked_protocol_data + ~timestamp + ~pred_info + ~ordered_pool + ~payload_hash + cctxt | Local context_index, Filter operation_pool -> - filter_with_context ~context_index ~operation_pool + let faked_protocol_data = + forge_faked_protocol_data + ~payload_round + ~seed_nonce_hash + ~liquidity_baking_toggle_vote + () + in + filter_with_context + ~chain_id + ~faked_protocol_data + ~fees_config + ~hard_gas_limit_per_block + ~user_activated_upgrades + ~timestamp + ~pred_info + ~context_index + ~payload_round + ~operation_pool + cctxt | Local context_index, Apply {ordered_pool; payload_hash} -> - apply_with_context ~context_index ~ordered_pool ~payload_hash + let faked_protocol_data = + forge_faked_protocol_data + ~payload_hash + ~payload_round + ~seed_nonce_hash + ~liquidity_baking_toggle_vote + () + in + apply_with_context + ~chain_id + ~faked_protocol_data + ~user_activated_upgrades + ~timestamp + ~pred_info + ~ordered_pool + ~context_index + ~payload_hash + cctxt in let* contents = Baking_pow.mine -- GitLab From aa0c68847935c4920217c07b7039a65b7b9e7516 Mon Sep 17 00:00:00 2001 From: Albin Coquereau Date: Tue, 17 Jan 2023 18:06:40 +0100 Subject: [PATCH 10/46] lib_delegate: add force_apply argument to the baker configuration this argument is use to choose the way operation are handle by the baker. If force_apply is set, the operations are applied after being validated. --- .../lib_delegate/baking_commands.ml | 21 +++- .../lib_delegate/baking_configuration.ml | 96 ++++++++++--------- .../lib_delegate/baking_configuration.mli | 4 + .../lib_delegate/baking_lib.ml | 6 +- .../lib_delegate/baking_lib.mli | 2 + .../lib_delegate/client_daemon.ml | 3 +- .../lib_delegate/client_daemon.mli | 1 + .../lib_delegate/baking_commands.ml | 21 +++- .../lib_delegate/baking_configuration.ml | 82 +++++++++------- .../lib_delegate/baking_configuration.mli | 4 + src/proto_alpha/lib_delegate/baking_lib.ml | 6 +- src/proto_alpha/lib_delegate/baking_lib.mli | 2 + src/proto_alpha/lib_delegate/client_daemon.ml | 3 +- .../lib_delegate/client_daemon.mli | 1 + 14 files changed, 161 insertions(+), 91 deletions(-) diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_commands.ml b/src/proto_016_PtMumbai/lib_delegate/baking_commands.ml index 9d76b8745fff..1e6dbe5a6152 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_commands.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_commands.ml @@ -114,6 +114,12 @@ let context_path_arg = 'preapply' RPC." string_parameter +let force_apply_switch_arg = + Tezos_clic.switch + ~long:"force-apply" + ~doc:"Force the baker to not only validate but also apply operations." + () + let endorsement_force_switch_arg = Tezos_clic.switch ~long:"force" @@ -218,11 +224,12 @@ let delegate_commands () : Protocol_client_context.full Tezos_clic.command list command ~group ~desc:"Forge and inject block using the delegates' rights." - (args8 + (args9 minimal_fees_arg minimal_nanotez_per_gas_unit_arg minimal_nanotez_per_byte_arg minimal_timestamp_switch + force_apply_switch_arg force_switch operations_arg context_path_arg @@ -232,6 +239,7 @@ let delegate_commands () : Protocol_client_context.full Tezos_clic.command list minimal_nanotez_per_gas_unit, minimal_nanotez_per_byte, minimal_timestamp, + force_apply, force, extra_operations, context_path, @@ -245,6 +253,7 @@ let delegate_commands () : Protocol_client_context.full Tezos_clic.command list ~minimal_timestamp ~minimal_nanotez_per_byte ~minimal_fees + ~force_apply ~force ~monitor_node_mempool:(not do_not_monitor_node_mempool) ?extra_operations @@ -269,11 +278,12 @@ let delegate_commands () : Protocol_client_context.full Tezos_clic.command list command ~group ~desc:"Send a Tenderbake proposal" - (args7 + (args8 minimal_fees_arg minimal_nanotez_per_gas_unit_arg minimal_nanotez_per_byte_arg minimal_timestamp_switch + force_apply_switch_arg force_switch operations_arg context_path_arg) @@ -282,6 +292,7 @@ let delegate_commands () : Protocol_client_context.full Tezos_clic.command list minimal_nanotez_per_gas_unit, minimal_nanotez_per_byte, minimal_timestamp, + force_apply, force, extra_operations, context_path ) @@ -294,6 +305,7 @@ let delegate_commands () : Protocol_client_context.full Tezos_clic.command list ~minimal_timestamp ~minimal_nanotez_per_byte ~minimal_fees + ~force_apply ~force ?extra_operations ?context_path @@ -326,11 +338,12 @@ let baker_commands () : Protocol_client_context.full Tezos_clic.command list = command ~group ~desc:"Launch the baker daemon." - (args8 + (args9 pidfile_arg minimal_fees_arg minimal_nanotez_per_gas_unit_arg minimal_nanotez_per_byte_arg + force_apply_switch_arg keep_alive_arg liquidity_baking_toggle_vote_arg per_block_vote_file_arg @@ -345,6 +358,7 @@ let baker_commands () : Protocol_client_context.full Tezos_clic.command list = minimal_fees, minimal_nanotez_per_gas_unit, minimal_nanotez_per_byte, + force_apply, keep_alive, liquidity_baking_toggle_vote, per_block_vote_file, @@ -378,6 +392,7 @@ let baker_commands () : Protocol_client_context.full Tezos_clic.command list = ~liquidity_baking_toggle_vote ?per_block_vote_file ?extra_operations + ~force_apply ~chain:cctxt#chain ~context_path ~keep_alive diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_configuration.ml b/src/proto_016_PtMumbai/lib_delegate/baking_configuration.ml index 2b8dca57b0f6..6db8eccb9f3d 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_configuration.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_configuration.ml @@ -85,6 +85,7 @@ type t = { liquidity_baking_toggle_vote : Protocol.Alpha_context.Liquidity_baking.liquidity_baking_toggle_vote; per_block_vote_file : string option; + force_apply : bool; force : bool; state_recorder : state_recorder_config; extra_operations : Operations_source.t option; @@ -113,6 +114,8 @@ let default_liquidity_baking_toggle_vote = let default_force = false +let default_force_apply = false + let default_state_recorder_config = Filesystem let default_extra_operations = None @@ -127,6 +130,7 @@ let default_config = retries_on_failure = default_retries_on_failure_config; user_activated_upgrades = default_user_activated_upgrades; liquidity_baking_toggle_vote = default_liquidity_baking_toggle_vote; + force_apply = default_force_apply; force = default_force; state_recorder = default_state_recorder_config; extra_operations = default_extra_operations; @@ -141,8 +145,9 @@ let make ?(minimal_fees = default_fees_config.minimal_fees) ?(retries_on_failure = default_retries_on_failure_config) ?(user_activated_upgrades = default_user_activated_upgrades) ?(liquidity_baking_toggle_vote = default_liquidity_baking_toggle_vote) - ?per_block_vote_file ?(force = default_force) - ?(state_recorder = default_state_recorder_config) ?extra_operations () = + ?per_block_vote_file ?(force_apply = default_force_apply) + ?(force = default_force) ?(state_recorder = default_state_recorder_config) + ?extra_operations () = let fees = {minimal_fees; minimal_nanotez_per_gas_unit; minimal_nanotez_per_byte} in @@ -159,6 +164,7 @@ let make ?(minimal_fees = default_fees_config.minimal_fees) user_activated_upgrades; liquidity_baking_toggle_vote; per_block_vote_file; + force_apply; force; state_recorder; extra_operations; @@ -228,6 +234,8 @@ let liquidity_baking_toggle_vote_config_encoding = let force_config_encoding = Data_encoding.bool +let force_apply_config_encoding = Data_encoding.bool + let state_recorder_config_encoding = let open Data_encoding in union @@ -254,37 +262,28 @@ let encoding : t Data_encoding.t = ~title:"Baking configuration" ~description:"Baking configuration" @@ conv - (fun { - fees; - validation; - nonce; - retries_on_failure; - user_activated_upgrades; - liquidity_baking_toggle_vote; - per_block_vote_file; - force; - state_recorder; - extra_operations; - } -> - ( fees, - validation, - nonce, - retries_on_failure, - user_activated_upgrades, - liquidity_baking_toggle_vote, - per_block_vote_file, - force, - state_recorder, - extra_operations )) - (fun ( fees, - validation, - nonce, - retries_on_failure, - user_activated_upgrades, - liquidity_baking_toggle_vote, - per_block_vote_file, - force, - state_recorder, + (fun c -> + ( ( c.fees, + c.validation, + c.nonce, + c.retries_on_failure, + c.user_activated_upgrades, + c.liquidity_baking_toggle_vote, + c.per_block_vote_file, + c.force_apply, + c.force, + c.state_recorder ), + c.extra_operations )) + (fun ( ( fees, + validation, + nonce, + retries_on_failure, + user_activated_upgrades, + liquidity_baking_toggle_vote, + per_block_vote_file, + force_apply, + force, + state_recorder ), extra_operations ) -> { fees; @@ -294,23 +293,28 @@ let encoding : t Data_encoding.t = user_activated_upgrades; liquidity_baking_toggle_vote; per_block_vote_file; + force_apply; force; state_recorder; extra_operations; }) - (obj10 - (req "fees" fees_config_encoding) - (req "validation" validation_config_encoding) - (req "nonce" nonce_config_encoding) - (req "retries_on_failure" retries_on_failure_config_encoding) - (req "user_activated_upgrades" user_activate_upgrades_config_encoding) - (req - "liquidity_baking_toggle_vote" - liquidity_baking_toggle_vote_config_encoding) - (opt "per_block_vote_file" Data_encoding.string) - (req "force" force_config_encoding) - (req "state_recorder" state_recorder_config_encoding) - (opt "extra_operations" Operations_source.encoding)) + (merge_objs + (obj10 + (req "fees" fees_config_encoding) + (req "validation" validation_config_encoding) + (req "nonce" nonce_config_encoding) + (req "retries_on_failure" retries_on_failure_config_encoding) + (req + "user_activated_upgrades" + user_activate_upgrades_config_encoding) + (req + "liquidity_baking_toggle_vote" + liquidity_baking_toggle_vote_config_encoding) + (opt "per_block_vote_file" Data_encoding.string) + (req "force_apply" force_apply_config_encoding) + (req "force" force_config_encoding) + (req "state_recorder" state_recorder_config_encoding)) + (obj1 (opt "extra_operations" Operations_source.encoding))) let pp fmt t = let json = Data_encoding.Json.construct encoding t in diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_configuration.mli b/src/proto_016_PtMumbai/lib_delegate/baking_configuration.mli index a1908d29dd22..2cd4946e14a8 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_configuration.mli +++ b/src/proto_016_PtMumbai/lib_delegate/baking_configuration.mli @@ -60,6 +60,7 @@ type t = { liquidity_baking_toggle_vote : Protocol.Alpha_context.Liquidity_baking.liquidity_baking_toggle_vote; per_block_vote_file : string option; + force_apply : bool; force : bool; state_recorder : state_recorder_config; extra_operations : Operations_source.t option; @@ -78,6 +79,8 @@ val default_user_activated_upgrades : (int32 * Protocol_hash.t) list val default_liquidity_baking_toggle_vote : Protocol.Alpha_context.Liquidity_baking.liquidity_baking_toggle_vote +val default_force_apply : bool + val default_force : bool val default_state_recorder_config : state_recorder_config @@ -99,6 +102,7 @@ val make : ?liquidity_baking_toggle_vote: Protocol.Alpha_context.Liquidity_baking.liquidity_baking_toggle_vote -> ?per_block_vote_file:string -> + ?force_apply:bool -> ?force:bool -> ?state_recorder:state_recorder_config -> ?extra_operations:Operations_source.t -> diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_lib.ml b/src/proto_016_PtMumbai/lib_delegate/baking_lib.ml index 7ec3ae000cd5..24ba15e8db03 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_lib.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_lib.ml @@ -296,7 +296,7 @@ let endorsement_quorum state = - Yes :: repropose block with right payload and preendorsements for current round - No :: repropose fresh block for current round *) let propose (cctxt : Protocol_client_context.full) ?minimal_fees - ?minimal_nanotez_per_gas_unit ?minimal_nanotez_per_byte ?force + ?minimal_nanotez_per_gas_unit ?minimal_nanotez_per_byte ?force_apply ?force ?(minimal_timestamp = false) ?extra_operations ?context_path delegates = let open Lwt_result_syntax in let* _block_stream, current_proposal = get_current_proposal cctxt in @@ -306,6 +306,7 @@ let propose (cctxt : Protocol_client_context.full) ?minimal_fees ?minimal_nanotez_per_gas_unit ?minimal_nanotez_per_byte ?context_path + ?force_apply ?force ?extra_operations () @@ -507,7 +508,7 @@ let baking_minimal_timestamp state = return_unit let bake (cctxt : Protocol_client_context.full) ?minimal_fees - ?minimal_nanotez_per_gas_unit ?minimal_nanotez_per_byte ?force + ?minimal_nanotez_per_gas_unit ?minimal_nanotez_per_byte ?force_apply ?force ?(minimal_timestamp = false) ?extra_operations ?(monitor_node_mempool = true) ?context_path delegates = let open Lwt_result_syntax in @@ -517,6 +518,7 @@ let bake (cctxt : Protocol_client_context.full) ?minimal_fees ?minimal_nanotez_per_gas_unit ?minimal_nanotez_per_byte ?context_path + ?force_apply ?force ?extra_operations () diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_lib.mli b/src/proto_016_PtMumbai/lib_delegate/baking_lib.mli index e37f01ac7bb5..b1de5798925a 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_lib.mli +++ b/src/proto_016_PtMumbai/lib_delegate/baking_lib.mli @@ -32,6 +32,7 @@ val bake : ?minimal_fees:Tez.t -> ?minimal_nanotez_per_gas_unit:Q.t -> ?minimal_nanotez_per_byte:Q.t -> + ?force_apply:bool -> ?force:bool -> ?minimal_timestamp:bool -> ?extra_operations:Baking_configuration.Operations_source.t -> @@ -57,6 +58,7 @@ val propose : ?minimal_fees:Tez.t -> ?minimal_nanotez_per_gas_unit:Q.t -> ?minimal_nanotez_per_byte:Q.t -> + ?force_apply:bool -> ?force:bool -> ?minimal_timestamp:bool -> ?extra_operations:Baking_configuration.Operations_source.t -> diff --git a/src/proto_016_PtMumbai/lib_delegate/client_daemon.ml b/src/proto_016_PtMumbai/lib_delegate/client_daemon.ml index ba2cfdac03b3..617935658380 100644 --- a/src/proto_016_PtMumbai/lib_delegate/client_daemon.ml +++ b/src/proto_016_PtMumbai/lib_delegate/client_daemon.ml @@ -70,7 +70,7 @@ module Baker = struct let run (cctxt : Protocol_client_context.full) ?minimal_fees ?minimal_nanotez_per_gas_unit ?minimal_nanotez_per_byte ?liquidity_baking_toggle_vote ?per_block_vote_file ?extra_operations - ~chain ~context_path ~keep_alive delegates = + ?force_apply ~chain ~context_path ~keep_alive delegates = let process () = Config_services.user_activated_upgrades cctxt >>=? fun user_activated_upgrades -> @@ -82,6 +82,7 @@ module Baker = struct ?liquidity_baking_toggle_vote ?per_block_vote_file ?extra_operations + ?force_apply ~context_path ~user_activated_upgrades () diff --git a/src/proto_016_PtMumbai/lib_delegate/client_daemon.mli b/src/proto_016_PtMumbai/lib_delegate/client_daemon.mli index 751db5058857..65119e84040c 100644 --- a/src/proto_016_PtMumbai/lib_delegate/client_daemon.mli +++ b/src/proto_016_PtMumbai/lib_delegate/client_daemon.mli @@ -36,6 +36,7 @@ module Baker : sig Protocol.Alpha_context.Liquidity_baking.liquidity_baking_toggle_vote -> ?per_block_vote_file:string -> ?extra_operations:Baking_configuration.Operations_source.t -> + ?force_apply:bool -> chain:Shell_services.chain -> context_path:string -> keep_alive:bool -> diff --git a/src/proto_alpha/lib_delegate/baking_commands.ml b/src/proto_alpha/lib_delegate/baking_commands.ml index b827d5b7cadd..33dd83a2bdcc 100644 --- a/src/proto_alpha/lib_delegate/baking_commands.ml +++ b/src/proto_alpha/lib_delegate/baking_commands.ml @@ -114,6 +114,12 @@ let context_path_arg = 'preapply' RPC." string_parameter +let force_apply_switch_arg = + Tezos_clic.switch + ~long:"force-apply" + ~doc:"Force the baker to not only validate but also apply operations." + () + let endorsement_force_switch_arg = Tezos_clic.switch ~long:"force" @@ -218,11 +224,12 @@ let delegate_commands () : Protocol_client_context.full Tezos_clic.command list command ~group ~desc:"Forge and inject block using the delegates' rights." - (args8 + (args9 minimal_fees_arg minimal_nanotez_per_gas_unit_arg minimal_nanotez_per_byte_arg minimal_timestamp_switch + force_apply_switch_arg force_switch operations_arg context_path_arg @@ -232,6 +239,7 @@ let delegate_commands () : Protocol_client_context.full Tezos_clic.command list minimal_nanotez_per_gas_unit, minimal_nanotez_per_byte, minimal_timestamp, + force_apply, force, extra_operations, context_path, @@ -245,6 +253,7 @@ let delegate_commands () : Protocol_client_context.full Tezos_clic.command list ~minimal_timestamp ~minimal_nanotez_per_byte ~minimal_fees + ~force_apply ~force ~monitor_node_mempool:(not do_not_monitor_node_mempool) ?extra_operations @@ -269,11 +278,12 @@ let delegate_commands () : Protocol_client_context.full Tezos_clic.command list command ~group ~desc:"Send a Tenderbake proposal" - (args7 + (args8 minimal_fees_arg minimal_nanotez_per_gas_unit_arg minimal_nanotez_per_byte_arg minimal_timestamp_switch + force_apply_switch_arg force_switch operations_arg context_path_arg) @@ -282,6 +292,7 @@ let delegate_commands () : Protocol_client_context.full Tezos_clic.command list minimal_nanotez_per_gas_unit, minimal_nanotez_per_byte, minimal_timestamp, + force_apply, force, extra_operations, context_path ) @@ -294,6 +305,7 @@ let delegate_commands () : Protocol_client_context.full Tezos_clic.command list ~minimal_timestamp ~minimal_nanotez_per_byte ~minimal_fees + ~force_apply ~force ?extra_operations ?context_path @@ -326,11 +338,12 @@ let baker_commands () : Protocol_client_context.full Tezos_clic.command list = command ~group ~desc:"Launch the baker daemon." - (args8 + (args9 pidfile_arg minimal_fees_arg minimal_nanotez_per_gas_unit_arg minimal_nanotez_per_byte_arg + force_apply_switch_arg keep_alive_arg liquidity_baking_toggle_vote_arg per_block_vote_file_arg @@ -345,6 +358,7 @@ let baker_commands () : Protocol_client_context.full Tezos_clic.command list = minimal_fees, minimal_nanotez_per_gas_unit, minimal_nanotez_per_byte, + force_apply, keep_alive, liquidity_baking_toggle_vote, per_block_vote_file, @@ -378,6 +392,7 @@ let baker_commands () : Protocol_client_context.full Tezos_clic.command list = ~liquidity_baking_toggle_vote ?per_block_vote_file ?extra_operations + ~force_apply ~chain:cctxt#chain ~context_path ~keep_alive diff --git a/src/proto_alpha/lib_delegate/baking_configuration.ml b/src/proto_alpha/lib_delegate/baking_configuration.ml index 2b8dca57b0f6..d8a6686d119b 100644 --- a/src/proto_alpha/lib_delegate/baking_configuration.ml +++ b/src/proto_alpha/lib_delegate/baking_configuration.ml @@ -85,6 +85,7 @@ type t = { liquidity_baking_toggle_vote : Protocol.Alpha_context.Liquidity_baking.liquidity_baking_toggle_vote; per_block_vote_file : string option; + force_apply : bool; force : bool; state_recorder : state_recorder_config; extra_operations : Operations_source.t option; @@ -113,6 +114,8 @@ let default_liquidity_baking_toggle_vote = let default_force = false +let default_force_apply = false + let default_state_recorder_config = Filesystem let default_extra_operations = None @@ -127,6 +130,7 @@ let default_config = retries_on_failure = default_retries_on_failure_config; user_activated_upgrades = default_user_activated_upgrades; liquidity_baking_toggle_vote = default_liquidity_baking_toggle_vote; + force_apply = default_force_apply; force = default_force; state_recorder = default_state_recorder_config; extra_operations = default_extra_operations; @@ -141,8 +145,9 @@ let make ?(minimal_fees = default_fees_config.minimal_fees) ?(retries_on_failure = default_retries_on_failure_config) ?(user_activated_upgrades = default_user_activated_upgrades) ?(liquidity_baking_toggle_vote = default_liquidity_baking_toggle_vote) - ?per_block_vote_file ?(force = default_force) - ?(state_recorder = default_state_recorder_config) ?extra_operations () = + ?per_block_vote_file ?(force_apply = default_force_apply) + ?(force = default_force) ?(state_recorder = default_state_recorder_config) + ?extra_operations () = let fees = {minimal_fees; minimal_nanotez_per_gas_unit; minimal_nanotez_per_byte} in @@ -159,6 +164,7 @@ let make ?(minimal_fees = default_fees_config.minimal_fees) user_activated_upgrades; liquidity_baking_toggle_vote; per_block_vote_file; + force_apply; force; state_recorder; extra_operations; @@ -228,6 +234,8 @@ let liquidity_baking_toggle_vote_config_encoding = let force_config_encoding = Data_encoding.bool +let force_apply_config_encoding = Data_encoding.bool + let state_recorder_config_encoding = let open Data_encoding in union @@ -262,29 +270,32 @@ let encoding : t Data_encoding.t = user_activated_upgrades; liquidity_baking_toggle_vote; per_block_vote_file; + force_apply; force; state_recorder; extra_operations; } -> - ( fees, - validation, - nonce, - retries_on_failure, - user_activated_upgrades, - liquidity_baking_toggle_vote, - per_block_vote_file, - force, - state_recorder, + ( ( fees, + validation, + nonce, + retries_on_failure, + user_activated_upgrades, + liquidity_baking_toggle_vote, + per_block_vote_file, + force_apply, + force, + state_recorder ), extra_operations )) - (fun ( fees, - validation, - nonce, - retries_on_failure, - user_activated_upgrades, - liquidity_baking_toggle_vote, - per_block_vote_file, - force, - state_recorder, + (fun ( ( fees, + validation, + nonce, + retries_on_failure, + user_activated_upgrades, + liquidity_baking_toggle_vote, + per_block_vote_file, + force_apply, + force, + state_recorder ), extra_operations ) -> { fees; @@ -294,23 +305,28 @@ let encoding : t Data_encoding.t = user_activated_upgrades; liquidity_baking_toggle_vote; per_block_vote_file; + force_apply; force; state_recorder; extra_operations; }) - (obj10 - (req "fees" fees_config_encoding) - (req "validation" validation_config_encoding) - (req "nonce" nonce_config_encoding) - (req "retries_on_failure" retries_on_failure_config_encoding) - (req "user_activated_upgrades" user_activate_upgrades_config_encoding) - (req - "liquidity_baking_toggle_vote" - liquidity_baking_toggle_vote_config_encoding) - (opt "per_block_vote_file" Data_encoding.string) - (req "force" force_config_encoding) - (req "state_recorder" state_recorder_config_encoding) - (opt "extra_operations" Operations_source.encoding)) + (merge_objs + (obj10 + (req "fees" fees_config_encoding) + (req "validation" validation_config_encoding) + (req "nonce" nonce_config_encoding) + (req "retries_on_failure" retries_on_failure_config_encoding) + (req + "user_activated_upgrades" + user_activate_upgrades_config_encoding) + (req + "liquidity_baking_toggle_vote" + liquidity_baking_toggle_vote_config_encoding) + (opt "per_block_vote_file" Data_encoding.string) + (req "force_apply" force_apply_config_encoding) + (req "force" force_config_encoding) + (req "state_recorder" state_recorder_config_encoding)) + (obj1 (opt "extra_operations" Operations_source.encoding))) let pp fmt t = let json = Data_encoding.Json.construct encoding t in diff --git a/src/proto_alpha/lib_delegate/baking_configuration.mli b/src/proto_alpha/lib_delegate/baking_configuration.mli index a1908d29dd22..2cd4946e14a8 100644 --- a/src/proto_alpha/lib_delegate/baking_configuration.mli +++ b/src/proto_alpha/lib_delegate/baking_configuration.mli @@ -60,6 +60,7 @@ type t = { liquidity_baking_toggle_vote : Protocol.Alpha_context.Liquidity_baking.liquidity_baking_toggle_vote; per_block_vote_file : string option; + force_apply : bool; force : bool; state_recorder : state_recorder_config; extra_operations : Operations_source.t option; @@ -78,6 +79,8 @@ val default_user_activated_upgrades : (int32 * Protocol_hash.t) list val default_liquidity_baking_toggle_vote : Protocol.Alpha_context.Liquidity_baking.liquidity_baking_toggle_vote +val default_force_apply : bool + val default_force : bool val default_state_recorder_config : state_recorder_config @@ -99,6 +102,7 @@ val make : ?liquidity_baking_toggle_vote: Protocol.Alpha_context.Liquidity_baking.liquidity_baking_toggle_vote -> ?per_block_vote_file:string -> + ?force_apply:bool -> ?force:bool -> ?state_recorder:state_recorder_config -> ?extra_operations:Operations_source.t -> diff --git a/src/proto_alpha/lib_delegate/baking_lib.ml b/src/proto_alpha/lib_delegate/baking_lib.ml index bb2cb7b3f4ec..5adbda789e44 100644 --- a/src/proto_alpha/lib_delegate/baking_lib.ml +++ b/src/proto_alpha/lib_delegate/baking_lib.ml @@ -279,7 +279,7 @@ let endorsement_quorum state = - Yes :: repropose block with right payload and preendorsements for current round - No :: repropose fresh block for current round *) let propose (cctxt : Protocol_client_context.full) ?minimal_fees - ?minimal_nanotez_per_gas_unit ?minimal_nanotez_per_byte ?force + ?minimal_nanotez_per_gas_unit ?minimal_nanotez_per_byte ?force_apply ?force ?(minimal_timestamp = false) ?extra_operations ?context_path delegates = let open Lwt_result_syntax in let* _block_stream, current_proposal = get_current_proposal cctxt in @@ -289,6 +289,7 @@ let propose (cctxt : Protocol_client_context.full) ?minimal_fees ?minimal_nanotez_per_gas_unit ?minimal_nanotez_per_byte ?context_path + ?force_apply ?force ?extra_operations () @@ -490,7 +491,7 @@ let baking_minimal_timestamp state = return_unit let bake (cctxt : Protocol_client_context.full) ?minimal_fees - ?minimal_nanotez_per_gas_unit ?minimal_nanotez_per_byte ?force + ?minimal_nanotez_per_gas_unit ?minimal_nanotez_per_byte ?force_apply ?force ?(minimal_timestamp = false) ?extra_operations ?(monitor_node_mempool = true) ?context_path delegates = let open Lwt_result_syntax in @@ -500,6 +501,7 @@ let bake (cctxt : Protocol_client_context.full) ?minimal_fees ?minimal_nanotez_per_gas_unit ?minimal_nanotez_per_byte ?context_path + ?force_apply ?force ?extra_operations () diff --git a/src/proto_alpha/lib_delegate/baking_lib.mli b/src/proto_alpha/lib_delegate/baking_lib.mli index e37f01ac7bb5..b1de5798925a 100644 --- a/src/proto_alpha/lib_delegate/baking_lib.mli +++ b/src/proto_alpha/lib_delegate/baking_lib.mli @@ -32,6 +32,7 @@ val bake : ?minimal_fees:Tez.t -> ?minimal_nanotez_per_gas_unit:Q.t -> ?minimal_nanotez_per_byte:Q.t -> + ?force_apply:bool -> ?force:bool -> ?minimal_timestamp:bool -> ?extra_operations:Baking_configuration.Operations_source.t -> @@ -57,6 +58,7 @@ val propose : ?minimal_fees:Tez.t -> ?minimal_nanotez_per_gas_unit:Q.t -> ?minimal_nanotez_per_byte:Q.t -> + ?force_apply:bool -> ?force:bool -> ?minimal_timestamp:bool -> ?extra_operations:Baking_configuration.Operations_source.t -> diff --git a/src/proto_alpha/lib_delegate/client_daemon.ml b/src/proto_alpha/lib_delegate/client_daemon.ml index ba2cfdac03b3..617935658380 100644 --- a/src/proto_alpha/lib_delegate/client_daemon.ml +++ b/src/proto_alpha/lib_delegate/client_daemon.ml @@ -70,7 +70,7 @@ module Baker = struct let run (cctxt : Protocol_client_context.full) ?minimal_fees ?minimal_nanotez_per_gas_unit ?minimal_nanotez_per_byte ?liquidity_baking_toggle_vote ?per_block_vote_file ?extra_operations - ~chain ~context_path ~keep_alive delegates = + ?force_apply ~chain ~context_path ~keep_alive delegates = let process () = Config_services.user_activated_upgrades cctxt >>=? fun user_activated_upgrades -> @@ -82,6 +82,7 @@ module Baker = struct ?liquidity_baking_toggle_vote ?per_block_vote_file ?extra_operations + ?force_apply ~context_path ~user_activated_upgrades () diff --git a/src/proto_alpha/lib_delegate/client_daemon.mli b/src/proto_alpha/lib_delegate/client_daemon.mli index 751db5058857..65119e84040c 100644 --- a/src/proto_alpha/lib_delegate/client_daemon.mli +++ b/src/proto_alpha/lib_delegate/client_daemon.mli @@ -36,6 +36,7 @@ module Baker : sig Protocol.Alpha_context.Liquidity_baking.liquidity_baking_toggle_vote -> ?per_block_vote_file:string -> ?extra_operations:Baking_configuration.Operations_source.t -> + ?force_apply:bool -> chain:Shell_services.chain -> context_path:string -> keep_alive:bool -> -- GitLab From 99459c1c950f33f02e4ee95e86ec4dd24418a723 Mon Sep 17 00:00:00 2001 From: Albin Coquereau Date: Tue, 24 Jan 2023 15:19:37 +0100 Subject: [PATCH 11/46] lib_delegate: do not apply operation in the baker if force_apply is not set --- .../lib_delegate/baking_actions.ml | 2 + .../lib_delegate/baking_simulator.ml | 32 +- .../lib_delegate/baking_simulator.mli | 7 +- .../lib_delegate/block_forge.ml | 308 +++++++++++------- .../lib_delegate/block_forge.mli | 10 +- .../lib_delegate/operation_selection.ml | 29 +- .../lib_delegate/operation_selection.mli | 4 +- .../lib_delegate/baking_actions.ml | 2 + .../lib_delegate/baking_simulator.ml | 32 +- .../lib_delegate/baking_simulator.mli | 7 +- src/proto_alpha/lib_delegate/block_forge.ml | 307 ++++++++++------- src/proto_alpha/lib_delegate/block_forge.mli | 10 +- .../lib_delegate/operation_selection.ml | 29 +- .../lib_delegate/operation_selection.mli | 4 +- 14 files changed, 473 insertions(+), 310 deletions(-) diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_actions.ml b/src/proto_016_PtMumbai/lib_delegate/baking_actions.ml index 050d62846e4c..89cd0af885cb 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_actions.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_actions.ml @@ -301,10 +301,12 @@ let inject_block ~state_recorder state block_to_bake ~updated_state = ~chain_id ~pred_info:predecessor ~timestamp + ~round ~seed_nonce_hash ~payload_round ~liquidity_baking_toggle_vote ~user_activated_upgrades + ~force_apply:state.global_state.config.force_apply state.global_state.config.fees simulation_mode simulation_kind diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_simulator.ml b/src/proto_016_PtMumbai/lib_delegate/baking_simulator.ml index 2d09542279ad..5ec568002836 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_simulator.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_simulator.ml @@ -54,7 +54,7 @@ let () = type incremental = { predecessor : Baking_state.block_info; context : Tezos_protocol_environment.Context.t; - state : Protocol.validation_state * Protocol.application_state; + state : Protocol.validation_state * Protocol.application_state option; rev_operations : Operation.packed list; header : Tezos_base.Block_header.shell_header; } @@ -76,7 +76,7 @@ let check_context_consistency (abstract_index : Abstract_context_index.t) | true -> return_unit | false -> fail Invalid_context)) -let begin_construction ~timestamp ~protocol_data +let begin_construction ~timestamp ~protocol_data ~force_apply (abstract_index : Abstract_context_index.t) predecessor chain_id = protect (fun () -> let { @@ -119,12 +119,15 @@ let begin_construction ~timestamp ~protocol_data ~predecessor:pred_shell ~cache:`Lazy >>=? fun validation_state -> - Lifted_protocol.begin_application - context - chain_id - mode - ~predecessor:pred_shell - ~cache:`Lazy + (if force_apply then + Lifted_protocol.begin_application + context + chain_id + mode + ~predecessor:pred_shell + ~cache:`Lazy + >>=? return_some + else return_none) >>=? fun application_state -> let state = (validation_state, application_state) in return {predecessor; context; state; rev_operations = []; header}) @@ -143,7 +146,12 @@ let add_operation st (op : Operation.packed) = Protocol.validate_operation validation_state oph op in let** application_state, receipt = - Protocol.apply_operation application_state oph op + match application_state with + | Some application_state -> + Protocol.apply_operation application_state oph op + >>=? fun (application_state, receipt) -> + return (Some application_state, Some receipt) + | None -> return (None, None) in let state = (validation_state, application_state) in return ({st with state; rev_operations = op :: st.rev_operations}, receipt)) @@ -153,6 +161,10 @@ let finalize_construction inc = let validation_state, application_state = inc.state in let** () = Protocol.finalize_validation validation_state in let** result = - Protocol.finalize_application application_state (Some inc.header) + match application_state with + | Some application_state -> + Protocol.finalize_application application_state (Some inc.header) + >>=? return_some + | None -> return_none in return result) diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_simulator.mli b/src/proto_016_PtMumbai/lib_delegate/baking_simulator.mli index 0d0c7ac9f4d2..b05daf4f73da 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_simulator.mli +++ b/src/proto_016_PtMumbai/lib_delegate/baking_simulator.mli @@ -29,7 +29,7 @@ open Alpha_context type incremental = { predecessor : Baking_state.block_info; context : Tezos_protocol_environment.Context.t; - state : validation_state * application_state; + state : validation_state * application_state option; rev_operations : Operation.packed list; header : Tezos_base.Block_header.shell_header; } @@ -44,6 +44,7 @@ val check_context_consistency : val begin_construction : timestamp:Time.Protocol.t -> protocol_data:block_header_data -> + force_apply:bool -> Abstract_context_index.t -> Baking_state.block_info -> Chain_id.t -> @@ -52,10 +53,10 @@ val begin_construction : val add_operation : incremental -> Operation.packed -> - (incremental * operation_receipt) tzresult Lwt.t + (incremental * operation_receipt option) tzresult Lwt.t val finalize_construction : incremental -> - (Tezos_protocol_environment.validation_result * block_header_metadata) + (Tezos_protocol_environment.validation_result * block_header_metadata) option tzresult Lwt.t diff --git a/src/proto_016_PtMumbai/lib_delegate/block_forge.ml b/src/proto_016_PtMumbai/lib_delegate/block_forge.ml index 5ca78bd91bba..829d56a1570f 100644 --- a/src/proto_016_PtMumbai/lib_delegate/block_forge.ml +++ b/src/proto_016_PtMumbai/lib_delegate/block_forge.ml @@ -69,47 +69,93 @@ let convert_operation (op : packed_operation) : Tezos_base.Operation.t = op.protocol_data; } -let finalize_block_header shell_header timestamp validation_result - operations_hash predecessor_block_metadata_hash - predecessor_ops_metadata_hash predecessor_resulting_context = +(* [finalize_block_header ~chain ~shell_header ~timestamp ~validation_result + ~operations_hash ~pred_info ~round ~locked_round cctxt] updates the + [shell_header] that was created with dummy fields at the beginning of the + block construction. It increments the [level] and sets the actual + [operations_hash], [fitness], [validation_passes], and [context] (the + predecessor resulting context hash). + + When the operations from the block have been applied, the [fitness] is simply + retrieved from the [validation_result]. Otherwise, the [fitness] is computed + from the [round] and [locked_round] arguments. *) +let finalize_block_header ~chain ~shell_header ~timestamp ~validation_result + ~operations_hash ~(pred_info : Baking_state.block_info) ~round ~locked_round + cctxt = let open Lwt_result_syntax in - let {Tezos_protocol_environment.context; fitness; message; _} = - validation_result + let* fitness = + match validation_result with + | Some {Tezos_protocol_environment.context; fitness; message; _} -> + let*! test_chain = Context_ops.get_test_chain context in + let* context = + match test_chain with + | Not_running -> return context + | Running {expiration; _} -> + if Time.Protocol.(expiration <= timestamp) then + let*! context = + Context_ops.add_test_chain context Not_running + in + return context + else return context + | Forking _ -> assert false + in + let* context = + protect + ~on_error:(fun _ -> return context) + (fun () -> + let* predecessor_block_metadata_hash = + Shell_services.Blocks.metadata_hash + cctxt + ~block:(`Hash (pred_info.hash, 0)) + ~chain + () + in + Lwt.map + Result.ok + (Context_ops.add_predecessor_block_metadata_hash + context + predecessor_block_metadata_hash)) + in + let* context = + protect + ~on_error:(fun _ -> return context) + (fun () -> + let* predecessor_ops_metadata_hash = + Shell_services.Blocks.Operation_metadata_hashes.root + cctxt + ~block:(`Hash (pred_info.hash, 0)) + ~chain + () + in + Lwt.map + Result.ok + (Context_ops.add_predecessor_ops_metadata_hash + context + predecessor_ops_metadata_hash)) + in + let context = Context_ops.hash ~time:timestamp ?message context in + (* For the time being, we still fully build the block while we build + confidence to fully unplug the baker validation. The resulting + context hash is ignored as it is not necessary to craft a block. + See: https://gitlab.com/tezos/tezos/-/issues/4285 *) + ignore context ; + return fitness + | None -> + let*? level = + Environment.wrap_tzresult @@ Raw_level.of_int32 + @@ Int32.succ shell_header.Tezos_base.Block_header.level + in + let*? fitness = + Environment.wrap_tzresult + @@ Fitness.create + ~level + ~round + ~predecessor_round:pred_info.round + ~locked_round + in + return (Fitness.to_raw fitness) in let validation_passes = List.length Main.validation_passes in - let*! test_chain = Context_ops.get_test_chain context in - let* context = - match test_chain with - | Not_running -> return context - | Running {expiration; _} -> - if Time.Protocol.(expiration <= timestamp) then - let*! context = Context_ops.add_test_chain context Not_running in - return context - else return context - | Forking _ -> assert false - in - let*! context = - match predecessor_block_metadata_hash with - | Some predecessor_block_metadata_hash -> - Context_ops.add_predecessor_block_metadata_hash - context - predecessor_block_metadata_hash - | None -> Lwt.return context - in - let*! context = - match predecessor_ops_metadata_hash with - | Some predecessor_ops_metadata_hash -> - Context_ops.add_predecessor_ops_metadata_hash - context - predecessor_ops_metadata_hash - | None -> Lwt.return context - in - let context = Context_ops.hash ~time:timestamp ?message context in - (* For the time being, we still fully build the block while we build - confidence to fully unplug the baker validation. The resulting - context hash is ignored as it is not necessary to craft a block. - See: https://gitlab.com/tezos/tezos/-/issues/4285 *) - ignore context ; let header = Tezos_base.Block_header. { @@ -118,7 +164,7 @@ let finalize_block_header shell_header timestamp validation_result validation_passes; operations_hash; fitness; - context = predecessor_resulting_context; + context = pred_info.resulting_context_hash; } in return header @@ -129,21 +175,46 @@ let retain_live_operations_only ~live_blocks operation_pool = Block_hash.Set.mem shell.branch live_blocks) operation_pool +(* [check_protocol_changed] checks whether the protocol will change with the current + block. This function returns true if the block is the last of an [adoption] + period. It can also return true if an [user_activated_upgrades] is given. *) let check_protocol_changed ~user_activated_upgrades ~level - ~(validation_result : Tezos_protocol_environment.validation_result) = + ~(validation_result : Tezos_protocol_environment.validation_result option) + ~(incremental : Baking_simulator.incremental) = let open Lwt_result_syntax in - let*! next_protocol = Context_ops.get_protocol validation_result.context in - let next_protocol = - match - Tezos_base.Block_header.get_forced_protocol_upgrade - ~user_activated_upgrades - ~level - with - | None -> next_protocol - | Some hash -> hash - in - return Protocol_hash.(Protocol.hash <> next_protocol) + match + Tezos_base.Block_header.get_forced_protocol_upgrade + ~user_activated_upgrades + ~level + with + | None -> ( + match validation_result with + | None -> ( + let context = Validate.get_initial_ctxt (fst incremental.state) in + let* voting_period = + Lwt.map + Environment.wrap_tzresult + (Voting_period.get_current context) + in + match voting_period.kind with + | Voting_period.Proposal | Exploration | Cooldown | Promotion -> + return false + | Adoption -> + Lwt.map + Environment.wrap_tzresult + (Voting_period.is_last_block context)) + | Some validation_result -> + let*! next_protocol = + Context_ops.get_protocol validation_result.context + in + return Protocol_hash.(Protocol.hash <> next_protocol)) + | Some next_protocol -> return Protocol_hash.(Protocol.hash <> next_protocol) +(* [filter_via_node] filters operations using + {!Operation_selection.filter_operations_without_simulation} and then applies + them in a block via {!Node_rpc.preapply_block}. [filter_via_node] returns a + [shell_header], the list of operations that have been applied in the block + and the [payload_hash] corresponding to these operations. *) let filter_via_node ~chain_id ~fees_config ~hard_gas_limit_per_block ~faked_protocol_data ~timestamp ~(pred_info : Baking_state.block_info) ~payload_round ~operation_pool cctxt = @@ -180,16 +251,23 @@ let filter_via_node ~chain_id ~fees_config ~hard_gas_limit_per_block in return (shell_header, operations, payload_hash) +(* [filter_with_context] filters operations using a local context via + {!Operation_selection.filter_operations_with_simulation} and a fresh state + from {!Baking_simulator.begin_construction}. [finalize_block_header] is then + called and a [shell_header], the list of operations and the corresponding + [payload_hash] are returned. If the block is a transition block, + [filter_via_node] is called to return these values. *) let filter_with_context ~chain_id ~fees_config ~hard_gas_limit_per_block ~faked_protocol_data ~user_activated_upgrades ~timestamp - ~(pred_info : Baking_state.block_info) ~context_index ~payload_round - ~operation_pool cctxt = + ~(pred_info : Baking_state.block_info) ~force_apply ~round ~context_index + ~payload_round ~operation_pool cctxt = let open Lwt_result_syntax in let chain = `Hash chain_id in let* incremental = Baking_simulator.begin_construction ~timestamp ~protocol_data:faked_protocol_data + ~force_apply context_index pred_info chain_id @@ -206,6 +284,7 @@ let filter_with_context ~chain_id ~fees_config ~hard_gas_limit_per_block ~level:(Int32.succ pred_info.shell.level) ~user_activated_upgrades ~validation_result + ~incremental in if changed then (* Fallback to processing via node, which knows both old and new protocol. *) @@ -220,41 +299,17 @@ let filter_with_context ~chain_id ~fees_config ~hard_gas_limit_per_block ~operation_pool cctxt else - let* pred_block_metadata_hash = - protect - ~on_error:(fun _ -> return_none) - (fun () -> - let+ pred_block_metadata_hash = - Shell_services.Blocks.metadata_hash - cctxt - ~block:(`Hash (pred_info.hash, 0)) - ~chain - () - in - Some pred_block_metadata_hash) - in - let* pred_op_metadata_hash = - protect - ~on_error:(fun _ -> return_none) - (fun () -> - let+ pred_op_metadata_hash = - Shell_services.Blocks.Operation_metadata_hashes.root - cctxt - ~block:(`Hash (pred_info.hash, 0)) - ~chain - () - in - Some pred_op_metadata_hash) - in let* shell_header = finalize_block_header - incremental.header - timestamp - validation_result - operations_hash - pred_block_metadata_hash - pred_op_metadata_hash - pred_info.resulting_context_hash + ~chain + ~shell_header:incremental.header + ~timestamp + ~validation_result + ~operations_hash + ~pred_info + ~round + ~locked_round:None + cctxt in let operations = List.map (List.map convert_operation) operations in let payload_hash = @@ -269,6 +324,9 @@ let filter_with_context ~chain_id ~fees_config ~hard_gas_limit_per_block in return (shell_header, operations, payload_hash) +(* [apply_via_node] applies already filtered and validated operations in a block + via {!Node_rpc.preapply_block}. A [shell_header] is recovered from this call + and returned alongside of the list of operations and the payload_hash. *) let apply_via_node ~chain_id ~faked_protocol_data ~timestamp ~(pred_info : Baking_state.block_info) ~ordered_pool ~payload_hash cctxt = let open Lwt_result_syntax in @@ -286,15 +344,19 @@ let apply_via_node ~chain_id ~faked_protocol_data ~timestamp let operations = List.map (List.map convert_operation) operations in return (shell_header, operations, payload_hash) +(* [apply_with_context] is similar to [filter_with_context] but filters + consensus operations only from an [ordered_pool] via + {!Operation_selection.filter_consensus_operations_only}. *) let apply_with_context ~chain_id ~faked_protocol_data ~user_activated_upgrades - ~timestamp ~(pred_info : Baking_state.block_info) ~ordered_pool - ~context_index ~payload_hash cctxt = + ~timestamp ~(pred_info : Baking_state.block_info) ~force_apply ~round + ~ordered_pool ~context_index ~payload_hash cctxt = let open Lwt_result_syntax in let chain = `Hash chain_id in let* incremental = Baking_simulator.begin_construction ~timestamp ~protocol_data:faked_protocol_data + ~force_apply context_index pred_info chain_id @@ -319,14 +381,14 @@ let apply_with_context ~chain_id ~faked_protocol_data ~user_activated_upgrades let incremental = {incremental with header = {incremental.header with operations_hash}} in - let* validation_result, _ = - Baking_simulator.finalize_construction incremental - in + let* validation_result = Baking_simulator.finalize_construction incremental in + let validation_result = Option.map fst validation_result in let* changed = check_protocol_changed ~level:(Int32.succ pred_info.shell.level) ~user_activated_upgrades ~validation_result + ~incremental in if changed then (* Fallback to processing via node, which knows both old and new protocol. *) @@ -339,50 +401,40 @@ let apply_with_context ~chain_id ~faked_protocol_data ~user_activated_upgrades ~payload_hash cctxt else - let* pred_block_metadata_hash = - protect - ~on_error:(fun _ -> return_none) - (fun () -> - let+ pred_block_metadata_hash = - Shell_services.Blocks.metadata_hash - cctxt - ~block:(`Hash (pred_info.hash, 0)) - ~chain - () - in - Some pred_block_metadata_hash) - in - let* pred_op_metadata_hash = - protect - ~on_error:(fun _ -> return_none) - (fun () -> - let+ pred_op_metadata_hash = - Shell_services.Blocks.Operation_metadata_hashes.root - cctxt - ~block:(`Hash (pred_info.hash, 0)) - ~chain - () - in - Some pred_op_metadata_hash) + let locked_round_when_no_validation_result = + (* [locked_round] will not be used in [finalize_block_header] if there is + a [validation_result] *) + if Option.is_some validation_result then None + else + List.find_map + (fun {protocol_data = Operation_data protocol_data; _} -> + match protocol_data.contents with + | Single (Preendorsement {round; _}) -> Some round + | _ -> None) + (Option.value (List.hd operations) ~default:[]) in let* shell_header = finalize_block_header - incremental.header - timestamp - validation_result - operations_hash - pred_block_metadata_hash - pred_op_metadata_hash - pred_info.resulting_context_hash + ~chain + ~shell_header:incremental.header + ~timestamp + ~validation_result + ~operations_hash + ~pred_info + ~round + ~locked_round:locked_round_when_no_validation_result + cctxt in let operations = List.map (List.map convert_operation) operations in return (shell_header, operations, payload_hash) -(* [forge] a new [unsigned_block] regarding of [simulation_kind] and [simulation_mode] *) +(* [forge] a new [unsigned_block] in accordance with [simulation_kind] and + [simulation_mode] *) let forge (cctxt : #Protocol_client_context.full) ~chain_id - ~(pred_info : Baking_state.block_info) ~timestamp + ~(pred_info : Baking_state.block_info) ~timestamp ~round ~liquidity_baking_toggle_vote ~user_activated_upgrades fees_config - ~seed_nonce_hash ~payload_round simulation_mode simulation_kind constants = + ~force_apply ~seed_nonce_hash ~payload_round simulation_mode simulation_kind + constants = let open Lwt_result_syntax in let hard_gas_limit_per_block = constants.Constants.Parametric.hard_gas_limit_per_block @@ -453,6 +505,8 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~user_activated_upgrades ~timestamp ~pred_info + ~force_apply + ~round ~context_index ~payload_round ~operation_pool @@ -472,6 +526,8 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~user_activated_upgrades ~timestamp ~pred_info + ~force_apply + ~round ~ordered_pool ~context_index ~payload_hash diff --git a/src/proto_016_PtMumbai/lib_delegate/block_forge.mli b/src/proto_016_PtMumbai/lib_delegate/block_forge.mli index b111e27b417c..77fb619a192f 100644 --- a/src/proto_016_PtMumbai/lib_delegate/block_forge.mli +++ b/src/proto_016_PtMumbai/lib_delegate/block_forge.mli @@ -40,22 +40,16 @@ type simulation_kind = type simulation_mode = Local of Context.index | Node -val forge_faked_protocol_data : - ?payload_hash:Block_payload_hash.t -> - payload_round:Round.t -> - seed_nonce_hash:Nonce_hash.t option -> - liquidity_baking_toggle_vote:Liquidity_baking.liquidity_baking_toggle_vote -> - unit -> - block_header_data - val forge : #Protocol_client_context.full -> chain_id:Chain_id.t -> pred_info:Baking_state.block_info -> timestamp:Time.Protocol.t -> + round:Round.t -> liquidity_baking_toggle_vote:Liquidity_baking.liquidity_baking_toggle_vote -> user_activated_upgrades:User_activated.upgrades -> Baking_configuration.fees_config -> + force_apply:bool -> seed_nonce_hash:Nonce_hash.t option -> payload_round:Round.t -> Baking_state.validation_mode -> diff --git a/src/proto_016_PtMumbai/lib_delegate/operation_selection.ml b/src/proto_016_PtMumbai/lib_delegate/operation_selection.ml index 089f4b439a93..f08add33e4c6 100644 --- a/src/proto_016_PtMumbai/lib_delegate/operation_selection.ml +++ b/src/proto_016_PtMumbai/lib_delegate/operation_selection.ml @@ -162,8 +162,8 @@ let prioritize_managers ~hard_gas_limit_per_block ~minimal_fees (** Simulation *) type simulation_result = { - validation_result : Tezos_protocol_environment.validation_result; - block_header_metadata : block_header_metadata; + validation_result : Tezos_protocol_environment.validation_result option; + block_header_metadata : block_header_metadata option; operations : packed_operation list list; operations_hash : Operation_list_list_hash.t; } @@ -173,7 +173,10 @@ let validate_operation inc op = | Error errs -> Events.(emit invalid_operation_filtered) (Operation.hash_packed op, errs) >>= fun () -> Lwt.return_none - | Ok (resulting_state, receipt) -> ( + | Ok (resulting_state, None) -> + (* No receipt if force_apply is not set *) + Lwt.return_some resulting_state + | Ok (resulting_state, Some receipt) -> ( (* Check that the metadata are serializable/deserializable *) let encoding_result = let enc = Protocol.operation_receipt_encoding in @@ -257,9 +260,23 @@ let filter_operations_with_simulation initial_inc fees_config operations) in let inc = {inc with header = {inc.header with operations_hash}} in - Baking_simulator.finalize_construction inc - >>=? fun (validation_result, block_header_metadata) -> - return {validation_result; block_header_metadata; operations; operations_hash} + Baking_simulator.finalize_construction inc >>=? function + | Some (validation_result, block_header_metadata) -> + return + { + validation_result = Some validation_result; + block_header_metadata = Some block_header_metadata; + operations; + operations_hash; + } + | None -> + return + { + validation_result = None; + block_header_metadata = None; + operations; + operations_hash; + } let filter_valid_operations_up_to_quota_without_simulation (ops, quota) = let {Tezos_protocol_environment.max_size; max_op} = quota in diff --git a/src/proto_016_PtMumbai/lib_delegate/operation_selection.mli b/src/proto_016_PtMumbai/lib_delegate/operation_selection.mli index 78caab5c158c..8844dbed806d 100644 --- a/src/proto_016_PtMumbai/lib_delegate/operation_selection.mli +++ b/src/proto_016_PtMumbai/lib_delegate/operation_selection.mli @@ -28,8 +28,8 @@ open Alpha_context open Tezos_protocol_environment type simulation_result = { - validation_result : validation_result; - block_header_metadata : Apply_results.block_metadata; + validation_result : validation_result option; + block_header_metadata : Apply_results.block_metadata option; operations : packed_operation list list; operations_hash : Operation_list_list_hash.t; } diff --git a/src/proto_alpha/lib_delegate/baking_actions.ml b/src/proto_alpha/lib_delegate/baking_actions.ml index 56f56616c151..e05821d643bc 100644 --- a/src/proto_alpha/lib_delegate/baking_actions.ml +++ b/src/proto_alpha/lib_delegate/baking_actions.ml @@ -306,10 +306,12 @@ let inject_block ~state_recorder state block_to_bake ~updated_state = ~chain_id ~pred_info:predecessor ~timestamp + ~round ~seed_nonce_hash ~payload_round ~liquidity_baking_toggle_vote ~user_activated_upgrades + ~force_apply:state.global_state.config.force_apply state.global_state.config.fees simulation_mode simulation_kind diff --git a/src/proto_alpha/lib_delegate/baking_simulator.ml b/src/proto_alpha/lib_delegate/baking_simulator.ml index 2d09542279ad..5ec568002836 100644 --- a/src/proto_alpha/lib_delegate/baking_simulator.ml +++ b/src/proto_alpha/lib_delegate/baking_simulator.ml @@ -54,7 +54,7 @@ let () = type incremental = { predecessor : Baking_state.block_info; context : Tezos_protocol_environment.Context.t; - state : Protocol.validation_state * Protocol.application_state; + state : Protocol.validation_state * Protocol.application_state option; rev_operations : Operation.packed list; header : Tezos_base.Block_header.shell_header; } @@ -76,7 +76,7 @@ let check_context_consistency (abstract_index : Abstract_context_index.t) | true -> return_unit | false -> fail Invalid_context)) -let begin_construction ~timestamp ~protocol_data +let begin_construction ~timestamp ~protocol_data ~force_apply (abstract_index : Abstract_context_index.t) predecessor chain_id = protect (fun () -> let { @@ -119,12 +119,15 @@ let begin_construction ~timestamp ~protocol_data ~predecessor:pred_shell ~cache:`Lazy >>=? fun validation_state -> - Lifted_protocol.begin_application - context - chain_id - mode - ~predecessor:pred_shell - ~cache:`Lazy + (if force_apply then + Lifted_protocol.begin_application + context + chain_id + mode + ~predecessor:pred_shell + ~cache:`Lazy + >>=? return_some + else return_none) >>=? fun application_state -> let state = (validation_state, application_state) in return {predecessor; context; state; rev_operations = []; header}) @@ -143,7 +146,12 @@ let add_operation st (op : Operation.packed) = Protocol.validate_operation validation_state oph op in let** application_state, receipt = - Protocol.apply_operation application_state oph op + match application_state with + | Some application_state -> + Protocol.apply_operation application_state oph op + >>=? fun (application_state, receipt) -> + return (Some application_state, Some receipt) + | None -> return (None, None) in let state = (validation_state, application_state) in return ({st with state; rev_operations = op :: st.rev_operations}, receipt)) @@ -153,6 +161,10 @@ let finalize_construction inc = let validation_state, application_state = inc.state in let** () = Protocol.finalize_validation validation_state in let** result = - Protocol.finalize_application application_state (Some inc.header) + match application_state with + | Some application_state -> + Protocol.finalize_application application_state (Some inc.header) + >>=? return_some + | None -> return_none in return result) diff --git a/src/proto_alpha/lib_delegate/baking_simulator.mli b/src/proto_alpha/lib_delegate/baking_simulator.mli index 0d0c7ac9f4d2..b05daf4f73da 100644 --- a/src/proto_alpha/lib_delegate/baking_simulator.mli +++ b/src/proto_alpha/lib_delegate/baking_simulator.mli @@ -29,7 +29,7 @@ open Alpha_context type incremental = { predecessor : Baking_state.block_info; context : Tezos_protocol_environment.Context.t; - state : validation_state * application_state; + state : validation_state * application_state option; rev_operations : Operation.packed list; header : Tezos_base.Block_header.shell_header; } @@ -44,6 +44,7 @@ val check_context_consistency : val begin_construction : timestamp:Time.Protocol.t -> protocol_data:block_header_data -> + force_apply:bool -> Abstract_context_index.t -> Baking_state.block_info -> Chain_id.t -> @@ -52,10 +53,10 @@ val begin_construction : val add_operation : incremental -> Operation.packed -> - (incremental * operation_receipt) tzresult Lwt.t + (incremental * operation_receipt option) tzresult Lwt.t val finalize_construction : incremental -> - (Tezos_protocol_environment.validation_result * block_header_metadata) + (Tezos_protocol_environment.validation_result * block_header_metadata) option tzresult Lwt.t diff --git a/src/proto_alpha/lib_delegate/block_forge.ml b/src/proto_alpha/lib_delegate/block_forge.ml index 36d743ba2a65..effadb789594 100644 --- a/src/proto_alpha/lib_delegate/block_forge.ml +++ b/src/proto_alpha/lib_delegate/block_forge.ml @@ -69,47 +69,92 @@ let convert_operation (op : packed_operation) : Tezos_base.Operation.t = op.protocol_data; } -let finalize_block_header shell_header timestamp validation_result - operations_hash predecessor_block_metadata_hash - predecessor_ops_metadata_hash predecessor_resulting_context = +(* [finalize_block_header ~shell_header ~validation_result ~operations_hash + ~pred_info ~round ~locked_round cctxt] updates the [shell_header] that was created + with dummy fields at the beginning of the block construction. It increments + the [level] and sets the actual [operations_hash], [fitness], + [validation_passes], and [context] (the predecessor resulting context hash). + + When the operations from the block have been applied, the [fitness] is simply + retrieved from the [validation_result]. Otherwise, the [fitness] is computed + from the [round] and [locked_round] arguments. *) +let finalize_block_header ~chain ~shell_header ~timestamp ~validation_result + ~operations_hash ~(pred_info : Baking_state.block_info) ~round ~locked_round + cctxt = let open Lwt_result_syntax in - let {Tezos_protocol_environment.context; fitness; message; _} = - validation_result + let* fitness = + match validation_result with + | Some {Tezos_protocol_environment.context; fitness; message; _} -> + let*! test_chain = Context_ops.get_test_chain context in + let* context = + match test_chain with + | Not_running -> return context + | Running {expiration; _} -> + if Time.Protocol.(expiration <= timestamp) then + let*! context = + Context_ops.add_test_chain context Not_running + in + return context + else return context + | Forking _ -> assert false + in + let* context = + protect + ~on_error:(fun _ -> return context) + (fun () -> + let* predecessor_block_metadata_hash = + Shell_services.Blocks.metadata_hash + cctxt + ~block:(`Hash (pred_info.hash, 0)) + ~chain + () + in + Lwt.map + Result.ok + (Context_ops.add_predecessor_block_metadata_hash + context + predecessor_block_metadata_hash)) + in + let* context = + protect + ~on_error:(fun _ -> return context) + (fun () -> + let* predecessor_ops_metadata_hash = + Shell_services.Blocks.Operation_metadata_hashes.root + cctxt + ~block:(`Hash (pred_info.hash, 0)) + ~chain + () + in + Lwt.map + Result.ok + (Context_ops.add_predecessor_ops_metadata_hash + context + predecessor_ops_metadata_hash)) + in + let context = Context_ops.hash ~time:timestamp ?message context in + (* For the time being, we still fully build the block while we build + confidence to fully unplug the baker validation. The resulting + context hash is ignored as it is not necessary to craft a block. + See: https://gitlab.com/tezos/tezos/-/issues/4285 *) + ignore context ; + return fitness + | None -> + let*? level = + Environment.wrap_tzresult @@ Raw_level.of_int32 + @@ Int32.succ shell_header.Tezos_base.Block_header.level + in + let*? fitness = + Environment.wrap_tzresult + @@ Fitness.create + ~level + ~round + ~predecessor_round:pred_info.round + ~locked_round + in + return (Fitness.to_raw fitness) in let validation_passes = List.length Main.validation_passes in - let*! test_chain = Context_ops.get_test_chain context in - let* context = - match test_chain with - | Not_running -> return context - | Running {expiration; _} -> - if Time.Protocol.(expiration <= timestamp) then - let*! context = Context_ops.add_test_chain context Not_running in - return context - else return context - | Forking _ -> assert false - in - let*! context = - match predecessor_block_metadata_hash with - | Some predecessor_block_metadata_hash -> - Context_ops.add_predecessor_block_metadata_hash - context - predecessor_block_metadata_hash - | None -> Lwt.return context - in - let*! context = - match predecessor_ops_metadata_hash with - | Some predecessor_ops_metadata_hash -> - Context_ops.add_predecessor_ops_metadata_hash - context - predecessor_ops_metadata_hash - | None -> Lwt.return context - in - let context = Context_ops.hash ~time:timestamp ?message context in - (* For the time being, we still fully build the block while we build - confidence to fully unplug the baker validation. The resulting - context hash is ignored as it is not necessary to craft a block. - See: https://gitlab.com/tezos/tezos/-/issues/4285 *) - ignore context ; let header = Tezos_base.Block_header. { @@ -118,7 +163,7 @@ let finalize_block_header shell_header timestamp validation_result validation_passes; operations_hash; fitness; - context = predecessor_resulting_context; + context = pred_info.resulting_context_hash; } in return header @@ -129,21 +174,46 @@ let retain_live_operations_only ~live_blocks operation_pool = Block_hash.Set.mem shell.branch live_blocks) operation_pool +(* [check_protocol_changed] checks whether the protocol will change with the current + block. This function returns true if the block is the last of an [adoption] + period. It can also return true if an [user_activated_upgrades] is given. *) let check_protocol_changed ~user_activated_upgrades ~level - ~(validation_result : Tezos_protocol_environment.validation_result) = + ~(validation_result : Tezos_protocol_environment.validation_result option) + ~(incremental : Baking_simulator.incremental) = let open Lwt_result_syntax in - let*! next_protocol = Context_ops.get_protocol validation_result.context in - let next_protocol = - match - Tezos_base.Block_header.get_forced_protocol_upgrade - ~user_activated_upgrades - ~level - with - | None -> next_protocol - | Some hash -> hash - in - return Protocol_hash.(Protocol.hash <> next_protocol) + match + Tezos_base.Block_header.get_forced_protocol_upgrade + ~user_activated_upgrades + ~level + with + | None -> ( + match validation_result with + | None -> ( + let context = Validate.get_initial_ctxt (fst incremental.state) in + let* voting_period = + Lwt.map + Environment.wrap_tzresult + (Voting_period.get_current context) + in + match voting_period.kind with + | Voting_period.Proposal | Exploration | Cooldown | Promotion -> + return false + | Adoption -> + Lwt.map + Environment.wrap_tzresult + (Voting_period.is_last_block context)) + | Some validation_result -> + let*! next_protocol = + Context_ops.get_protocol validation_result.context + in + return Protocol_hash.(Protocol.hash <> next_protocol)) + | Some next_protocol -> return Protocol_hash.(Protocol.hash <> next_protocol) +(* [filter_via_node] filters operations using + {!Operation_selection.filter_operations_without_simulation} and then applies + them in a block via {!Node_rpc.preapply_block}. [filter_via_node] returns a + [shell_header], the list of operations that have been applied in the block + and the [payload_hash] corresponding to these operations. *) let filter_via_node ~chain_id ~fees_config ~hard_gas_limit_per_block ~faked_protocol_data ~timestamp ~(pred_info : Baking_state.block_info) ~payload_round ~operation_pool cctxt = @@ -180,16 +250,23 @@ let filter_via_node ~chain_id ~fees_config ~hard_gas_limit_per_block in return (shell_header, operations, payload_hash) +(* [filter_with_context] filters operations using a local context via + {!Operation_selection.filter_operations_with_simulation} and a fresh state + from {!Baking_simulator.begin_construction}. [finalize_block_header] is then + called and a [shell_header], the list of operations and the corresponding + [payload_hash] are returned. If the block is a transition block, + [filter_via_node] is called to return these values. *) let filter_with_context ~chain_id ~fees_config ~hard_gas_limit_per_block ~faked_protocol_data ~user_activated_upgrades ~timestamp - ~(pred_info : Baking_state.block_info) ~context_index ~payload_round - ~operation_pool cctxt = + ~(pred_info : Baking_state.block_info) ~force_apply ~round ~context_index + ~payload_round ~operation_pool cctxt = let open Lwt_result_syntax in let chain = `Hash chain_id in let* incremental = Baking_simulator.begin_construction ~timestamp ~protocol_data:faked_protocol_data + ~force_apply context_index pred_info chain_id @@ -206,6 +283,7 @@ let filter_with_context ~chain_id ~fees_config ~hard_gas_limit_per_block ~level:(Int32.succ pred_info.shell.level) ~user_activated_upgrades ~validation_result + ~incremental in if changed then (* Fallback to processing via node, which knows both old and new protocol. *) @@ -220,41 +298,17 @@ let filter_with_context ~chain_id ~fees_config ~hard_gas_limit_per_block ~operation_pool cctxt else - let* pred_block_metadata_hash = - protect - ~on_error:(fun _ -> return_none) - (fun () -> - let+ pred_block_metadata_hash = - Shell_services.Blocks.metadata_hash - cctxt - ~block:(`Hash (pred_info.hash, 0)) - ~chain - () - in - Some pred_block_metadata_hash) - in - let* pred_op_metadata_hash = - protect - ~on_error:(fun _ -> return_none) - (fun () -> - let+ pred_op_metadata_hash = - Shell_services.Blocks.Operation_metadata_hashes.root - cctxt - ~block:(`Hash (pred_info.hash, 0)) - ~chain - () - in - Some pred_op_metadata_hash) - in let* shell_header = finalize_block_header - incremental.header - timestamp - validation_result - operations_hash - pred_block_metadata_hash - pred_op_metadata_hash - pred_info.resulting_context_hash + ~chain + ~shell_header:incremental.header + ~timestamp + ~validation_result + ~operations_hash + ~pred_info + ~round + ~locked_round:None + cctxt in let operations = List.map (List.map convert_operation) operations in let payload_hash = @@ -269,6 +323,9 @@ let filter_with_context ~chain_id ~fees_config ~hard_gas_limit_per_block in return (shell_header, operations, payload_hash) +(* [apply_via_node] applies already filtered and validated operations in a block + via {!Node_rpc.preapply_block}. A [shell_header] is recovered from this call + and returned alongside of the list of operations and the payload_hash. *) let apply_via_node ~chain_id ~faked_protocol_data ~timestamp ~(pred_info : Baking_state.block_info) ~ordered_pool ~payload_hash cctxt = let open Lwt_result_syntax in @@ -286,15 +343,19 @@ let apply_via_node ~chain_id ~faked_protocol_data ~timestamp let operations = List.map (List.map convert_operation) operations in return (shell_header, operations, payload_hash) +(* [apply_with_context] is similar to [filter_with_context] but filters + consensus operations only from an [ordered_pool] via + {!Operation_selection.filter_consensus_operations_only}. *) let apply_with_context ~chain_id ~faked_protocol_data ~user_activated_upgrades - ~timestamp ~(pred_info : Baking_state.block_info) ~ordered_pool - ~context_index ~payload_hash cctxt = + ~timestamp ~(pred_info : Baking_state.block_info) ~force_apply ~round + ~ordered_pool ~context_index ~payload_hash cctxt = let open Lwt_result_syntax in let chain = `Hash chain_id in let* incremental = Baking_simulator.begin_construction ~timestamp ~protocol_data:faked_protocol_data + ~force_apply context_index pred_info chain_id @@ -319,14 +380,14 @@ let apply_with_context ~chain_id ~faked_protocol_data ~user_activated_upgrades let incremental = {incremental with header = {incremental.header with operations_hash}} in - let* validation_result, _ = - Baking_simulator.finalize_construction incremental - in + let* validation_result = Baking_simulator.finalize_construction incremental in + let validation_result = Option.map fst validation_result in let* changed = check_protocol_changed ~level:(Int32.succ pred_info.shell.level) ~user_activated_upgrades ~validation_result + ~incremental in if changed then (* Fallback to processing via node, which knows both old and new protocol. *) @@ -339,50 +400,40 @@ let apply_with_context ~chain_id ~faked_protocol_data ~user_activated_upgrades ~payload_hash cctxt else - let* pred_block_metadata_hash = - protect - ~on_error:(fun _ -> return_none) - (fun () -> - let+ pred_block_metadata_hash = - Shell_services.Blocks.metadata_hash - cctxt - ~block:(`Hash (pred_info.hash, 0)) - ~chain - () - in - Some pred_block_metadata_hash) - in - let* pred_op_metadata_hash = - protect - ~on_error:(fun _ -> return_none) - (fun () -> - let+ pred_op_metadata_hash = - Shell_services.Blocks.Operation_metadata_hashes.root - cctxt - ~block:(`Hash (pred_info.hash, 0)) - ~chain - () - in - Some pred_op_metadata_hash) + let locked_round_when_no_validation_result = + (* [locked_round] will not be used in [finalize_block_header] if there is + a [validation_result] *) + if Option.is_some validation_result then None + else + List.find_map + (fun {protocol_data = Operation_data protocol_data; _} -> + match protocol_data.contents with + | Single (Preendorsement {round; _}) -> Some round + | _ -> None) + (Option.value (List.hd operations) ~default:[]) in let* shell_header = finalize_block_header - incremental.header - timestamp - validation_result - operations_hash - pred_block_metadata_hash - pred_op_metadata_hash - pred_info.resulting_context_hash + ~chain + ~shell_header:incremental.header + ~timestamp + ~validation_result + ~operations_hash + ~pred_info + ~round + ~locked_round:locked_round_when_no_validation_result + cctxt in let operations = List.map (List.map convert_operation) operations in return (shell_header, operations, payload_hash) -(* [forge] a new [unsigned_block] regarding of [simulation_kind] and [simulation_mode] *) +(* [forge] a new [unsigned_block] in accordance with [simulation_kind] and + [simulation_mode] *) let forge (cctxt : #Protocol_client_context.full) ~chain_id - ~(pred_info : Baking_state.block_info) ~timestamp + ~(pred_info : Baking_state.block_info) ~timestamp ~round ~liquidity_baking_toggle_vote ~user_activated_upgrades fees_config - ~seed_nonce_hash ~payload_round simulation_mode simulation_kind constants = + ~force_apply ~seed_nonce_hash ~payload_round simulation_mode simulation_kind + constants = let open Lwt_result_syntax in let hard_gas_limit_per_block = constants.Constants.Parametric.hard_gas_limit_per_block @@ -453,6 +504,8 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~user_activated_upgrades ~timestamp ~pred_info + ~force_apply + ~round ~context_index ~payload_round ~operation_pool @@ -472,6 +525,8 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~user_activated_upgrades ~timestamp ~pred_info + ~force_apply + ~round ~ordered_pool ~context_index ~payload_hash diff --git a/src/proto_alpha/lib_delegate/block_forge.mli b/src/proto_alpha/lib_delegate/block_forge.mli index b111e27b417c..77fb619a192f 100644 --- a/src/proto_alpha/lib_delegate/block_forge.mli +++ b/src/proto_alpha/lib_delegate/block_forge.mli @@ -40,22 +40,16 @@ type simulation_kind = type simulation_mode = Local of Context.index | Node -val forge_faked_protocol_data : - ?payload_hash:Block_payload_hash.t -> - payload_round:Round.t -> - seed_nonce_hash:Nonce_hash.t option -> - liquidity_baking_toggle_vote:Liquidity_baking.liquidity_baking_toggle_vote -> - unit -> - block_header_data - val forge : #Protocol_client_context.full -> chain_id:Chain_id.t -> pred_info:Baking_state.block_info -> timestamp:Time.Protocol.t -> + round:Round.t -> liquidity_baking_toggle_vote:Liquidity_baking.liquidity_baking_toggle_vote -> user_activated_upgrades:User_activated.upgrades -> Baking_configuration.fees_config -> + force_apply:bool -> seed_nonce_hash:Nonce_hash.t option -> payload_round:Round.t -> Baking_state.validation_mode -> diff --git a/src/proto_alpha/lib_delegate/operation_selection.ml b/src/proto_alpha/lib_delegate/operation_selection.ml index 9a4fbf4d0b92..cc33ddbfca71 100644 --- a/src/proto_alpha/lib_delegate/operation_selection.ml +++ b/src/proto_alpha/lib_delegate/operation_selection.ml @@ -160,8 +160,8 @@ let prioritize_managers ~hard_gas_limit_per_block ~minimal_fees (** Simulation *) type simulation_result = { - validation_result : Tezos_protocol_environment.validation_result; - block_header_metadata : block_header_metadata; + validation_result : Tezos_protocol_environment.validation_result option; + block_header_metadata : block_header_metadata option; operations : packed_operation list list; operations_hash : Operation_list_list_hash.t; } @@ -171,7 +171,10 @@ let validate_operation inc op = | Error errs -> Events.(emit invalid_operation_filtered) (Operation.hash_packed op, errs) >>= fun () -> Lwt.return_none - | Ok (resulting_state, receipt) -> ( + | Ok (resulting_state, None) -> + (* No receipt if force_apply is not set *) + Lwt.return_some resulting_state + | Ok (resulting_state, Some receipt) -> ( (* Check that the metadata are serializable/deserializable *) let encoding_result = let enc = Protocol.operation_receipt_encoding in @@ -255,9 +258,23 @@ let filter_operations_with_simulation initial_inc fees_config operations) in let inc = {inc with header = {inc.header with operations_hash}} in - Baking_simulator.finalize_construction inc - >>=? fun (validation_result, block_header_metadata) -> - return {validation_result; block_header_metadata; operations; operations_hash} + Baking_simulator.finalize_construction inc >>=? function + | Some (validation_result, block_header_metadata) -> + return + { + validation_result = Some validation_result; + block_header_metadata = Some block_header_metadata; + operations; + operations_hash; + } + | None -> + return + { + validation_result = None; + block_header_metadata = None; + operations; + operations_hash; + } let filter_valid_operations_up_to_quota_without_simulation (ops, quota) = let {Tezos_protocol_environment.max_size; max_op} = quota in diff --git a/src/proto_alpha/lib_delegate/operation_selection.mli b/src/proto_alpha/lib_delegate/operation_selection.mli index 78caab5c158c..8844dbed806d 100644 --- a/src/proto_alpha/lib_delegate/operation_selection.mli +++ b/src/proto_alpha/lib_delegate/operation_selection.mli @@ -28,8 +28,8 @@ open Alpha_context open Tezos_protocol_environment type simulation_result = { - validation_result : validation_result; - block_header_metadata : Apply_results.block_metadata; + validation_result : validation_result option; + block_header_metadata : Apply_results.block_metadata option; operations : packed_operation list list; operations_hash : Operation_list_list_hash.t; } -- GitLab From 3770323c498f3848941e3f27688fbc3b7f832f40 Mon Sep 17 00:00:00 2001 From: Albin Coquereau Date: Wed, 25 Jan 2023 16:39:48 +0100 Subject: [PATCH 12/46] lib_delegate: remove context finalisation in block_forge --- .../lib_delegate/block_forge.ml | 80 ++----------------- src/proto_alpha/lib_delegate/block_forge.ml | 71 +--------------- 2 files changed, 12 insertions(+), 139 deletions(-) diff --git a/src/proto_016_PtMumbai/lib_delegate/block_forge.ml b/src/proto_016_PtMumbai/lib_delegate/block_forge.ml index 829d56a1570f..fedc2f80510b 100644 --- a/src/proto_016_PtMumbai/lib_delegate/block_forge.ml +++ b/src/proto_016_PtMumbai/lib_delegate/block_forge.ml @@ -69,77 +69,21 @@ let convert_operation (op : packed_operation) : Tezos_base.Operation.t = op.protocol_data; } -(* [finalize_block_header ~chain ~shell_header ~timestamp ~validation_result - ~operations_hash ~pred_info ~round ~locked_round cctxt] updates the - [shell_header] that was created with dummy fields at the beginning of the - block construction. It increments the [level] and sets the actual - [operations_hash], [fitness], [validation_passes], and [context] (the - predecessor resulting context hash). +(* [finalize_block_header ~shell_header ~validation_result ~operations_hash + ~pred_info ~round ~locked_round] updates the [shell_header] that was created + with dummy fields at the beginning of the block construction. It increments + the [level] and sets the actual [operations_hash], [fitness], + [validation_passes], and [context] (the predecessor resulting context hash). When the operations from the block have been applied, the [fitness] is simply retrieved from the [validation_result]. Otherwise, the [fitness] is computed from the [round] and [locked_round] arguments. *) -let finalize_block_header ~chain ~shell_header ~timestamp ~validation_result - ~operations_hash ~(pred_info : Baking_state.block_info) ~round ~locked_round - cctxt = +let finalize_block_header ~shell_header ~validation_result ~operations_hash + ~(pred_info : Baking_state.block_info) ~round ~locked_round = let open Lwt_result_syntax in let* fitness = match validation_result with - | Some {Tezos_protocol_environment.context; fitness; message; _} -> - let*! test_chain = Context_ops.get_test_chain context in - let* context = - match test_chain with - | Not_running -> return context - | Running {expiration; _} -> - if Time.Protocol.(expiration <= timestamp) then - let*! context = - Context_ops.add_test_chain context Not_running - in - return context - else return context - | Forking _ -> assert false - in - let* context = - protect - ~on_error:(fun _ -> return context) - (fun () -> - let* predecessor_block_metadata_hash = - Shell_services.Blocks.metadata_hash - cctxt - ~block:(`Hash (pred_info.hash, 0)) - ~chain - () - in - Lwt.map - Result.ok - (Context_ops.add_predecessor_block_metadata_hash - context - predecessor_block_metadata_hash)) - in - let* context = - protect - ~on_error:(fun _ -> return context) - (fun () -> - let* predecessor_ops_metadata_hash = - Shell_services.Blocks.Operation_metadata_hashes.root - cctxt - ~block:(`Hash (pred_info.hash, 0)) - ~chain - () - in - Lwt.map - Result.ok - (Context_ops.add_predecessor_ops_metadata_hash - context - predecessor_ops_metadata_hash)) - in - let context = Context_ops.hash ~time:timestamp ?message context in - (* For the time being, we still fully build the block while we build - confidence to fully unplug the baker validation. The resulting - context hash is ignored as it is not necessary to craft a block. - See: https://gitlab.com/tezos/tezos/-/issues/4285 *) - ignore context ; - return fitness + | Some {Tezos_protocol_environment.fitness; _} -> return fitness | None -> let*? level = Environment.wrap_tzresult @@ Raw_level.of_int32 @@ -262,7 +206,6 @@ let filter_with_context ~chain_id ~fees_config ~hard_gas_limit_per_block ~(pred_info : Baking_state.block_info) ~force_apply ~round ~context_index ~payload_round ~operation_pool cctxt = let open Lwt_result_syntax in - let chain = `Hash chain_id in let* incremental = Baking_simulator.begin_construction ~timestamp @@ -301,15 +244,12 @@ let filter_with_context ~chain_id ~fees_config ~hard_gas_limit_per_block else let* shell_header = finalize_block_header - ~chain ~shell_header:incremental.header - ~timestamp ~validation_result ~operations_hash ~pred_info ~round ~locked_round:None - cctxt in let operations = List.map (List.map convert_operation) operations in let payload_hash = @@ -351,7 +291,6 @@ let apply_with_context ~chain_id ~faked_protocol_data ~user_activated_upgrades ~timestamp ~(pred_info : Baking_state.block_info) ~force_apply ~round ~ordered_pool ~context_index ~payload_hash cctxt = let open Lwt_result_syntax in - let chain = `Hash chain_id in let* incremental = Baking_simulator.begin_construction ~timestamp @@ -415,15 +354,12 @@ let apply_with_context ~chain_id ~faked_protocol_data ~user_activated_upgrades in let* shell_header = finalize_block_header - ~chain ~shell_header:incremental.header - ~timestamp ~validation_result ~operations_hash ~pred_info ~round ~locked_round:locked_round_when_no_validation_result - cctxt in let operations = List.map (List.map convert_operation) operations in return (shell_header, operations, payload_hash) diff --git a/src/proto_alpha/lib_delegate/block_forge.ml b/src/proto_alpha/lib_delegate/block_forge.ml index effadb789594..d3be206ac6b6 100644 --- a/src/proto_alpha/lib_delegate/block_forge.ml +++ b/src/proto_alpha/lib_delegate/block_forge.ml @@ -70,7 +70,7 @@ let convert_operation (op : packed_operation) : Tezos_base.Operation.t = } (* [finalize_block_header ~shell_header ~validation_result ~operations_hash - ~pred_info ~round ~locked_round cctxt] updates the [shell_header] that was created + ~pred_info ~round ~locked_round] updates the [shell_header] that was created with dummy fields at the beginning of the block construction. It increments the [level] and sets the actual [operations_hash], [fitness], [validation_passes], and [context] (the predecessor resulting context hash). @@ -78,67 +78,12 @@ let convert_operation (op : packed_operation) : Tezos_base.Operation.t = When the operations from the block have been applied, the [fitness] is simply retrieved from the [validation_result]. Otherwise, the [fitness] is computed from the [round] and [locked_round] arguments. *) -let finalize_block_header ~chain ~shell_header ~timestamp ~validation_result - ~operations_hash ~(pred_info : Baking_state.block_info) ~round ~locked_round - cctxt = +let finalize_block_header ~shell_header ~validation_result ~operations_hash + ~(pred_info : Baking_state.block_info) ~round ~locked_round = let open Lwt_result_syntax in let* fitness = match validation_result with - | Some {Tezos_protocol_environment.context; fitness; message; _} -> - let*! test_chain = Context_ops.get_test_chain context in - let* context = - match test_chain with - | Not_running -> return context - | Running {expiration; _} -> - if Time.Protocol.(expiration <= timestamp) then - let*! context = - Context_ops.add_test_chain context Not_running - in - return context - else return context - | Forking _ -> assert false - in - let* context = - protect - ~on_error:(fun _ -> return context) - (fun () -> - let* predecessor_block_metadata_hash = - Shell_services.Blocks.metadata_hash - cctxt - ~block:(`Hash (pred_info.hash, 0)) - ~chain - () - in - Lwt.map - Result.ok - (Context_ops.add_predecessor_block_metadata_hash - context - predecessor_block_metadata_hash)) - in - let* context = - protect - ~on_error:(fun _ -> return context) - (fun () -> - let* predecessor_ops_metadata_hash = - Shell_services.Blocks.Operation_metadata_hashes.root - cctxt - ~block:(`Hash (pred_info.hash, 0)) - ~chain - () - in - Lwt.map - Result.ok - (Context_ops.add_predecessor_ops_metadata_hash - context - predecessor_ops_metadata_hash)) - in - let context = Context_ops.hash ~time:timestamp ?message context in - (* For the time being, we still fully build the block while we build - confidence to fully unplug the baker validation. The resulting - context hash is ignored as it is not necessary to craft a block. - See: https://gitlab.com/tezos/tezos/-/issues/4285 *) - ignore context ; - return fitness + | Some {Tezos_protocol_environment.fitness; _} -> return fitness | None -> let*? level = Environment.wrap_tzresult @@ Raw_level.of_int32 @@ -261,7 +206,6 @@ let filter_with_context ~chain_id ~fees_config ~hard_gas_limit_per_block ~(pred_info : Baking_state.block_info) ~force_apply ~round ~context_index ~payload_round ~operation_pool cctxt = let open Lwt_result_syntax in - let chain = `Hash chain_id in let* incremental = Baking_simulator.begin_construction ~timestamp @@ -300,15 +244,12 @@ let filter_with_context ~chain_id ~fees_config ~hard_gas_limit_per_block else let* shell_header = finalize_block_header - ~chain ~shell_header:incremental.header - ~timestamp ~validation_result ~operations_hash ~pred_info ~round ~locked_round:None - cctxt in let operations = List.map (List.map convert_operation) operations in let payload_hash = @@ -350,7 +291,6 @@ let apply_with_context ~chain_id ~faked_protocol_data ~user_activated_upgrades ~timestamp ~(pred_info : Baking_state.block_info) ~force_apply ~round ~ordered_pool ~context_index ~payload_hash cctxt = let open Lwt_result_syntax in - let chain = `Hash chain_id in let* incremental = Baking_simulator.begin_construction ~timestamp @@ -414,15 +354,12 @@ let apply_with_context ~chain_id ~faked_protocol_data ~user_activated_upgrades in let* shell_header = finalize_block_header - ~chain ~shell_header:incremental.header - ~timestamp ~validation_result ~operations_hash ~pred_info ~round ~locked_round:locked_round_when_no_validation_result - cctxt in let operations = List.map (List.map convert_operation) operations in return (shell_header, operations, payload_hash) -- GitLab From 6b0aead212f16af2aae29b7399fd0d51fff5f5af Mon Sep 17 00:00:00 2001 From: Albin Coquereau Date: Tue, 24 Jan 2023 17:05:17 +0100 Subject: [PATCH 13/46] lib_delegate: disable signature check on operations already validated in the prevalidator --- .../lib_delegate/baking_simulator.ml | 10 +++++++++- src/proto_alpha/lib_delegate/baking_simulator.ml | 10 +++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_simulator.ml b/src/proto_016_PtMumbai/lib_delegate/baking_simulator.ml index 5ec568002836..5bb1fa77808f 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_simulator.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_simulator.ml @@ -143,7 +143,15 @@ let add_operation st (op : Operation.packed) = let validation_state, application_state = st.state in let oph = Operation.hash_packed op in let** validation_state = - Protocol.validate_operation validation_state oph op + Protocol.validate_operation + ~check_signature:false + (* We assume that the operation has already been validated in the + node, therefore the signature has already been checked, but we + still need to validate it again because the context may be + different. *) + validation_state + oph + op in let** application_state, receipt = match application_state with diff --git a/src/proto_alpha/lib_delegate/baking_simulator.ml b/src/proto_alpha/lib_delegate/baking_simulator.ml index 5ec568002836..5bb1fa77808f 100644 --- a/src/proto_alpha/lib_delegate/baking_simulator.ml +++ b/src/proto_alpha/lib_delegate/baking_simulator.ml @@ -143,7 +143,15 @@ let add_operation st (op : Operation.packed) = let validation_state, application_state = st.state in let oph = Operation.hash_packed op in let** validation_state = - Protocol.validate_operation validation_state oph op + Protocol.validate_operation + ~check_signature:false + (* We assume that the operation has already been validated in the + node, therefore the signature has already been checked, but we + still need to validate it again because the context may be + different. *) + validation_state + oph + op in let** application_state, receipt = match application_state with -- GitLab From 928acd14d5c231420fa91a7711d0de02ddd03095 Mon Sep 17 00:00:00 2001 From: Albin Coquereau Date: Wed, 25 Jan 2023 17:32:19 +0100 Subject: [PATCH 14/46] lib_delegate: add some documentation for operation_selection --- .../lib_delegate/operation_selection.mli | 18 ++++++++++++++++++ .../lib_delegate/operation_selection.mli | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/proto_016_PtMumbai/lib_delegate/operation_selection.mli b/src/proto_016_PtMumbai/lib_delegate/operation_selection.mli index 8844dbed806d..d83fd0dda258 100644 --- a/src/proto_016_PtMumbai/lib_delegate/operation_selection.mli +++ b/src/proto_016_PtMumbai/lib_delegate/operation_selection.mli @@ -34,6 +34,15 @@ type simulation_result = { operations_hash : Operation_list_list_hash.t; } +(** [filter_operations_with_simulation incremental fees_config + ~hard_gas_limit_per_block ops] tries to validate prioritized operations (and + apply them if [incremental] has been initialised with an + [application_state]) and filter them regarding the quota of each validation + pass. Manager operations are prioritized based on a weight computed from + their fees/gas/bytes. [filter_operations_with_simulation] function returns a + [simulation_result], containing the validated operation, their resulting + [operations_hash], optional [validation_result] and [block_header_metadata] + if the operations were applied. *) val filter_operations_with_simulation : Baking_simulator.incremental -> Baking_configuration.fees_config -> @@ -41,12 +50,21 @@ val filter_operations_with_simulation : Operation_pool.Prioritized.t -> simulation_result tzresult Lwt.t +(** [filter_operations_without_simulation fees_config ~hard_gas_limit_per_block + ops] is similar to [filter_operations_with_simulation] but does not validate + (and apply) operations from [ops] and returns only the operations instead of + a [simulation_result]. + + Hypothesis: operations from [ops] have previously been validated. *) val filter_operations_without_simulation : Baking_configuration.fees_config -> hard_gas_limit_per_block:Gas.Arith.integral -> Operation_pool.Prioritized.t -> packed_operation list list +(** [filter_consensus_operations_only incremental ops] is similar to + [filter_operations_with_simulation] but only filters consensus operations + from [ops]. *) val filter_consensus_operations_only : Baking_simulator.incremental -> Operation_pool.ordered_pool -> diff --git a/src/proto_alpha/lib_delegate/operation_selection.mli b/src/proto_alpha/lib_delegate/operation_selection.mli index 8844dbed806d..d83fd0dda258 100644 --- a/src/proto_alpha/lib_delegate/operation_selection.mli +++ b/src/proto_alpha/lib_delegate/operation_selection.mli @@ -34,6 +34,15 @@ type simulation_result = { operations_hash : Operation_list_list_hash.t; } +(** [filter_operations_with_simulation incremental fees_config + ~hard_gas_limit_per_block ops] tries to validate prioritized operations (and + apply them if [incremental] has been initialised with an + [application_state]) and filter them regarding the quota of each validation + pass. Manager operations are prioritized based on a weight computed from + their fees/gas/bytes. [filter_operations_with_simulation] function returns a + [simulation_result], containing the validated operation, their resulting + [operations_hash], optional [validation_result] and [block_header_metadata] + if the operations were applied. *) val filter_operations_with_simulation : Baking_simulator.incremental -> Baking_configuration.fees_config -> @@ -41,12 +50,21 @@ val filter_operations_with_simulation : Operation_pool.Prioritized.t -> simulation_result tzresult Lwt.t +(** [filter_operations_without_simulation fees_config ~hard_gas_limit_per_block + ops] is similar to [filter_operations_with_simulation] but does not validate + (and apply) operations from [ops] and returns only the operations instead of + a [simulation_result]. + + Hypothesis: operations from [ops] have previously been validated. *) val filter_operations_without_simulation : Baking_configuration.fees_config -> hard_gas_limit_per_block:Gas.Arith.integral -> Operation_pool.Prioritized.t -> packed_operation list list +(** [filter_consensus_operations_only incremental ops] is similar to + [filter_operations_with_simulation] but only filters consensus operations + from [ops]. *) val filter_consensus_operations_only : Baking_simulator.incremental -> Operation_pool.ordered_pool -> -- GitLab From ef0601c63cdfe12999fbfe9c4a07057320048141 Mon Sep 17 00:00:00 2001 From: Albin Coquereau Date: Wed, 25 Jan 2023 17:59:52 +0100 Subject: [PATCH 15/46] lib_delegate: add some documentation to baking_simulator --- .../lib_delegate/baking_simulator.mli | 18 ++++++++++++++++++ .../lib_delegate/baking_simulator.mli | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_simulator.mli b/src/proto_016_PtMumbai/lib_delegate/baking_simulator.mli index b05daf4f73da..959546fe6d21 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_simulator.mli +++ b/src/proto_016_PtMumbai/lib_delegate/baking_simulator.mli @@ -41,6 +41,13 @@ val load_context : val check_context_consistency : Abstract_context_index.t -> Context_hash.t -> unit tzresult Lwt.t +(** [begin_construction ~timestamp ~protocol_data ~force_apply abstract_context + predecessor chain_id] creates a new [incremental] value with an empty + operation list. A [context] is recovered from the [abstract_index] and the + resulting_context_hash from [predecessor]. This context is used to create a + [validation_state] and an [application_state] (if [force_apply] is set). A + partial [shell_header] is created from [predecessor] information and + [timestamp]. *) val begin_construction : timestamp:Time.Protocol.t -> protocol_data:block_header_data -> @@ -50,11 +57,22 @@ val begin_construction : Chain_id.t -> incremental tzresult Lwt.t +(** [add_operation incremental op] validates [op] in + [incremental.validation_state] without checking its signature. Indeed, the + operation has already been validated in the node so it has a correct + signature. We still need to validate it again because the context may be + different. [op] is also applied if [incremental] has been created with + [force_apply] set. This function returns an [incremental] with updated + operations list and [validation_state] (and [application_state]). *) val add_operation : incremental -> Operation.packed -> (incremental * operation_receipt option) tzresult Lwt.t +(** [finalize_construction incremental] calls the [finalize_validation] of the + protocol on the [validation_state] from [incremental]. If [incremental] has + been created with [force_apply] set, [finalize_application] is also called + and its results returned. *) val finalize_construction : incremental -> (Tezos_protocol_environment.validation_result * block_header_metadata) option diff --git a/src/proto_alpha/lib_delegate/baking_simulator.mli b/src/proto_alpha/lib_delegate/baking_simulator.mli index b05daf4f73da..959546fe6d21 100644 --- a/src/proto_alpha/lib_delegate/baking_simulator.mli +++ b/src/proto_alpha/lib_delegate/baking_simulator.mli @@ -41,6 +41,13 @@ val load_context : val check_context_consistency : Abstract_context_index.t -> Context_hash.t -> unit tzresult Lwt.t +(** [begin_construction ~timestamp ~protocol_data ~force_apply abstract_context + predecessor chain_id] creates a new [incremental] value with an empty + operation list. A [context] is recovered from the [abstract_index] and the + resulting_context_hash from [predecessor]. This context is used to create a + [validation_state] and an [application_state] (if [force_apply] is set). A + partial [shell_header] is created from [predecessor] information and + [timestamp]. *) val begin_construction : timestamp:Time.Protocol.t -> protocol_data:block_header_data -> @@ -50,11 +57,22 @@ val begin_construction : Chain_id.t -> incremental tzresult Lwt.t +(** [add_operation incremental op] validates [op] in + [incremental.validation_state] without checking its signature. Indeed, the + operation has already been validated in the node so it has a correct + signature. We still need to validate it again because the context may be + different. [op] is also applied if [incremental] has been created with + [force_apply] set. This function returns an [incremental] with updated + operations list and [validation_state] (and [application_state]). *) val add_operation : incremental -> Operation.packed -> (incremental * operation_receipt option) tzresult Lwt.t +(** [finalize_construction incremental] calls the [finalize_validation] of the + protocol on the [validation_state] from [incremental]. If [incremental] has + been created with [force_apply] set, [finalize_application] is also called + and its results returned. *) val finalize_construction : incremental -> (Tezos_protocol_environment.validation_result * block_header_metadata) option -- GitLab From c9ea712e8d652c67cb5b8e5e2d97dec9fc2470ca Mon Sep 17 00:00:00 2001 From: Albin Coquereau Date: Thu, 26 Jan 2023 17:15:05 +0100 Subject: [PATCH 16/46] tezt/lib_tezos: add support for force-apply baker option --- tezt/lib_tezos/baker.ml | 13 ++++++++++--- tezt/lib_tezos/baker.mli | 13 ++++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/tezt/lib_tezos/baker.ml b/tezt/lib_tezos/baker.ml index 5197e0679dfe..d5982799b242 100644 --- a/tezt/lib_tezos/baker.ml +++ b/tezt/lib_tezos/baker.ml @@ -46,6 +46,7 @@ module Parameters = struct mutable pending_ready : unit option Lwt.u list; votefile : string option; liquidity_baking_toggle_vote : liquidity_baking_vote option; + force_apply : bool; } type session_state = {mutable ready : bool} @@ -88,7 +89,8 @@ let liquidity_baking_votefile ?path vote = votefile let create ~protocol ?name ?color ?event_pipe ?runner ?(delegates = []) - ?votefile ?(liquidity_baking_toggle_vote = Some Pass) node client = + ?votefile ?(liquidity_baking_toggle_vote = Some Pass) ?(force_apply = false) + node client = let baker = create ~path:(Protocol.baker protocol) @@ -105,6 +107,7 @@ let create ~protocol ?name ?color ?event_pipe ?runner ?(delegates = []) pending_ready = []; votefile; liquidity_baking_toggle_vote; + force_apply; } in on_stdout baker (handle_raw_stdout baker) ; @@ -130,6 +133,9 @@ let run ?event_level (baker : t) = liquidity_baking_vote_to_string baker.persistent_state.liquidity_baking_toggle_vote in + let force_apply = + Cli_arg.optional_switch "force-apply" baker.persistent_state.force_apply + in let arguments = [ "--endpoint"; @@ -142,7 +148,7 @@ let run ?event_level (baker : t) = "node"; Node.data_dir node; ] - @ liquidity_baking_toggle_vote @ votefile @ delegates + @ liquidity_baking_toggle_vote @ votefile @ force_apply @ delegates in let on_terminate _ = (* Cancel all [Ready] event listeners. *) @@ -168,7 +174,7 @@ let wait_for_ready baker = check_event baker "Baker started." promise let init ~protocol ?name ?color ?event_pipe ?runner ?(delegates = []) ?votefile - ?liquidity_baking_toggle_vote node client = + ?liquidity_baking_toggle_vote ?force_apply node client = let* () = Node.wait_for_ready node in let baker = create @@ -179,6 +185,7 @@ let init ~protocol ?name ?color ?event_pipe ?runner ?(delegates = []) ?votefile ?runner ?votefile ?liquidity_baking_toggle_vote + ?force_apply ~delegates node client diff --git a/tezt/lib_tezos/baker.mli b/tezt/lib_tezos/baker.mli index fd02cfd68e2f..c04ab16ab2a9 100644 --- a/tezt/lib_tezos/baker.mli +++ b/tezt/lib_tezos/baker.mli @@ -113,7 +113,10 @@ val liquidity_baking_votefile : ?path:string -> liquidity_baking_vote -> string [--liquidity-baking-toggle-vote] is [None], then [--liquidity-baking-toggle-vote] is not passed. If it is [Some x] then [--liquidity-baking-toggle-vote x] is - passed. The default value is [Some Pass]. *) + passed. The default value is [Some Pass]. + + [force_apply] is passed to the baker daemon through + the flag [--force_apply]. *) val create : protocol:Protocol.t -> ?name:string -> @@ -123,6 +126,7 @@ val create : ?delegates:string list -> ?votefile:string -> ?liquidity_baking_toggle_vote:liquidity_baking_vote option -> + ?force_apply:bool -> Node.t -> Client.t -> t @@ -157,8 +161,10 @@ val create : baker. This defaults to the empty list, which is a shortcut for "every known account". - [votefile] and [liquidity_baking_toggle_vote] are passed to the baker daemon - through the flags [--votefile] and [--liquidity-baking-toggle-vote]. *) + [votefile], [liquidity_baking_toggle_vote] and [force_apply] are + respectively passed to the baker daemon through the flags + [--votefile], [--liquidity-baking-toggle-vote] and + [--should-apply]. *) val init : protocol:Protocol.t -> ?name:string -> @@ -168,6 +174,7 @@ val init : ?delegates:string list -> ?votefile:string -> ?liquidity_baking_toggle_vote:liquidity_baking_vote option -> + ?force_apply:bool -> Node.t -> Client.t -> t Lwt.t -- GitLab From 12dc0f8ed69cb62e70ccc98461abe7d0220fdfaf Mon Sep 17 00:00:00 2001 From: Albin Coquereau Date: Thu, 26 Jan 2023 17:16:08 +0100 Subject: [PATCH 17/46] tezt/lib_tezos: add baker test that force apply operation on alpha --- tezt/tests/baker_test.ml | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/tezt/tests/baker_test.ml b/tezt/tests/baker_test.ml index 63c01c506d9d..a2d2f8a19418 100644 --- a/tezt/tests/baker_test.ml +++ b/tezt/tests/baker_test.ml @@ -30,7 +30,7 @@ Subject: Run the baker while performing a lot of transfers *) -let baker_test protocol ~keys = +let baker_test ?force_apply protocol ~keys = let* parameter_file = Protocol.write_parameter_file ~bootstrap_accounts:(List.map (fun k -> (k, None)) keys) @@ -48,7 +48,7 @@ let baker_test protocol ~keys = in let level_2_promise = Node.wait_for_level node 2 in let level_3_promise = Node.wait_for_level node 3 in - let* baker = Baker.init ~protocol node client in + let* baker = Baker.init ?force_apply ~protocol node client in Log.info "Wait for new head." ; Baker.log_events baker ; let* _ = level_2_promise in @@ -80,6 +80,23 @@ let baker_stresstest = let* () = Client.stresstest ~tps:25 ~transfers:100 client in Lwt.return_unit +(* Force the baker to apply operations after validating them *) +let baker_stresstest_apply = + Protocol.register_test + ~__FILE__ + ~supports:Protocol.(From_protocol (number Mumbai)) + ~title:"baker stresstest with forced application" + ~tags:["node"; "baker"; "stresstest"; "apply"] + @@ fun protocol -> + let* node, client = + Client.init_with_protocol `Client ~protocol () ~timestamp:Now + in + let* _ = Baker.init ~force_apply:true ~protocol node client in + let* _ = Node.wait_for_level node 3 in + (* Use a large tps, to have failing operations too *) + let* () = Client.stresstest ~tps:25 ~transfers:100 client in + unit + let baker_bls_test = Protocol.register_test ~__FILE__ @@ -124,4 +141,5 @@ let baker_bls_test = let register ~protocols = baker_simple_test protocols ; baker_stresstest protocols ; + baker_stresstest_apply protocols ; baker_bls_test protocols -- GitLab From c3911659d09ecc391b77337c922ddc52b8dd2b27 Mon Sep 17 00:00:00 2001 From: Albin Coquereau Date: Wed, 25 Jan 2023 14:16:10 +0100 Subject: [PATCH 18/46] Changes: add entries for the baker modifications --- CHANGES.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 06e68eaf4847..e5065aa150aa 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -168,6 +168,21 @@ Baker necessary causing unexpected out of space errors making the baker crash. +- Changed the baker default semantics so that it performs a light + validation of operations to classify them instead of fully applying + them. Hence, the block production is now more + time/cpu/disk-efficient. In this mode, application-dependent checks + are disabled. Setting the ``--force-apply`` flag on the command line + restores the previous behavior. (MR :gl:`!7490`) + +- **Breaking Change**: Disabled the verification of signature of + operations in the baker when baking a block. The baker must always + be provided operations with a valid signature, otherwise produced + blocks will be invalid and rejected by local nodes during their + injection. Default setups are not affected but external mempools + should make sure that their operations' signatures are correct. + (MR :gl:`!7490`) + Accuser ------- -- GitLab From b2d16b87a3e3808e2b4277e12559e96f6a7358a4 Mon Sep 17 00:00:00 2001 From: Albin Coquereau Date: Thu, 12 Jan 2023 17:10:38 +0100 Subject: [PATCH 19/46] lib_store: rename prechecked into validated --- src/lib_shell/block_validator_process.ml | 8 ++-- src/lib_shell/distributed_db.ml | 2 +- src/lib_shell/distributed_db_requester.ml | 6 +-- src/lib_shell/p2p_reader.ml | 2 +- .../block_validator_errors.ml | 14 +++---- .../block_validator_errors.mli | 2 +- src/lib_store/mocked/store.ml | 34 ++++++++--------- src/lib_store/shared/store_events.ml | 6 +-- src/lib_store/store.mli | 29 +++++++------- src/lib_store/unix/store.ml | 38 +++++++++---------- src/lib_store/unix/store.mli | 29 +++++++------- 11 files changed, 84 insertions(+), 86 deletions(-) diff --git a/src/lib_shell/block_validator_process.ml b/src/lib_shell/block_validator_process.ml index 82aab9f4861b..c96f1f9e7388 100644 --- a/src/lib_shell/block_validator_process.ml +++ b/src/lib_shell/block_validator_process.ml @@ -1014,12 +1014,12 @@ let apply_block ?(simulate = false) ?(should_precheck = true) let block_hash = Block_header.hash header in let* () = when_ (not should_precheck) (fun () -> - let*! is_prechecked = - Store.Block.is_known_prechecked chain_store block_hash + let*! is_validated = + Store.Block.is_known_validated chain_store block_hash in fail_unless - is_prechecked - (Block_validator_errors.Applying_non_prechecked_block block_hash)) + is_validated + (Block_validator_errors.Applying_non_validated_block block_hash)) in let* metadata = Store.Block.get_block_metadata chain_store predecessor in let max_operations_ttl = Store.Block.max_operations_ttl metadata in diff --git a/src/lib_shell/distributed_db.ml b/src/lib_shell/distributed_db.ml index 7ca276f20c0a..29d926470e0b 100644 --- a/src/lib_shell/distributed_db.ml +++ b/src/lib_shell/distributed_db.ml @@ -291,7 +291,7 @@ let inject_operation chain_db h op = op let inject_prechecked_block chain_db hash block_header operations = - Store.Block.store_prechecked_block + Store.Block.store_validated_block chain_db.reader_chain_db.chain_store ~hash ~block_header diff --git a/src/lib_shell/distributed_db_requester.ml b/src/lib_shell/distributed_db_requester.ml index b62ad7ba3d2e..db45b4590f96 100644 --- a/src/lib_shell/distributed_db_requester.ml +++ b/src/lib_shell/distributed_db_requester.ml @@ -207,7 +207,7 @@ module Block_header_storage = struct let* b = Store.Block.is_known_valid chain_store hash in match b with | true -> Lwt.return_true - | false -> Store.Block.is_known_prechecked chain_store hash + | false -> Store.Block.is_known_validated chain_store hash let read chain_store h = let open Lwt_result_syntax in @@ -215,7 +215,7 @@ module Block_header_storage = struct let*! r = Store.Block.read_block chain_store h in match r with | Ok b -> return b - | Error _ -> Store.Block.read_prechecked_block chain_store h + | Error _ -> Store.Block.read_validated_block chain_store h in return (Store.Block.header b) @@ -225,7 +225,7 @@ module Block_header_storage = struct let* o = Store.Block.read_block_opt chain_store h in match o with | Some b -> Lwt.return_some b - | None -> Store.Block.read_prechecked_block_opt chain_store h + | None -> Store.Block.read_validated_block_opt chain_store h in Lwt.return (Option.map Store.Block.header b) end diff --git a/src/lib_shell/p2p_reader.ml b/src/lib_shell/p2p_reader.ml index 945db5b63dcb..b30090736b7a 100644 --- a/src/lib_shell/p2p_reader.ml +++ b/src/lib_shell/p2p_reader.ml @@ -120,7 +120,7 @@ let read_block {disk; _} h = let* o = Store.Block.read_block_opt chain_store h in let* o = match o with - | None -> Store.Block.read_prechecked_block_opt chain_store h + | None -> Store.Block.read_validated_block_opt chain_store h | Some b -> Lwt.return_some b in Option.map_s (fun b -> Lwt.return (Store.Chain.chain_id chain_store, b)) o) diff --git a/src/lib_shell_services/block_validator_errors.ml b/src/lib_shell_services/block_validator_errors.ml index 4a0731054cef..136a94aa5533 100644 --- a/src/lib_shell_services/block_validator_errors.ml +++ b/src/lib_shell_services/block_validator_errors.ml @@ -405,7 +405,7 @@ type error += expected : Operation_list_list_hash.t; found : Operation_list_list_hash.t; } - | Applying_non_prechecked_block of Block_hash.t + | Applying_non_validated_block of Block_hash.t | Failed_to_checkout_context of Context_hash.t | System_error of {errno : string; fn : string; msg : string} | Missing_test_protocol of Protocol_hash.t @@ -483,18 +483,18 @@ let () = Inconsistent_operations_hash {block; expected; found}) ; Error_monad.register_error_kind `Permanent - ~id:"Block_validator_process.applying_non_prechecked_block" - ~title:"Applying non prechecked block" - ~description:"Applying non prechecked block" + ~id:"Block_validator_process.applying_non_validated_block" + ~title:"Applying non validated block" + ~description:"Applying non validated block" ~pp:(fun ppf (hash : Block_hash.t) -> Format.fprintf ppf - "Applying non prechecked block %a" + "Applying non validated block %a" Block_hash.pp_short hash) Data_encoding.(obj1 (req "hash" Block_hash.encoding)) - (function Applying_non_prechecked_block bh -> Some bh | _ -> None) - (fun bh -> Applying_non_prechecked_block bh) ; + (function Applying_non_validated_block bh -> Some bh | _ -> None) + (fun bh -> Applying_non_validated_block bh) ; Error_monad.register_error_kind `Permanent ~id:"Block_validator_process.failed_to_checkout_context" diff --git a/src/lib_shell_services/block_validator_errors.mli b/src/lib_shell_services/block_validator_errors.mli index 1a5c333cc901..104f9d80ce17 100644 --- a/src/lib_shell_services/block_validator_errors.mli +++ b/src/lib_shell_services/block_validator_errors.mli @@ -70,7 +70,7 @@ type error += expected : Operation_list_list_hash.t; found : Operation_list_list_hash.t; } - | Applying_non_prechecked_block of Block_hash.t + | Applying_non_validated_block of Block_hash.t | Failed_to_checkout_context of Context_hash.t | System_error of {errno : string; fn : string; msg : string} | Missing_test_protocol of Protocol_hash.t diff --git a/src/lib_store/mocked/store.ml b/src/lib_store/mocked/store.ml index 7b340f645436..90fca8515e51 100644 --- a/src/lib_store/mocked/store.ml +++ b/src/lib_store/mocked/store.ml @@ -99,7 +99,7 @@ and chain_state = { live_operations : Operation_hash.Set.t; mutable live_data_cache : (Block_hash.t * Operation_hash.Set.t) Ringo.Ring.t option; - prechecked_blocks : Block_repr.t Block_lru_cache.t; + validated_blocks : Block_repr.t Block_lru_cache.t; } and testchain = {forked_block : Block_hash.t; testchain_store : chain_store} @@ -215,10 +215,10 @@ module Block = struct Shared.use chain_state (fun chain_state -> locked_is_known_invalid chain_state hash) - let is_known_prechecked {chain_state; _} hash = - Shared.use chain_state (fun {prechecked_blocks; _} -> + let is_known_validated {chain_state; _} hash = + Shared.use chain_state (fun {validated_blocks; _} -> Option.value ~default:Lwt.return_false - @@ Block_lru_cache.bind prechecked_blocks hash (function + @@ Block_lru_cache.bind validated_blocks hash (function | None -> Lwt.return_false | Some _ -> Lwt.return_true)) @@ -350,14 +350,14 @@ module Block = struct let* current_head = current_head chain_store in locked_read_block_by_level_opt chain_store current_head level - let read_prechecked_block_opt {chain_state; _} hash = - Shared.use chain_state (fun {prechecked_blocks; _} -> + let read_validated_block_opt {chain_state; _} hash = + Shared.use chain_state (fun {validated_blocks; _} -> Option.value ~default:Lwt.return_none - @@ Block_lru_cache.bind prechecked_blocks hash Lwt.return) + @@ Block_lru_cache.bind validated_blocks hash Lwt.return) - let read_prechecked_block chain_store hash = + let read_validated_block chain_store hash = let open Lwt_result_syntax in - let*! o = read_prechecked_block_opt chain_store hash in + let*! o = read_validated_block_opt chain_store hash in match o with | Some b -> return b | None -> tzfail (Block_not_found {hash; distance = 0}) @@ -527,8 +527,8 @@ module Block = struct Store_events.(emit store_block) (hash, block_header.shell.level) in let*! () = - Shared.use chain_store.chain_state (fun {prechecked_blocks; _} -> - Block_lru_cache.remove prechecked_blocks hash ; + Shared.use chain_store.chain_state (fun {validated_blocks; _} -> + Block_lru_cache.remove validated_blocks hash ; Lwt.return_unit) in Lwt_watcher.notify chain_store.block_watcher block ; @@ -537,7 +537,7 @@ module Block = struct (chain_store, block) ; return_some block - let store_prechecked_block chain_store ~hash ~block_header ~operations = + let store_validated_block chain_store ~hash ~block_header ~operations = let open Lwt_result_syntax in let operations_length = List.length operations in let validation_passes = block_header.Block_header.shell.validation_passes in @@ -563,12 +563,12 @@ module Block = struct } in let*! () = - Shared.use chain_store.chain_state (fun {prechecked_blocks; _} -> - Block_lru_cache.put prechecked_blocks hash (Lwt.return_some block) ; + Shared.use chain_store.chain_state (fun {validated_blocks; _} -> + Block_lru_cache.put validated_blocks hash (Lwt.return_some block) ; Lwt.return_unit) in let*! () = - Store_events.(emit store_prechecked_block) (hash, block_header.shell.level) + Store_events.(emit store_validated_block) (hash, block_header.shell.level) in return_unit @@ -1422,7 +1422,7 @@ module Chain = struct let live_blocks = Block_hash.Set.singleton genesis_block.hash in let live_operations = Operation_hash.Set.empty in let live_data_cache = None in - let prechecked_blocks = Block_lru_cache.create 10 in + let validated_blocks = Block_lru_cache.create 10 in return { current_head_data; @@ -1438,7 +1438,7 @@ module Chain = struct live_blocks; live_operations; live_data_cache; - prechecked_blocks; + validated_blocks; } let create_chain_store ?block_cache_limit global_store chain_dir ?target diff --git a/src/lib_store/shared/store_events.ml b/src/lib_store/shared/store_events.ml index 0aa9acc9af97..525c040c8b6e 100644 --- a/src/lib_store/shared/store_events.ml +++ b/src/lib_store/shared/store_events.ml @@ -83,12 +83,12 @@ let store_block = ~pp1:pp_block_descriptor ("block", block_descriptor_encoding) -let store_prechecked_block = +let store_validated_block = declare_1 ~section ~level:Info - ~name:"store_prechecked_block" - ~msg:"prechecked block {block} was stored" + ~name:"store_validated_block" + ~msg:"validated block {block} was stored" ~pp1:pp_block_descriptor ("block", block_descriptor_encoding) diff --git a/src/lib_store/store.mli b/src/lib_store/store.mli index 5dd0d26b469d..9e98d883a1e1 100644 --- a/src/lib_store/store.mli +++ b/src/lib_store/store.mli @@ -310,10 +310,10 @@ module Block : sig invalid blocks file). *) val is_known_invalid : chain_store -> Block_hash.t -> bool Lwt.t - (** [is_known_prechecked chain_store bh] tests that the block [bh] - is prechecked in [chain_store] (i.e. the block is present in the - prechecked block cache). *) - val is_known_prechecked : chain_store -> Block_hash.t -> bool Lwt.t + (** [is_known_validated chain_store bh] tests that the block [bh] + is validated in [chain_store] (i.e. the block is present in the + validated block cache). *) + val is_known_validated : chain_store -> Block_hash.t -> bool Lwt.t (** [is_known chain_store bh] tests that the block [bh] is either known valid or known invalid in [chain_store]. *) @@ -411,14 +411,13 @@ module Block : sig val read_predecessor_of_hash_opt : chain_store -> Block_hash.t -> block option Lwt.t - (** [read_prechecked_block chain_store bh] tries to read in the - [chain_store]'s prechecked block cache the block [bh].*) - val read_prechecked_block : - chain_store -> Block_hash.t -> block tzresult Lwt.t + (** [read_validated_block chain_store bh] tries to read in the + [chain_store]'s validated block cache the block [bh].*) + val read_validated_block : chain_store -> Block_hash.t -> block tzresult Lwt.t - (** [read_prechecked_block_opt chain_store bh] optional version of - [read_prechecked_block].*) - val read_prechecked_block_opt : + (** [read_validated_block_opt chain_store bh] optional version of + [read_validated_block].*) + val read_validated_block_opt : chain_store -> Block_hash.t -> block option Lwt.t (** [store_block chain_store ~block_header ~operations @@ -429,7 +428,7 @@ module Block : sig the newly created block is returned. If the block was successfully stored, then the block is removed - from the prechecked block cache. + from the validated block cache. {b Warning} The store will refuse to store blocks with no associated context's commit. *) @@ -440,10 +439,10 @@ module Block : sig Block_validation.result -> block option tzresult Lwt.t - (** [store_prechecked_block chain_store ~hash ~block_header ~operations] - stores in [chain_store]'s prechecked block cache the block with + (** [store_validated_block chain_store ~hash ~block_header ~operations] + stores in [chain_store]'s validated block cache the block with its [block_header] and [operations]. *) - val store_prechecked_block : + val store_validated_block : chain_store -> hash:Block_hash.t -> block_header:Block_header.t -> diff --git a/src/lib_store/unix/store.ml b/src/lib_store/unix/store.ml index f4d79e2d7688..e7cbdc8a8d52 100644 --- a/src/lib_store/unix/store.ml +++ b/src/lib_store/unix/store.ml @@ -103,7 +103,7 @@ and chain_state = { live_operations : Operation_hash.Set.t; mutable live_data_cache : (Block_hash.t * Operation_hash.Set.t) Ringo.Ring.t option; - prechecked_blocks : Block_repr.t Block_lru_cache.t; + validated_blocks : Block_repr.t Block_lru_cache.t; } and testchain = {forked_block : Block_hash.t; testchain_store : chain_store} @@ -268,10 +268,10 @@ module Block = struct Shared.use chain_state (fun chain_state -> locked_is_known_invalid chain_state hash) - let is_known_prechecked {chain_state; _} hash = - Shared.use chain_state (fun {prechecked_blocks; _} -> + let is_known_validated {chain_state; _} hash = + Shared.use chain_state (fun {validated_blocks; _} -> Option.value ~default:Lwt.return_false - @@ Block_lru_cache.bind prechecked_blocks hash (function + @@ Block_lru_cache.bind validated_blocks hash (function | None -> Lwt.return_false | Some _ -> Lwt.return_true)) @@ -403,14 +403,14 @@ module Block = struct let* current_head = current_head chain_store in locked_read_block_by_level_opt chain_store current_head level - let read_prechecked_block_opt {chain_state; _} hash = - Shared.use chain_state (fun {prechecked_blocks; _} -> + let read_validated_block_opt {chain_state; _} hash = + Shared.use chain_state (fun {validated_blocks; _} -> Option.value ~default:Lwt.return_none - @@ Block_lru_cache.bind prechecked_blocks hash Lwt.return) + @@ Block_lru_cache.bind validated_blocks hash Lwt.return) - let read_prechecked_block chain_store hash = + let read_validated_block chain_store hash = let open Lwt_result_syntax in - let*! o = read_prechecked_block_opt chain_store hash in + let*! o = read_validated_block_opt chain_store hash in match o with | Some b -> return b | None -> tzfail (Block_not_found {hash; distance = 0}) @@ -580,8 +580,8 @@ module Block = struct Store_events.(emit store_block) (hash, block_header.shell.level) in let*! () = - Shared.use chain_store.chain_state (fun {prechecked_blocks; _} -> - Block_lru_cache.remove prechecked_blocks hash ; + Shared.use chain_store.chain_state (fun {validated_blocks; _} -> + Block_lru_cache.remove validated_blocks hash ; Lwt.return_unit) in Lwt_watcher.notify chain_store.block_watcher block ; @@ -590,7 +590,7 @@ module Block = struct (chain_store, block) ; return_some block - let store_prechecked_block chain_store ~hash ~block_header ~operations = + let store_validated_block chain_store ~hash ~block_header ~operations = let open Lwt_result_syntax in let operations_length = List.length operations in let validation_passes = block_header.Block_header.shell.validation_passes in @@ -616,12 +616,12 @@ module Block = struct } in let*! () = - Shared.use chain_store.chain_state (fun {prechecked_blocks; _} -> - Block_lru_cache.put prechecked_blocks hash (Lwt.return_some block) ; + Shared.use chain_store.chain_state (fun {validated_blocks; _} -> + Block_lru_cache.put validated_blocks hash (Lwt.return_some block) ; Lwt.return_unit) in let*! () = - Store_events.(emit store_prechecked_block) (hash, block_header.shell.level) + Store_events.(emit store_validated_block) (hash, block_header.shell.level) in return_unit @@ -1768,7 +1768,7 @@ module Chain = struct let live_blocks = Block_hash.Set.singleton genesis_block.hash in let live_operations = Operation_hash.Set.empty in let live_data_cache = None in - let prechecked_blocks = Block_lru_cache.create 10 in + let validated_blocks = Block_lru_cache.create 10 in return { current_head_data; @@ -1784,7 +1784,7 @@ module Chain = struct live_blocks; live_operations; live_data_cache; - prechecked_blocks; + validated_blocks; } (* In some case, when a merge was interrupted, the highest cemented @@ -1864,7 +1864,7 @@ module Chain = struct let live_blocks = Block_hash.Set.empty in let live_operations = Operation_hash.Set.empty in let live_data_cache = None in - let prechecked_blocks = Block_lru_cache.create 10 in + let validated_blocks = Block_lru_cache.create 10 in return { current_head_data; @@ -1880,7 +1880,7 @@ module Chain = struct live_blocks; live_operations; live_data_cache; - prechecked_blocks; + validated_blocks; } let create_chain_store ?block_cache_limit global_store chain_dir ?target diff --git a/src/lib_store/unix/store.mli b/src/lib_store/unix/store.mli index 9088771a1075..952ea4476e52 100644 --- a/src/lib_store/unix/store.mli +++ b/src/lib_store/unix/store.mli @@ -309,10 +309,10 @@ module Block : sig invalid blocks file). *) val is_known_invalid : chain_store -> Block_hash.t -> bool Lwt.t - (** [is_known_prechecked chain_store bh] tests that the block [bh] - is prechecked in [chain_store] (i.e. the block is present in the - prechecked block cache). *) - val is_known_prechecked : chain_store -> Block_hash.t -> bool Lwt.t + (** [is_known_validated chain_store bh] tests that the block [bh] + is validated in [chain_store] (i.e. the block is present in the + validated block cache). *) + val is_known_validated : chain_store -> Block_hash.t -> bool Lwt.t (** [is_known chain_store bh] tests that the block [bh] is either known valid or known invalid in [chain_store]. *) @@ -410,14 +410,13 @@ module Block : sig val read_predecessor_of_hash_opt : chain_store -> Block_hash.t -> block option Lwt.t - (** [read_prechecked_block chain_store bh] tries to read in the - [chain_store]'s prechecked block cache the block [bh].*) - val read_prechecked_block : - chain_store -> Block_hash.t -> block tzresult Lwt.t + (** [read_validated_block chain_store bh] tries to read in the + [chain_store]'s validated block cache the block [bh].*) + val read_validated_block : chain_store -> Block_hash.t -> block tzresult Lwt.t - (** [read_prechecked_block_opt chain_store bh] optional version of - [read_prechecked_block].*) - val read_prechecked_block_opt : + (** [read_validated_block_opt chain_store bh] optional version of + [read_validated_block].*) + val read_validated_block_opt : chain_store -> Block_hash.t -> block option Lwt.t (** [store_block chain_store ~block_header ~operations @@ -428,7 +427,7 @@ module Block : sig the newly created block is returned. If the block was successfully stored, then the block is removed - from the prechecked block cache. *) + from the validated block cache. *) val store_block : chain_store -> block_header:Block_header.t -> @@ -436,10 +435,10 @@ module Block : sig Block_validation.result -> block option tzresult Lwt.t - (** [store_prechecked_block chain_store ~hash ~block_header ~operations] - stores in [chain_store]'s prechecked block cache the block with + (** [store_validated_block chain_store ~hash ~block_header ~operations] + stores in [chain_store]'s validated block cache the block with its [block_header] and [operations]. *) - val store_prechecked_block : + val store_validated_block : chain_store -> hash:Block_hash.t -> block_header:Block_header.t -> -- GitLab From 0a7b359cf871a1961a2f190344accc52aa84bb68 Mon Sep 17 00:00:00 2001 From: vbot Date: Thu, 26 Jan 2023 13:50:40 +0100 Subject: [PATCH 20/46] Shell: deprecate '/monitor/valid_blocks' and expose 'applied_blocks' --- src/lib_shell/monitor_directory.ml | 59 ++++++++++++++++++- src/lib_shell_services/monitor_services.ml | 43 +++++++++++--- src/lib_shell_services/monitor_services.mli | 37 +++++++++++- .../lib_delegate/baking_vdf.ml | 2 +- .../lib_delegate/client_baking_blocks.ml | 6 +- .../lib_delegate/client_baking_blocks.mli | 2 +- .../lib_delegate/client_daemon.ml | 2 +- .../lib_delegate/baking_vdf.ml | 2 +- .../lib_delegate/client_baking_blocks.ml | 6 +- .../lib_delegate/client_baking_blocks.mli | 2 +- .../lib_delegate/client_daemon.ml | 2 +- src/proto_alpha/lib_delegate/baking_vdf.ml | 2 +- .../lib_delegate/client_baking_blocks.ml | 6 +- .../lib_delegate/client_baking_blocks.mli | 2 +- src/proto_alpha/lib_delegate/client_daemon.ml | 2 +- 15 files changed, 147 insertions(+), 28 deletions(-) diff --git a/src/lib_shell/monitor_directory.ml b/src/lib_shell/monitor_directory.ml index 1e410538b420..02f0737cc726 100644 --- a/src/lib_shell/monitor_directory.ml +++ b/src/lib_shell/monitor_directory.ml @@ -61,7 +61,7 @@ let build_rpc_directory validator mainchain_validator = in let shutdown () = Lwt_watcher.shutdown stopper in Tezos_rpc.Answer.return_stream {next; shutdown}) ; - gen_register0 Monitor_services.S.valid_blocks (fun q () -> + gen_register0 Monitor_services.S.legacy_valid_blocks (fun q () -> let block_stream, stopper = Store.global_block_watcher store in let shutdown () = Lwt_watcher.shutdown stopper in let in_chains (chain_store, _block) = @@ -115,6 +115,63 @@ let build_rpc_directory validator mainchain_validator = in let next () = Lwt_stream.get stream in Tezos_rpc.Answer.return_stream {next; shutdown}) ; + gen_register0 Monitor_services.S.applied_blocks (fun q () -> + let block_stream, stopper = Store.global_block_watcher store in + let shutdown () = Lwt_watcher.shutdown stopper in + let in_chains (chain_store, _block) = + match q#chains with + | [] -> Lwt.return_true + | chains -> + let that_chain_id = Store.Chain.chain_id chain_store in + List.exists_p + (fun chain -> + let+ o = Chain_directory.get_chain_id_opt store chain in + match o with + | None -> false + | Some this_chain_id -> + Chain_id.equal this_chain_id that_chain_id) + chains + in + let in_protocols (chain_store, block) = + match q#protocols with + | [] -> Lwt.return_true + | protocols -> ( + let* o = Store.Block.read_predecessor_opt chain_store block in + match o with + | None -> Lwt.return_false (* won't happen *) + | Some pred -> + let* context = Store.Block.context_exn chain_store pred in + let* protocol = Context_ops.get_protocol context in + Lwt.return + (List.exists (Protocol_hash.equal protocol) protocols)) + in + let in_next_protocols (chain_store, block) = + match q#next_protocols with + | [] -> Lwt.return_true + | protocols -> + let* context = Store.Block.context_exn chain_store block in + let* next_protocol = Context_ops.get_protocol context in + Lwt.return + (List.exists (Protocol_hash.equal next_protocol) protocols) + in + let stream = + Lwt_stream.filter_map_s + (fun ((chain_store, block) as elt) -> + let* in_chains = in_chains elt in + let* in_next_protocols = in_next_protocols elt in + let* in_protocols = in_protocols elt in + if in_chains && in_protocols && in_next_protocols then + let chain_id = Store.Chain.chain_id chain_store in + Lwt.return_some + ( chain_id, + Store.Block.hash block, + Store.Block.header block, + Store.Block.operations block ) + else Lwt.return_none) + block_stream + in + let next () = Lwt_stream.get stream in + Tezos_rpc.Answer.return_stream {next; shutdown}) ; gen_register1 Monitor_services.S.heads (fun chain q () -> (* TODO: when `chain = `Test`, should we reset then stream when the `testnet` change, or dias we currently do ?? *) diff --git a/src/lib_shell_services/monitor_services.ml b/src/lib_shell_services/monitor_services.ml index f409c78c3b52..2664d385c405 100644 --- a/src/lib_shell_services/monitor_services.ml +++ b/src/lib_shell_services/monitor_services.ml @@ -84,7 +84,7 @@ module S = struct (req "timestamp" Time.Protocol.encoding)) Tezos_rpc.Path.(path / "bootstrapped") - let valid_blocks_query = + let validated_or_apply_blocks_query = let open Tezos_rpc.Query in query (fun protocols next_protocols chains -> object @@ -100,12 +100,12 @@ module S = struct |+ multi_field "chain" Chain_services.chain_arg (fun t -> t#chains) |> seal - let valid_blocks = + let legacy_valid_blocks = Tezos_rpc.Service.get_service ~description: - "Monitor all blocks that are successfully validated by the node, \ - disregarding whether they were selected as the new head or not." - ~query:valid_blocks_query + "(Deprecated) Monitor all blocks that are successfully applied by the \ + node, disregarding whether they were selected as the new head or not." + ~query:validated_or_apply_blocks_query ~output: (merge_objs (obj2 @@ -114,6 +114,20 @@ module S = struct Block_header.encoding) Tezos_rpc.Path.(path / "valid_blocks") + let applied_blocks = + Tezos_rpc.Service.get_service + ~description: + "Monitor all blocks that are successfully applied and stored by the \ + node, disregarding whether they were selected as the new head or not." + ~query:evaluated_blocks_query + ~output: + (obj4 + (req "chain_id" Chain_id.encoding) + (req "hash" Block_hash.encoding) + (req "header" (dynamic_size Block_header.encoding)) + (req "operations" (list (list (dynamic_size Operation.encoding))))) + Tezos_rpc.Path.(path / "applied_blocks") + let heads_query = let open Tezos_rpc.Query in query (fun next_protocols -> @@ -167,10 +181,25 @@ open Tezos_rpc.Context let bootstrapped ctxt = make_streamed_call S.bootstrapped ctxt () () () -let valid_blocks ctxt ?(chains = [`Main]) ?(protocols = []) +let legacy_valid_blocks ctxt ?(chains = [`Main]) ?(protocols = []) + ?(next_protocols = []) () = + make_streamed_call + S.legacy_valid_blocks + ctxt + () + (object + method chains = chains + + method protocols = protocols + + method next_protocols = next_protocols + end) + () + +let applied_blocks ctxt ?(chains = [`Main]) ?(protocols = []) ?(next_protocols = []) () = make_streamed_call - S.valid_blocks + S.applied_blocks ctxt () (object diff --git a/src/lib_shell_services/monitor_services.mli b/src/lib_shell_services/monitor_services.mli index 47aee81f5f52..0fd6b6d37168 100644 --- a/src/lib_shell_services/monitor_services.mli +++ b/src/lib_shell_services/monitor_services.mli @@ -38,7 +38,12 @@ val bootstrapped : #streamed -> ((Block_hash.t * Time.Protocol.t) Lwt_stream.t * stopper) tzresult Lwt.t -val valid_blocks : +(** Call RPC GET /monitor/valid_blocks (Deprecated) + + - Default [chains] is [Main]. + - Default [protocols] is [[]]. + - Default [next_protocols] is [[]]. *) +val legacy_valid_blocks : #streamed -> ?chains:Chain_services.chain list -> ?protocols:Protocol_hash.t list -> @@ -48,6 +53,18 @@ val valid_blocks : tzresult Lwt.t +val applied_blocks : + #streamed -> + ?chains:Chain_services.chain list -> + ?protocols:Protocol_hash.t list -> + ?next_protocols:Protocol_hash.t list -> + unit -> + ((Chain_id.t * Block_hash.t * Block_header.t * Operation.t list list) + Lwt_stream.t + * stopper) + tzresult + Lwt.t + val heads : #streamed -> ?next_protocols:Protocol_hash.t list -> @@ -72,7 +89,12 @@ module S : sig Block_hash.t * Time.Protocol.t ) Tezos_rpc.Service.t - val valid_blocks : + (** Define RPC GET /monitor/valid_blocks (Deprecated) + + - Default [chains] is [Main]. + - Default [protocols] is [[]]. + - Default [next_protocols] is [[]]. *) + val legacy_valid_blocks : ( [`GET], unit, unit, @@ -83,6 +105,17 @@ module S : sig (Chain_id.t * Block_hash.t) * Block_header.t ) Tezos_rpc.Service.t + val applied_blocks : + ( [`GET], + unit, + unit, + < chains : Chain_services.chain list + ; next_protocols : Protocol_hash.t list + ; protocols : Protocol_hash.t list >, + unit, + Chain_id.t * Block_hash.t * Block_header.t * Operation.t list list ) + Tezos_rpc.Service.t + val heads : ( [`GET], unit, diff --git a/src/proto_015_PtLimaPt/lib_delegate/baking_vdf.ml b/src/proto_015_PtLimaPt/lib_delegate/baking_vdf.ml index 621daa8e4022..9dcef0d556f5 100644 --- a/src/proto_015_PtLimaPt/lib_delegate/baking_vdf.ml +++ b/src/proto_015_PtLimaPt/lib_delegate/baking_vdf.ml @@ -51,7 +51,7 @@ type 'a state = { } let init_block_stream_with_stopper cctxt chain = - Client_baking_blocks.monitor_valid_blocks + Client_baking_blocks.monitor_applied_blocks ~next_protocols:(Some [Protocol.hash]) cctxt ~chains:[chain] diff --git a/src/proto_015_PtLimaPt/lib_delegate/client_baking_blocks.ml b/src/proto_015_PtLimaPt/lib_delegate/client_baking_blocks.ml index d7064545b7bd..0c255d63327f 100644 --- a/src/proto_015_PtLimaPt/lib_delegate/client_baking_blocks.ml +++ b/src/proto_015_PtLimaPt/lib_delegate/client_baking_blocks.ml @@ -143,12 +143,12 @@ module Block_seen_event = struct module Event = Internal_event.Make (Definition) end -let monitor_valid_blocks cctxt ?chains ?protocols ~next_protocols () = - Monitor_services.valid_blocks cctxt ?chains ?protocols ?next_protocols () +let monitor_applied_blocks cctxt ?chains ?protocols ~next_protocols () = + Monitor_services.applied_blocks cctxt ?chains ?protocols ?next_protocols () >>=? fun (block_stream, stop) -> return ( Lwt_stream.map_s - (fun ((chain, block), header) -> + (fun (chain, block, header, _ops) -> Block_seen_event.( Event.emit (make block header (`Valid_blocks chain))) >>=? fun () -> diff --git a/src/proto_015_PtLimaPt/lib_delegate/client_baking_blocks.mli b/src/proto_015_PtLimaPt/lib_delegate/client_baking_blocks.mli index c3358f61f82f..41b9f70b8619 100644 --- a/src/proto_015_PtLimaPt/lib_delegate/client_baking_blocks.mli +++ b/src/proto_015_PtLimaPt/lib_delegate/client_baking_blocks.mli @@ -45,7 +45,7 @@ val info : Block_services.block -> block_info tzresult Lwt.t -val monitor_valid_blocks : +val monitor_applied_blocks : #Protocol_client_context.rpc_context -> ?chains:Chain_services.chain list -> ?protocols:Protocol_hash.t list -> diff --git a/src/proto_015_PtLimaPt/lib_delegate/client_daemon.ml b/src/proto_015_PtLimaPt/lib_delegate/client_daemon.ml index ba2cfdac03b3..afc9f52d6aa7 100644 --- a/src/proto_015_PtLimaPt/lib_delegate/client_daemon.ml +++ b/src/proto_015_PtLimaPt/lib_delegate/client_daemon.ml @@ -124,7 +124,7 @@ module Accuser = struct Protocol_hash.pp_short Protocol.hash >>= fun () -> - Client_baking_blocks.monitor_valid_blocks + Client_baking_blocks.monitor_applied_blocks ~next_protocols:(Some [Protocol.hash]) cctxt ~chains:[chain] diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_vdf.ml b/src/proto_016_PtMumbai/lib_delegate/baking_vdf.ml index 8d2caefb709b..eaad6ce638af 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_vdf.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_vdf.ml @@ -51,7 +51,7 @@ type 'a state = { } let init_block_stream_with_stopper cctxt chain = - Client_baking_blocks.monitor_valid_blocks + Client_baking_blocks.monitor_applied_blocks ~next_protocols:(Some [Protocol.hash]) cctxt ~chains:[chain] diff --git a/src/proto_016_PtMumbai/lib_delegate/client_baking_blocks.ml b/src/proto_016_PtMumbai/lib_delegate/client_baking_blocks.ml index d7064545b7bd..0c255d63327f 100644 --- a/src/proto_016_PtMumbai/lib_delegate/client_baking_blocks.ml +++ b/src/proto_016_PtMumbai/lib_delegate/client_baking_blocks.ml @@ -143,12 +143,12 @@ module Block_seen_event = struct module Event = Internal_event.Make (Definition) end -let monitor_valid_blocks cctxt ?chains ?protocols ~next_protocols () = - Monitor_services.valid_blocks cctxt ?chains ?protocols ?next_protocols () +let monitor_applied_blocks cctxt ?chains ?protocols ~next_protocols () = + Monitor_services.applied_blocks cctxt ?chains ?protocols ?next_protocols () >>=? fun (block_stream, stop) -> return ( Lwt_stream.map_s - (fun ((chain, block), header) -> + (fun (chain, block, header, _ops) -> Block_seen_event.( Event.emit (make block header (`Valid_blocks chain))) >>=? fun () -> diff --git a/src/proto_016_PtMumbai/lib_delegate/client_baking_blocks.mli b/src/proto_016_PtMumbai/lib_delegate/client_baking_blocks.mli index c3358f61f82f..41b9f70b8619 100644 --- a/src/proto_016_PtMumbai/lib_delegate/client_baking_blocks.mli +++ b/src/proto_016_PtMumbai/lib_delegate/client_baking_blocks.mli @@ -45,7 +45,7 @@ val info : Block_services.block -> block_info tzresult Lwt.t -val monitor_valid_blocks : +val monitor_applied_blocks : #Protocol_client_context.rpc_context -> ?chains:Chain_services.chain list -> ?protocols:Protocol_hash.t list -> diff --git a/src/proto_016_PtMumbai/lib_delegate/client_daemon.ml b/src/proto_016_PtMumbai/lib_delegate/client_daemon.ml index 617935658380..a76a83271938 100644 --- a/src/proto_016_PtMumbai/lib_delegate/client_daemon.ml +++ b/src/proto_016_PtMumbai/lib_delegate/client_daemon.ml @@ -125,7 +125,7 @@ module Accuser = struct Protocol_hash.pp_short Protocol.hash >>= fun () -> - Client_baking_blocks.monitor_valid_blocks + Client_baking_blocks.monitor_applied_blocks ~next_protocols:(Some [Protocol.hash]) cctxt ~chains:[chain] diff --git a/src/proto_alpha/lib_delegate/baking_vdf.ml b/src/proto_alpha/lib_delegate/baking_vdf.ml index b08fd5ebea7b..d6ce4c580274 100644 --- a/src/proto_alpha/lib_delegate/baking_vdf.ml +++ b/src/proto_alpha/lib_delegate/baking_vdf.ml @@ -51,7 +51,7 @@ type 'a state = { } let init_block_stream_with_stopper cctxt chain = - Client_baking_blocks.monitor_valid_blocks + Client_baking_blocks.monitor_applied_blocks ~next_protocols:(Some [Protocol.hash]) cctxt ~chains:[chain] diff --git a/src/proto_alpha/lib_delegate/client_baking_blocks.ml b/src/proto_alpha/lib_delegate/client_baking_blocks.ml index d7064545b7bd..0c255d63327f 100644 --- a/src/proto_alpha/lib_delegate/client_baking_blocks.ml +++ b/src/proto_alpha/lib_delegate/client_baking_blocks.ml @@ -143,12 +143,12 @@ module Block_seen_event = struct module Event = Internal_event.Make (Definition) end -let monitor_valid_blocks cctxt ?chains ?protocols ~next_protocols () = - Monitor_services.valid_blocks cctxt ?chains ?protocols ?next_protocols () +let monitor_applied_blocks cctxt ?chains ?protocols ~next_protocols () = + Monitor_services.applied_blocks cctxt ?chains ?protocols ?next_protocols () >>=? fun (block_stream, stop) -> return ( Lwt_stream.map_s - (fun ((chain, block), header) -> + (fun (chain, block, header, _ops) -> Block_seen_event.( Event.emit (make block header (`Valid_blocks chain))) >>=? fun () -> diff --git a/src/proto_alpha/lib_delegate/client_baking_blocks.mli b/src/proto_alpha/lib_delegate/client_baking_blocks.mli index c3358f61f82f..41b9f70b8619 100644 --- a/src/proto_alpha/lib_delegate/client_baking_blocks.mli +++ b/src/proto_alpha/lib_delegate/client_baking_blocks.mli @@ -45,7 +45,7 @@ val info : Block_services.block -> block_info tzresult Lwt.t -val monitor_valid_blocks : +val monitor_applied_blocks : #Protocol_client_context.rpc_context -> ?chains:Chain_services.chain list -> ?protocols:Protocol_hash.t list -> diff --git a/src/proto_alpha/lib_delegate/client_daemon.ml b/src/proto_alpha/lib_delegate/client_daemon.ml index 617935658380..a76a83271938 100644 --- a/src/proto_alpha/lib_delegate/client_daemon.ml +++ b/src/proto_alpha/lib_delegate/client_daemon.ml @@ -125,7 +125,7 @@ module Accuser = struct Protocol_hash.pp_short Protocol.hash >>= fun () -> - Client_baking_blocks.monitor_valid_blocks + Client_baking_blocks.monitor_applied_blocks ~next_protocols:(Some [Protocol.hash]) cctxt ~chains:[chain] -- GitLab From 10a67f38be44efbe9ea9aa592b86112ed00bb382 Mon Sep 17 00:00:00 2001 From: vbot Date: Thu, 26 Jan 2023 13:55:15 +0100 Subject: [PATCH 21/46] Shell: expose '/monitor/validated_blocks' RPC --- src/lib_shell/monitor_directory.ml | 67 +++++++++++++++++++++ src/lib_shell_services/monitor_services.ml | 36 ++++++++++- src/lib_shell_services/monitor_services.mli | 43 +++++++++++++ src/lib_store/mocked/store.ml | 6 ++ src/lib_store/store.mli | 5 ++ src/lib_store/unix/store.ml | 9 +++ src/lib_store/unix/store.mli | 5 ++ 7 files changed, 168 insertions(+), 3 deletions(-) diff --git a/src/lib_shell/monitor_directory.ml b/src/lib_shell/monitor_directory.ml index 02f0737cc726..30a61126c17b 100644 --- a/src/lib_shell/monitor_directory.ml +++ b/src/lib_shell/monitor_directory.ml @@ -172,6 +172,73 @@ let build_rpc_directory validator mainchain_validator = in let next () = Lwt_stream.get stream in Tezos_rpc.Answer.return_stream {next; shutdown}) ; + gen_register0 Monitor_services.S.validated_blocks (fun q () -> + let* chains = + match q#chains with + | [] -> + let* all_chain_stores = Store.all_chain_stores store in + Lwt.return + (List.map (fun cs -> Store.Chain.chain_id cs) all_chain_stores) + | l -> + List.map_s (fun chain -> Chain_directory.get_chain_id store chain) l + in + let* block_streams, stoppers = + List.fold_left_s + (fun (block_streams, stoppers) chain_id -> + let* r = Store.get_chain_store store chain_id in + match r with + | Error _ -> Lwt.fail Not_found + | Ok chain_store -> + let block_stream, stopper = + Store.Chain.validated_watcher chain_store + in + let bs = + Lwt_stream.map + (fun b -> (chain_id, chain_store, b)) + block_stream + in + return (bs :: block_streams, stopper :: stoppers)) + ([], []) + chains + in + let block_stream = Lwt_stream.choose block_streams in + let shutdown () = List.iter Lwt_watcher.shutdown stoppers in + let in_next_protocols chain_store block = + match q#next_protocols with + | [] -> Lwt.return_true + | protocols -> ( + let* block_protocol_opt = + Store.Chain.find_protocol + chain_store + ~protocol_level:(Store.Block.proto_level block) + in + match block_protocol_opt with + | None -> + (* If we do not know the protocol hash associated to the + proto level, it means either that it is a transition + block for which we do not know the protocol yet, or + that something really bad occurred. In both cases, we + do not advertise it. *) + Lwt.return_false + | Some next_protocol -> + Lwt.return + (List.exists (Protocol_hash.equal next_protocol) protocols)) + in + let stream = + Lwt_stream.filter_map_s + (fun (chain_id, chain_store, block) -> + let* in_next_protocols = in_next_protocols chain_store block in + if in_next_protocols then + Lwt.return_some + ( chain_id, + Store.Block.hash block, + Store.Block.header block, + Store.Block.operations block ) + else Lwt.return_none) + block_stream + in + let next () = Lwt_stream.get stream in + Tezos_rpc.Answer.return_stream {next; shutdown}) ; gen_register1 Monitor_services.S.heads (fun chain q () -> (* TODO: when `chain = `Test`, should we reset then stream when the `testnet` change, or dias we currently do ?? *) diff --git a/src/lib_shell_services/monitor_services.ml b/src/lib_shell_services/monitor_services.ml index 2664d385c405..6a668e87d893 100644 --- a/src/lib_shell_services/monitor_services.ml +++ b/src/lib_shell_services/monitor_services.ml @@ -114,12 +114,27 @@ module S = struct Block_header.encoding) Tezos_rpc.Path.(path / "valid_blocks") + let validated_blocks = + Tezos_rpc.Service.get_service + ~description: + "Monitor all blocks that were successfully validated by the node but \ + are not applied nor stored yet, disregarding whether they are going \ + to be selected as the new head or not." + ~query:validated_or_apply_blocks_query + ~output: + (obj4 + (req "chain_id" Chain_id.encoding) + (req "hash" Block_hash.encoding) + (req "header" (dynamic_size Block_header.encoding)) + (req "operations" (list (list (dynamic_size Operation.encoding))))) + Tezos_rpc.Path.(path / "validated_blocks") + let applied_blocks = Tezos_rpc.Service.get_service ~description: "Monitor all blocks that are successfully applied and stored by the \ node, disregarding whether they were selected as the new head or not." - ~query:evaluated_blocks_query + ~query:validated_or_apply_blocks_query ~output: (obj4 (req "chain_id" Chain_id.encoding) @@ -141,8 +156,8 @@ module S = struct let heads = Tezos_rpc.Service.get_service ~description: - "Monitor all blocks that are successfully validated by the node and \ - selected as the new head of the given chain." + "Monitor all blocks that are successfully validated and applied by the \ + node and selected as the new head of the given chain." ~query:heads_query ~output: (merge_objs @@ -196,6 +211,21 @@ let legacy_valid_blocks ctxt ?(chains = [`Main]) ?(protocols = []) end) () +let validated_blocks ctxt ?(chains = [`Main]) ?(protocols = []) + ?(next_protocols = []) () = + make_streamed_call + S.validated_blocks + ctxt + () + (object + method chains = chains + + method protocols = protocols + + method next_protocols = next_protocols + end) + () + let applied_blocks ctxt ?(chains = [`Main]) ?(protocols = []) ?(next_protocols = []) () = make_streamed_call diff --git a/src/lib_shell_services/monitor_services.mli b/src/lib_shell_services/monitor_services.mli index 0fd6b6d37168..a2635743d64e 100644 --- a/src/lib_shell_services/monitor_services.mli +++ b/src/lib_shell_services/monitor_services.mli @@ -53,6 +53,28 @@ val legacy_valid_blocks : tzresult Lwt.t +(** Call RPC GET /monitor/validated_blocks + + - Default [chains] is [Main]. + - Default [protocols] is [[]]. + - Default [next_protocols] is [[]]. *) +val validated_blocks : + #streamed -> + ?chains:Chain_services.chain list -> + ?protocols:Protocol_hash.t list -> + ?next_protocols:Protocol_hash.t list -> + unit -> + ((Chain_id.t * Block_hash.t * Block_header.t * Operation.t list list) + Lwt_stream.t + * stopper) + tzresult + Lwt.t + +(** Call RPC GET /monitor/applied_blocks + + - Default [chains] is [Main]. + - Default [protocols] is [[]]. + - Default [next_protocols] is [[]]. *) val applied_blocks : #streamed -> ?chains:Chain_services.chain list -> @@ -105,6 +127,27 @@ module S : sig (Chain_id.t * Block_hash.t) * Block_header.t ) Tezos_rpc.Service.t + (** Define RPC GET /monitor/validated_blocks + + - Default [chains] is [Main]. + - Default [protocols] is [[]]. + - Default [next_protocols] is [[]]. *) + val validated_blocks : + ( [`GET], + unit, + unit, + < chains : Chain_services.chain list + ; next_protocols : Protocol_hash.t list + ; protocols : Protocol_hash.t list >, + unit, + Chain_id.t * Block_hash.t * Block_header.t * Operation.t list list ) + Tezos_rpc.Service.t + + (** Define RPC GET /monitor/applied_blocks + + - Default [chains] is [Main]. + - Default [protocols] is [[]]. + - Default [next_protocols] is [[]]. *) val applied_blocks : ( [`GET], unit, diff --git a/src/lib_store/mocked/store.ml b/src/lib_store/mocked/store.ml index 90fca8515e51..89ab66d92104 100644 --- a/src/lib_store/mocked/store.ml +++ b/src/lib_store/mocked/store.ml @@ -74,6 +74,7 @@ and chain_store = { (* Genesis is only on-disk: read-only except at creation *) genesis_block_data : block Stored_data.t; block_watcher : block Lwt_watcher.input; + validated_block_watcher : block Lwt_watcher.input; block_rpc_directories : (chain_store * block) Tezos_rpc.Directory.t Protocol_hash.Map.t Protocol_hash.Table.t; @@ -1473,6 +1474,7 @@ module Chain = struct in let chain_state = Shared.create chain_state in let block_watcher = Lwt_watcher.create_input () in + let validated_block_watcher = Lwt_watcher.create_input () in let block_rpc_directories = Protocol_hash.Table.create 7 in let chain_store : chain_store = { @@ -1484,6 +1486,7 @@ module Chain = struct genesis_block_data; block_store; block_watcher; + validated_block_watcher; block_rpc_directories; } in @@ -1665,6 +1668,9 @@ module Chain = struct Shared.use chain_store.chain_state (fun {protocol_levels_data; _} -> Stored_data.get protocol_levels_data) + let validated_watcher chain_store = + Lwt_watcher.create_stream chain_store.validated_block_watcher + let watcher chain_store = Lwt_watcher.create_stream chain_store.block_watcher let get_rpc_directory chain_store block = diff --git a/src/lib_store/store.mli b/src/lib_store/store.mli index 9e98d883a1e1..430a310e6d85 100644 --- a/src/lib_store/store.mli +++ b/src/lib_store/store.mli @@ -915,6 +915,11 @@ module Chain : sig [chain_store]. *) val watcher : chain_store -> Block.t Lwt_stream.t * Lwt_watcher.stopper + (** [validated_watcher chain_store] instantiates a new validated block + watcher for [chain_store]. *) + val validated_watcher : + chain_store -> Block.t Lwt_stream.t * Lwt_watcher.stopper + (** [get_rpc_directory chain_store block] returns the RPC directory associated to the [block]. *) val get_rpc_directory : diff --git a/src/lib_store/unix/store.ml b/src/lib_store/unix/store.ml index e7cbdc8a8d52..197e06286805 100644 --- a/src/lib_store/unix/store.ml +++ b/src/lib_store/unix/store.ml @@ -77,6 +77,7 @@ and chain_store = { (* Genesis is only on-disk: read-only except at creation *) genesis_block_data : block Stored_data.t; block_watcher : block Lwt_watcher.input; + validated_block_watcher : block Lwt_watcher.input; block_rpc_directories : (chain_store * block) Tezos_rpc.Directory.t Protocol_hash.Map.t Protocol_hash.Table.t; @@ -623,6 +624,7 @@ module Block = struct let*! () = Store_events.(emit store_validated_block) (hash, block_header.shell.level) in + Lwt_watcher.notify chain_store.validated_block_watcher block ; return_unit let resulting_context_hash chain_store block = @@ -1915,6 +1917,7 @@ module Chain = struct in let chain_state = Shared.create chain_state in let block_watcher = Lwt_watcher.create_input () in + let validated_block_watcher = Lwt_watcher.create_input () in let block_rpc_directories = Protocol_hash.Table.create 7 in let* lockfile = create_lockfile chain_dir in let chain_store : chain_store = @@ -1927,6 +1930,7 @@ module Chain = struct genesis_block_data; block_store; block_watcher; + validated_block_watcher; block_rpc_directories; lockfile; } @@ -1950,6 +1954,7 @@ module Chain = struct let* chain_state = load_chain_state chain_dir block_store in let chain_state = Shared.create chain_state in let block_watcher = Lwt_watcher.create_input () in + let validated_block_watcher = Lwt_watcher.create_input () in let block_rpc_directories = Protocol_hash.Table.create 7 in let* lockfile = create_lockfile chain_dir in let chain_store = @@ -1963,6 +1968,7 @@ module Chain = struct chain_state; genesis_block_data; block_watcher; + validated_block_watcher; block_rpc_directories; lockfile; } @@ -2297,6 +2303,9 @@ module Chain = struct Shared.use chain_store.chain_state (fun {protocol_levels_data; _} -> Stored_data.get protocol_levels_data) + let validated_watcher chain_store = + Lwt_watcher.create_stream chain_store.validated_block_watcher + let watcher chain_store = Lwt_watcher.create_stream chain_store.block_watcher let get_rpc_directory chain_store block = diff --git a/src/lib_store/unix/store.mli b/src/lib_store/unix/store.mli index 952ea4476e52..96827bbf0e87 100644 --- a/src/lib_store/unix/store.mli +++ b/src/lib_store/unix/store.mli @@ -909,6 +909,11 @@ module Chain : sig [chain_store]. *) val watcher : chain_store -> Block.t Lwt_stream.t * Lwt_watcher.stopper + (** [validated_watcher chain_store] instantiates a new validated block + watcher for [chain_store]. *) + val validated_watcher : + chain_store -> Block.t Lwt_stream.t * Lwt_watcher.stopper + (** [get_rpc_directory chain_store block] returns the RPC directory associated to the [block]. *) val get_rpc_directory : -- GitLab From bd7254da761b920c4feba0b96e3c14e89fe5c9bf Mon Sep 17 00:00:00 2001 From: vbot Date: Fri, 13 Jan 2023 16:33:14 +0100 Subject: [PATCH 22/46] Changelog: add entry for applied/validated_blocks RPCs --- CHANGES.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index e5065aa150aa..24c89cefeba7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -143,6 +143,15 @@ Node - **Breaking Change**: disabled snapshot export support for storage that was created with Octez v13 (or earlier). +- Deprecated the RPC ``GET /monitor/valid_blocks`` and introduced + ``GET /monitor/validated_blocks`` and ``GET /monitor/applied_blocks`` + which respectively returns validated blocks, which are not yet applied + nor stored, and applied blocks which are fully applied and stored by + the node. (MR :gl: `!7513`) + +- Replaced some "precheck" occurrences with "validate" in event and + error identifiers and messages. (MR :gl: `!7513`) + Client ------ -- GitLab From 31145354bd98594887367b2d6823ee77afac0e0e Mon Sep 17 00:00:00 2001 From: vbot Date: Thu, 26 Jan 2023 17:09:39 +0100 Subject: [PATCH 23/46] Alpha/Baker: discard corrupted/legacy state files --- src/proto_alpha/lib_delegate/baking_state.ml | 168 ++----------------- 1 file changed, 17 insertions(+), 151 deletions(-) diff --git a/src/proto_alpha/lib_delegate/baking_state.ml b/src/proto_alpha/lib_delegate/baking_state.ml index 475306cac7d8..8d7b97748316 100644 --- a/src/proto_alpha/lib_delegate/baking_state.ml +++ b/src/proto_alpha/lib_delegate/baking_state.ml @@ -436,138 +436,21 @@ let event_encoding = (fun tk -> Timeout tk); ] -(* Legacy disk state support: this will only be used when migrating - from a baker using the previous format to the new one and will - happen once. *) - -type legacy_block_info = { - legacy_hash : Block_hash.t; - legacy_shell : Block_header.shell_header; - legacy_payload_hash : Block_payload_hash.t; - legacy_payload_round : Round.t; - legacy_round : Round.t; - legacy_protocol : Protocol_hash.t; - legacy_next_protocol : Protocol_hash.t; - legacy_prequorum : prequorum option; - legacy_quorum : Kind.endorsement operation list; - legacy_payload : Operation_pool.payload; - legacy_live_blocks : Block_hash.Set.t; -} - -let legacy_block_info_encoding : legacy_block_info Data_encoding.t = - let open Data_encoding in - conv - (fun ({ - legacy_hash; - legacy_shell; - legacy_payload_hash; - legacy_payload_round; - legacy_round; - legacy_protocol; - legacy_next_protocol; - legacy_prequorum; - legacy_quorum; - legacy_payload; - legacy_live_blocks; - } : - legacy_block_info) -> - ( ( legacy_hash, - legacy_shell, - legacy_payload_hash, - legacy_payload_round, - legacy_round, - legacy_protocol, - legacy_next_protocol, - legacy_prequorum, - List.map Operation.pack legacy_quorum, - legacy_payload ), - legacy_live_blocks )) - (fun ( ( legacy_hash, - legacy_shell, - legacy_payload_hash, - legacy_payload_round, - legacy_round, - legacy_protocol, - legacy_next_protocol, - legacy_prequorum, - legacy_quorum, - legacy_payload ), - legacy_live_blocks ) -> - { - legacy_hash; - legacy_shell; - legacy_payload_hash; - legacy_payload_round; - legacy_round; - legacy_protocol; - legacy_next_protocol; - legacy_prequorum; - legacy_quorum = - List.filter_map Operation_pool.unpack_endorsement legacy_quorum; - legacy_payload; - legacy_live_blocks; - }) - (merge_objs - (obj10 - (req "hash" Block_hash.encoding) - (req "shell" Block_header.shell_header_encoding) - (req "payload_hash" Block_payload_hash.encoding) - (req "payload_round" Round.encoding) - (req "round" Round.encoding) - (req "protocol" Protocol_hash.encoding) - (req "next_protocol" Protocol_hash.encoding) - (req "prequorum" (option prequorum_encoding)) - (req "quorum" (list (dynamic_size Operation.encoding))) - (req "payload" Operation_pool.payload_encoding)) - (obj1 (req "live_blocks" Block_hash.Set.encoding))) - -type legacy_proposal = { - legacy_block : legacy_block_info; - legacy_predecessor : legacy_block_info; -} - -let legacy_proposal_encoding = - let open Data_encoding in - conv - (fun {legacy_block; legacy_predecessor} -> - (legacy_block, legacy_predecessor)) - (fun (legacy_block, legacy_predecessor) -> - {legacy_block; legacy_predecessor}) - (obj2 - (req "block" legacy_block_info_encoding) - (req "predecessor" legacy_block_info_encoding)) - -type legacy_endorsable_payload = { - legacy_proposal : legacy_proposal; - prequorum : prequorum; -} +(* Disk state *) -let legacy_endorsable_payload_encoding = - let open Data_encoding in - conv - (fun {legacy_proposal; prequorum} -> (legacy_proposal, prequorum)) - (fun (legacy_proposal, prequorum) -> {legacy_proposal; prequorum}) - (obj2 - (req "proposal" legacy_proposal_encoding) - (req "prequorum" prequorum_encoding)) +module Events = struct + include Internal_event.Simple -type legacy_state_data = { - level_data : int32; - locked_round_data : locked_round option; - legacy_endorsable_payload_data : legacy_endorsable_payload option; -} + let section = [Protocol.name; "baker"; "disk"] -let legacy_state_data_encoding = - let open Data_encoding in - conv - (fun {level_data; locked_round_data; legacy_endorsable_payload_data} -> - (level_data, locked_round_data, legacy_endorsable_payload_data)) - (fun (level_data, locked_round_data, legacy_endorsable_payload_data) -> - {level_data; locked_round_data; legacy_endorsable_payload_data}) - (obj3 - (req "level" int32) - (req "locked_round" (option locked_round_encoding)) - (req "endorsable_payload" (option legacy_endorsable_payload_encoding))) + let incompatible_stored_state = + declare_0 + ~section + ~name:"incompatible_stored_state" + ~level:Warning + ~msg:"found an outdated or corrupted baking state: discarding it" + () +end type state_data = { level_data : int32; @@ -575,25 +458,6 @@ type state_data = { endorsable_payload_data : endorsable_payload option; } -let update_legacy_state_data raw_data = - (* "Upgrade" the file format by dumping the legacy - endorsable data. It is sound as the new baker - cannot start (i.e. load the preexisting state) - until the new protocol activates. - - Note: the new encoding is not backward compatible - (unless the endorsable_payload_data is [None]). *) - let legacy_state_data = - Data_encoding.Binary.of_string_exn legacy_state_data_encoding raw_data - in - { - level_data = legacy_state_data.level_data; - locked_round_data = None; - endorsable_payload_data = None; - } - -(* Disk state *) - let state_data_encoding = let open Data_encoding in conv @@ -723,9 +587,11 @@ let load_endorsable_data cctxt location = match Data_encoding.Binary.of_string_opt state_data_encoding str with - | Some state_data -> Lwt.return state_data - | None -> Lwt.return (update_legacy_state_data str)) - >>= return_some) + | Some state_data -> return_some state_data + | None -> + (* The stored state format is incompatible: discard it. *) + Events.(emit incompatible_stored_state ()) >>= fun () -> + return_none)) let may_load_endorsable_data state = let cctxt = state.global_state.cctxt in -- GitLab From a057100fd4a4a4ac559c51183aa3696a38bb7f54 Mon Sep 17 00:00:00 2001 From: vbot Date: Thu, 9 Feb 2023 14:31:50 +0100 Subject: [PATCH 24/46] Mumbai/Baker: backport changes --- .../lib_delegate/baking_state.ml | 168 ++---------------- 1 file changed, 17 insertions(+), 151 deletions(-) diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_state.ml b/src/proto_016_PtMumbai/lib_delegate/baking_state.ml index 984c2bbbd237..c5f6dcab2b20 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_state.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_state.ml @@ -442,138 +442,21 @@ let event_encoding = (fun tk -> Timeout tk); ] -(* Legacy disk state support: this will only be used when migrating - from a baker using the previous format to the new one and will - happen once. *) - -type legacy_block_info = { - legacy_hash : Block_hash.t; - legacy_shell : Block_header.shell_header; - legacy_payload_hash : Block_payload_hash.t; - legacy_payload_round : Round.t; - legacy_round : Round.t; - legacy_protocol : Protocol_hash.t; - legacy_next_protocol : Protocol_hash.t; - legacy_prequorum : prequorum option; - legacy_quorum : Kind.endorsement operation list; - legacy_payload : Operation_pool.payload; - legacy_live_blocks : Block_hash.Set.t; -} - -let legacy_block_info_encoding : legacy_block_info Data_encoding.t = - let open Data_encoding in - conv - (fun ({ - legacy_hash; - legacy_shell; - legacy_payload_hash; - legacy_payload_round; - legacy_round; - legacy_protocol; - legacy_next_protocol; - legacy_prequorum; - legacy_quorum; - legacy_payload; - legacy_live_blocks; - } : - legacy_block_info) -> - ( ( legacy_hash, - legacy_shell, - legacy_payload_hash, - legacy_payload_round, - legacy_round, - legacy_protocol, - legacy_next_protocol, - legacy_prequorum, - List.map Operation.pack legacy_quorum, - legacy_payload ), - legacy_live_blocks )) - (fun ( ( legacy_hash, - legacy_shell, - legacy_payload_hash, - legacy_payload_round, - legacy_round, - legacy_protocol, - legacy_next_protocol, - legacy_prequorum, - legacy_quorum, - legacy_payload ), - legacy_live_blocks ) -> - { - legacy_hash; - legacy_shell; - legacy_payload_hash; - legacy_payload_round; - legacy_round; - legacy_protocol; - legacy_next_protocol; - legacy_prequorum; - legacy_quorum = - List.filter_map Operation_pool.unpack_endorsement legacy_quorum; - legacy_payload; - legacy_live_blocks; - }) - (merge_objs - (obj10 - (req "hash" Block_hash.encoding) - (req "shell" Block_header.shell_header_encoding) - (req "payload_hash" Block_payload_hash.encoding) - (req "payload_round" Round.encoding) - (req "round" Round.encoding) - (req "protocol" Protocol_hash.encoding) - (req "next_protocol" Protocol_hash.encoding) - (req "prequorum" (option prequorum_encoding)) - (req "quorum" (list (dynamic_size Operation.encoding))) - (req "payload" Operation_pool.payload_encoding)) - (obj1 (req "live_blocks" Block_hash.Set.encoding))) - -type legacy_proposal = { - legacy_block : legacy_block_info; - legacy_predecessor : legacy_block_info; -} - -let legacy_proposal_encoding = - let open Data_encoding in - conv - (fun {legacy_block; legacy_predecessor} -> - (legacy_block, legacy_predecessor)) - (fun (legacy_block, legacy_predecessor) -> - {legacy_block; legacy_predecessor}) - (obj2 - (req "block" legacy_block_info_encoding) - (req "predecessor" legacy_block_info_encoding)) - -type legacy_endorsable_payload = { - legacy_proposal : legacy_proposal; - prequorum : prequorum; -} +(* Disk state *) -let legacy_endorsable_payload_encoding = - let open Data_encoding in - conv - (fun {legacy_proposal; prequorum} -> (legacy_proposal, prequorum)) - (fun (legacy_proposal, prequorum) -> {legacy_proposal; prequorum}) - (obj2 - (req "proposal" legacy_proposal_encoding) - (req "prequorum" prequorum_encoding)) +module Events = struct + include Internal_event.Simple -type legacy_state_data = { - level_data : int32; - locked_round_data : locked_round option; - legacy_endorsable_payload_data : legacy_endorsable_payload option; -} + let section = [Protocol.name; "baker"; "disk"] -let legacy_state_data_encoding = - let open Data_encoding in - conv - (fun {level_data; locked_round_data; legacy_endorsable_payload_data} -> - (level_data, locked_round_data, legacy_endorsable_payload_data)) - (fun (level_data, locked_round_data, legacy_endorsable_payload_data) -> - {level_data; locked_round_data; legacy_endorsable_payload_data}) - (obj3 - (req "level" int32) - (req "locked_round" (option locked_round_encoding)) - (req "endorsable_payload" (option legacy_endorsable_payload_encoding))) + let incompatible_stored_state = + declare_0 + ~section + ~name:"incompatible_stored_state" + ~level:Warning + ~msg:"found an outdated or corrupted baking state: discarding it" + () +end type state_data = { level_data : int32; @@ -581,25 +464,6 @@ type state_data = { endorsable_payload_data : endorsable_payload option; } -let update_legacy_state_data raw_data = - (* "Upgrade" the file format by dumping the legacy - endorsable data. It is sound as the new baker - cannot start (i.e. load the preexisting state) - until the new protocol activates. - - Note: the new encoding is not backward compatible - (unless the endorsable_payload_data is [None]). *) - let legacy_state_data = - Data_encoding.Binary.of_string_exn legacy_state_data_encoding raw_data - in - { - level_data = legacy_state_data.level_data; - locked_round_data = None; - endorsable_payload_data = None; - } - -(* Disk state *) - let state_data_encoding = let open Data_encoding in conv @@ -729,9 +593,11 @@ let load_endorsable_data cctxt location = match Data_encoding.Binary.of_string_opt state_data_encoding str with - | Some state_data -> Lwt.return state_data - | None -> Lwt.return (update_legacy_state_data str)) - >>= return_some) + | Some state_data -> return_some state_data + | None -> + (* The stored state format is incompatible: discard it. *) + Events.(emit incompatible_stored_state ()) >>= fun () -> + return_none)) let may_load_endorsable_data state = let cctxt = state.global_state.cctxt in -- GitLab From e9cb8d9d5fc627eb04c9374a3a6fe1b8c72c8603 Mon Sep 17 00:00:00 2001 From: vbot Date: Thu, 9 Feb 2023 14:34:49 +0100 Subject: [PATCH 25/46] Changes: added entry for baker corrupted/legacy states discarding --- CHANGES.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 24c89cefeba7..ad58e1ce5630 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -192,6 +192,10 @@ Baker should make sure that their operations' signatures are correct. (MR :gl:`!7490`) +- Made the baker discard legacy or corrupted Tenderbake's saved + states in order to avoid unexpected crashes when the baker gets + updated, or when a new protocol's baker starts. (MR :gl:`!7640`) + Accuser ------- -- GitLab From c3610ad27c8fd36dc874cf69fa6585e14f3aa70d Mon Sep 17 00:00:00 2001 From: vbot Date: Tue, 31 Jan 2023 13:49:31 +0100 Subject: [PATCH 26/46] Mumbai/Baker: backport baker alpha changes to mumbai's baker --- .../lib_delegate/baking_actions.ml | 46 +++++++------- .../lib_delegate/baking_actions.mli | 11 +--- .../lib_delegate/baking_commands.ml | 2 +- .../lib_delegate/baking_events.ml | 4 +- .../lib_delegate/baking_highwatermarks.ml | 6 +- .../lib_delegate/baking_highwatermarks.mli | 12 ++-- .../lib_delegate/baking_lib.ml | 25 ++------ .../lib_delegate/baking_nonces.ml | 4 +- .../lib_delegate/baking_scheduling.ml | 54 ++++++----------- .../lib_delegate/baking_state.ml | 59 +++++++++--------- .../lib_delegate/baking_state.mli | 11 ++-- .../lib_delegate/baking_vdf.ml | 2 +- .../lib_delegate/block_forge.ml | 4 +- .../client_baking_denunciation.ml | 10 +--- .../lib_delegate/operation_selection.ml | 4 +- .../lib_delegate/state_transitions.ml | 60 +++++++++---------- .../lib_delegate/state_transitions.mli | 3 + .../test/mockup_simulator/mockup_simulator.ml | 20 +++---- .../mockup_simulator/mockup_simulator.mli | 18 +++--- .../test/tenderbrute/lib/tenderbrute.ml | 12 +--- .../test/tenderbrute/lib/tenderbrute.mli | 4 +- .../test/tenderbrute/tenderbrute_main.ml | 2 +- .../lib_delegate/test/test_scenario.ml | 27 +++++---- 23 files changed, 170 insertions(+), 230 deletions(-) diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_actions.ml b/src/proto_016_PtMumbai/lib_delegate/baking_actions.ml index 89cd0af885cb..cc6dfc5b54eb 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_actions.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_actions.ml @@ -126,14 +126,13 @@ type action = | Inject_block of {block_to_bake : block_to_bake; updated_state : state} | Inject_preendorsements of { preendorsements : (consensus_key_and_delegate * consensus_content) list; - updated_state : state; } | Inject_endorsements of { endorsements : (consensus_key_and_delegate * consensus_content) list; - updated_state : state; } | Update_to_level of level_update | Synchronize_round of round_update + | Watch_proposal and level_update = { new_level_proposal : proposal; @@ -158,6 +157,7 @@ let pp_action fmt = function | Inject_endorsements _ -> Format.fprintf fmt "inject endorsements" | Update_to_level _ -> Format.fprintf fmt "update to level" | Synchronize_round _ -> Format.fprintf fmt "synchronize round" + | Watch_proposal -> Format.fprintf fmt "watch proposal" let generate_seed_nonce_hash config delegate level = if level.Level.expected_commitment then @@ -337,8 +337,7 @@ let inject_block ~state_recorder state block_to_bake ~updated_state = emit block_injected (bh, signed_block_header.shell.level, round, delegate)) >>= fun () -> return updated_state -let inject_preendorsements ~state_recorder state ~preendorsements ~updated_state - = +let inject_preendorsements state ~preendorsements = let cctxt = state.global_state.cctxt in let chain_id = state.global_state.chain_id in (* N.b. signing a lot of operations may take some time *) @@ -401,7 +400,6 @@ let inject_preendorsements ~state_recorder state ~preendorsements ~updated_state return_some (delegate, operation)) preendorsements >>=? fun signed_operations -> - state_recorder ~new_state:updated_state >>=? fun () -> (* TODO: add a RPC to inject multiple operations *) List.iter_ep (fun (delegate, operation) -> @@ -421,7 +419,6 @@ let inject_preendorsements ~state_recorder state ~preendorsements ~updated_state Events.(emit preendorsement_injected (oph, delegate)) >>= fun () -> return_unit)) signed_operations - >>=? fun () -> return updated_state let sign_endorsements state endorsements = let cctxt = state.global_state.cctxt in @@ -488,11 +485,10 @@ let sign_endorsements state endorsements = return_some (delegate, operation)) endorsements -let inject_endorsements ~state_recorder state ~endorsements ~updated_state = +let inject_endorsements state ~endorsements = let cctxt = state.global_state.cctxt in let chain_id = state.global_state.chain_id in sign_endorsements state endorsements >>=? fun signed_operations -> - state_recorder ~new_state:updated_state >>=? fun () -> (* TODO: add a RPC to inject multiple operations *) List.iter_ep (fun (delegate, signed_operation) -> @@ -507,7 +503,6 @@ let inject_endorsements ~state_recorder state ~endorsements ~updated_state = Events.(emit endorsement_injected (oph, delegate)) >>= fun () -> return_unit) signed_operations - >>=? fun () -> return updated_state let prepare_waiting_for_quorum state = let consensus_threshold = @@ -613,31 +608,34 @@ let synchronize_round state {new_round_proposal; handle_proposal} = let new_state = {state with round_state = new_round_state} in handle_proposal new_state >>= return +(* TODO: https://gitlab.com/tezos/tezos/-/issues/4539 + Avoid updating the state here. + (See also comment in {!State_transitions.step}.) + + TODO: https://gitlab.com/tezos/tezos/-/issues/4538 + Improve/clarify when the state is recorded. +*) let rec perform_action ~state_recorder state (action : action) = match action with | Do_nothing -> state_recorder ~new_state:state >>=? fun () -> return state | Inject_block {block_to_bake; updated_state} -> inject_block state ~state_recorder block_to_bake ~updated_state - | Inject_preendorsements {preendorsements; updated_state} -> - inject_preendorsements - ~state_recorder - state - ~preendorsements - ~updated_state - >>=? fun new_state -> - (* We wait for preendorsements to trigger the - [Prequorum_reached] event *) - start_waiting_for_preendorsement_quorum state >>= fun () -> - return new_state - | Inject_endorsements {endorsements; updated_state} -> - inject_endorsements ~state_recorder state ~endorsements ~updated_state - >>=? fun new_state -> + | Inject_preendorsements {preendorsements} -> + inject_preendorsements state ~preendorsements >>=? fun () -> + perform_action ~state_recorder state Watch_proposal + | Inject_endorsements {endorsements} -> + state_recorder ~new_state:state >>=? fun () -> + inject_endorsements state ~endorsements >>=? fun () -> (* We wait for endorsements to trigger the [Quorum_reached] event *) - start_waiting_for_endorsement_quorum state >>= fun () -> return new_state + start_waiting_for_endorsement_quorum state >>= fun () -> return state | Update_to_level level_update -> update_to_level state level_update >>=? fun (new_state, new_action) -> perform_action ~state_recorder new_state new_action | Synchronize_round round_update -> synchronize_round state round_update >>=? fun (new_state, new_action) -> perform_action ~state_recorder new_state new_action + | Watch_proposal -> + (* We wait for preendorsements to trigger the + [Prequorum_reached] event *) + start_waiting_for_preendorsement_quorum state >>= fun () -> return state diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_actions.mli b/src/proto_016_PtMumbai/lib_delegate/baking_actions.mli index da181cda8c32..91b9bf772915 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_actions.mli +++ b/src/proto_016_PtMumbai/lib_delegate/baking_actions.mli @@ -48,14 +48,13 @@ type action = | Inject_block of {block_to_bake : block_to_bake; updated_state : state} | Inject_preendorsements of { preendorsements : (consensus_key_and_delegate * consensus_content) list; - updated_state : state; } | Inject_endorsements of { endorsements : (consensus_key_and_delegate * consensus_content) list; - updated_state : state; } | Update_to_level of level_update | Synchronize_round of round_update + | Watch_proposal and level_update = { new_level_proposal : proposal; @@ -87,11 +86,9 @@ val inject_block : state tzresult Lwt.t val inject_preendorsements : - state_recorder:(new_state:state -> unit tzresult Lwt.t) -> state -> preendorsements:(consensus_key_and_delegate * consensus_content) list -> - updated_state:state -> - state tzresult Lwt.t + unit tzresult Lwt.t val sign_endorsements : state -> @@ -99,11 +96,9 @@ val sign_endorsements : (consensus_key_and_delegate * packed_operation) list tzresult Lwt.t val inject_endorsements : - state_recorder:(new_state:state -> unit tzresult Lwt.t) -> state -> endorsements:(consensus_key_and_delegate * consensus_content) list -> - updated_state:state -> - state tzresult Lwt.t + unit tzresult Lwt.t val prepare_waiting_for_quorum : state -> int * (slot:Slot.t -> int) * Operation_worker.candidate diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_commands.ml b/src/proto_016_PtMumbai/lib_delegate/baking_commands.ml index 1e6dbe5a6152..33dd83a2bdcc 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_commands.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_commands.ml @@ -172,7 +172,7 @@ let liquidity_baking_toggle_vote_arg = liquidity_baking_toggle_vote_parameter let get_delegates (cctxt : Protocol_client_context.full) - (pkhs : Tezos_crypto.Signature.public_key_hash list) = + (pkhs : Signature.public_key_hash list) = let proj_delegate (alias, public_key_hash, public_key, secret_key_uri) = { Baking_state.alias = Some alias; diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_events.ml b/src/proto_016_PtMumbai/lib_delegate/baking_events.ml index 0dd45ce85c65..34cd55dec8ce 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_events.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_events.ml @@ -498,7 +498,7 @@ module Actions = struct ~section ~name:"skipping_preendorsement" ~level:Error - ~msg:"skipping preendorsement for {delegate} -- {trace}" + ~msg:"unable to sign preendorsement for {delegate} -- {trace}" ~pp1:Baking_state.pp_consensus_key_and_delegate ("delegate", Baking_state.consensus_key_and_delegate_encoding) ~pp2:Error_monad.pp_print_trace @@ -509,7 +509,7 @@ module Actions = struct ~section ~name:"skipping_endorsement" ~level:Error - ~msg:"skipping endorsement for {delegate} -- {trace}" + ~msg:"unable to sign endorsement for {delegate} -- {trace}" ~pp1:Baking_state.pp_consensus_key_and_delegate ("delegate", Baking_state.consensus_key_and_delegate_encoding) ~pp2:Error_monad.pp_print_trace diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_highwatermarks.ml b/src/proto_016_PtMumbai/lib_delegate/baking_highwatermarks.ml index 4af172f183fe..91b56b3e50b1 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_highwatermarks.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_highwatermarks.ml @@ -100,9 +100,9 @@ let () = (fun highwatermark -> Block_previously_endorsed highwatermark) module DelegateMap = Map.Make (struct - type t = Tezos_crypto.Signature.Public_key_hash.t + type t = Signature.Public_key_hash.t - let compare = Tezos_crypto.Signature.Public_key_hash.compare + let compare = Signature.Public_key_hash.compare end) let highwatermark_delegate_map_encoding = @@ -113,7 +113,7 @@ let highwatermark_delegate_map_encoding = fun l -> List.fold_left (fun map (k, v) -> add k v map) empty l) (list (obj2 - (req "delegate" Tezos_crypto.Signature.Public_key_hash.encoding) + (req "delegate" Signature.Public_key_hash.encoding) (req "highwatermark" highwatermark_encoding))) type highwatermarks = { diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_highwatermarks.mli b/src/proto_016_PtMumbai/lib_delegate/baking_highwatermarks.mli index 30d4d6f4ea3e..757b667c58bb 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_highwatermarks.mli +++ b/src/proto_016_PtMumbai/lib_delegate/baking_highwatermarks.mli @@ -45,7 +45,7 @@ val load : val may_sign_block : #Protocol_client_context.full -> [`Highwatermarks] Baking_files.location -> - delegate:Tezos_crypto.Signature.public_key_hash -> + delegate:Signature.public_key_hash -> level:int32 -> round:Round.t -> bool tzresult Lwt.t @@ -53,7 +53,7 @@ val may_sign_block : val may_sign_preendorsement : #Protocol_client_context.full -> [`Highwatermarks] Baking_files.location -> - delegate:Tezos_crypto.Signature.public_key_hash -> + delegate:Signature.public_key_hash -> level:int32 -> round:Round.t -> bool tzresult Lwt.t @@ -61,7 +61,7 @@ val may_sign_preendorsement : val may_sign_endorsement : #Protocol_client_context.full -> [`Highwatermarks] Baking_files.location -> - delegate:Tezos_crypto.Signature.public_key_hash -> + delegate:Signature.public_key_hash -> level:int32 -> round:Round.t -> bool tzresult Lwt.t @@ -69,7 +69,7 @@ val may_sign_endorsement : val record_block : #Protocol_client_context.full -> [`Highwatermarks] Baking_files.location -> - delegate:Tezos_crypto.Signature.public_key_hash -> + delegate:Signature.public_key_hash -> level:int32 -> round:Round.t -> unit tzresult Lwt.t @@ -77,7 +77,7 @@ val record_block : val record_preendorsement : #Protocol_client_context.full -> [`Highwatermarks] Baking_files.location -> - delegate:Tezos_crypto.Signature.public_key_hash -> + delegate:Signature.public_key_hash -> level:int32 -> round:Round.t -> unit tzresult Lwt.t @@ -85,7 +85,7 @@ val record_preendorsement : val record_endorsement : #Protocol_client_context.full -> [`Highwatermarks] Baking_files.location -> - delegate:Tezos_crypto.Signature.public_key_hash -> + delegate:Signature.public_key_hash -> level:int32 -> round:Round.t -> unit tzresult Lwt.t diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_lib.ml b/src/proto_016_PtMumbai/lib_delegate/baking_lib.ml index 24ba15e8db03..5adbda789e44 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_lib.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_lib.ml @@ -86,17 +86,7 @@ let preendorse (cctxt : Protocol_client_context.full) ?(force = false) delegates Baking_state.pp_consensus_key_and_delegate) (List.map fst consensus_list) in - let state_recorder ~new_state = - Baking_state.may_record_new_state ~previous_state:state ~new_state - in - let* _ = - Baking_actions.inject_preendorsements - ~state_recorder - state - ~preendorsements:consensus_list - ~updated_state:state - in - return_unit + Baking_actions.inject_preendorsements state ~preendorsements:consensus_list let endorse (cctxt : Protocol_client_context.full) ?(force = false) delegates = let open State_transitions in @@ -126,17 +116,10 @@ let endorse (cctxt : Protocol_client_context.full) ?(force = false) delegates = Baking_state.pp_consensus_key_and_delegate) (List.map fst consensus_list) in - let state_recorder ~new_state = - Baking_state.may_record_new_state ~previous_state:state ~new_state - in - let* _ = - Baking_actions.inject_endorsements - ~state_recorder - state - ~endorsements:consensus_list - ~updated_state:state + let* () = + Baking_state.may_record_new_state ~previous_state:state ~new_state:state in - return_unit + Baking_actions.inject_endorsements state ~endorsements:consensus_list let bake_at_next_level state = let open Lwt_result_syntax in diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_nonces.ml b/src/proto_016_PtMumbai/lib_delegate/baking_nonces.ml index 5af73f61735e..fc1c9ada635a 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_nonces.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_nonces.ml @@ -239,9 +239,7 @@ let inject_seed_nonce_revelation (cctxt : #Protocol_client_context.full) ~chain ~nonce () >>=? fun bytes -> - let bytes = - Tezos_crypto.Signature.concat bytes Tezos_crypto.Signature.zero - in + let bytes = Signature.concat bytes Signature.zero in Shell_services.Injection.operation ~async:true cctxt ~chain bytes >>=? fun oph -> Events.( diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_scheduling.ml b/src/proto_016_PtMumbai/lib_delegate/baking_scheduling.ml index 38d42a06cc3d..96350fe3ba3e 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_scheduling.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_scheduling.ml @@ -68,9 +68,9 @@ let find_in_known_round_intervals known_round_intervals ~predecessor_timestamp {predecessor_timestamp; predecessor_round; time_interval = (now, now)}) (** Memoization wrapper for [Round.timestamp_of_round]. *) -let timestamp_of_round known_timestamps round_durations ~predecessor_timestamp - ~predecessor_round ~round = +let timestamp_of_round state ~predecessor_timestamp ~predecessor_round ~round = let open Baking_cache in + let known_timestamps = state.global_state.cache.known_timestamps in match Timestamp_of_round_cache.find_opt known_timestamps @@ -79,7 +79,7 @@ let timestamp_of_round known_timestamps round_durations ~predecessor_timestamp (* Compute and register the timestamp if not already existing. *) | None -> Protocol.Alpha_context.Round.timestamp_of_round - round_durations + state.global_state.round_durations ~predecessor_timestamp ~predecessor_round ~round @@ -220,32 +220,18 @@ let compute_next_round_time state = repropose a block at next level. *) None | None -> ( - let first_round_duration = - state.global_state.constants.parametric.minimal_block_delay - in - let delay_increment_per_round = - state.global_state.constants.parametric.delay_increment_per_round - in + let predecessor_timestamp = proposal.predecessor.shell.timestamp in + let predecessor_round = proposal.predecessor.round in + let next_round = Round.succ state.round_state.current_round in match - Round.Durations.create_opt - ~first_round_duration - ~delay_increment_per_round + timestamp_of_round + state + ~predecessor_timestamp + ~predecessor_round + ~round:next_round with - | Some round_durations -> ( - let predecessor_timestamp = proposal.predecessor.shell.timestamp in - let predecessor_round = proposal.predecessor.round in - let next_round = Round.succ state.round_state.current_round in - match - timestamp_of_round - state.global_state.cache.known_timestamps - round_durations - ~predecessor_timestamp - ~predecessor_round - ~round:next_round - with - | Ok timestamp -> Some (timestamp, next_round) - | _ -> assert false) - | None -> assert false) + | Ok timestamp -> Some (timestamp, next_round) + | _ -> assert false) (** [first_potential_round_at_next_level state ~earliest_round] yields an optional pair of the earliest possible round (at or after @@ -387,8 +373,7 @@ let compute_next_potential_baking_time_at_next_level state = >>= fun () -> Lwt.return_none | None | Some _ -> ( timestamp_of_round - state.global_state.cache.known_timestamps - round_durations + state ~predecessor_timestamp ~predecessor_round ~round:first_potential_round @@ -475,13 +460,14 @@ let compute_next_timeout state : Baking_state.timeout_kind Lwt.t tzresult Lwt.t let wait_baking_time_next_level (next_baking_time, next_baking_round) = let now = Time.System.now () in let delay = Ptime.diff (Time.System.of_protocol_exn next_baking_time) now in - Events.(emit waiting_time_to_bake (delay, next_baking_time)) >>= fun () -> match sleep_until next_baking_time with | None -> Events.(emit no_need_to_wait_for_proposal ()) >>= fun () -> return (Lwt.return (Time_to_bake_next_level {at_round = next_baking_round})) | Some t -> + Events.(emit waiting_time_to_bake (delay, next_baking_time)) + >>= fun () -> return ( t >>= fun () -> Lwt.return (Time_to_bake_next_level {at_round = next_baking_round}) @@ -652,13 +638,7 @@ let create_initial_state cctxt ?(synchronize = true) ~chain config } in (if synchronize then - let round_durations = - Stdlib.Option.get - @@ Round.Durations.create_opt - ~first_round_duration:constants.parametric.minimal_block_delay - ~delay_increment_per_round: - constants.parametric.delay_increment_per_round - in + create_round_durations constants >>? fun round_durations -> Baking_actions.compute_round current_proposal round_durations >>? fun current_round -> ok {current_round; current_phase = Idle} else ok {Baking_state.current_round = Round.zero; current_phase = Idle}) diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_state.ml b/src/proto_016_PtMumbai/lib_delegate/baking_state.ml index c5f6dcab2b20..8d7b97748316 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_state.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_state.ml @@ -31,8 +31,8 @@ open Protocol_client_context public key, its public key hash, and its secret key. *) type consensus_key = { alias : string option; - public_key : Tezos_crypto.Signature.Public_key.t; - public_key_hash : Tezos_crypto.Signature.Public_key_hash.t; + public_key : Signature.Public_key.t; + public_key_hash : Signature.Public_key_hash.t; secret_key_uri : Client_keys.sk_uri; } @@ -56,48 +56,39 @@ let consensus_key_encoding = }) (obj4 (req "alias" (option string)) - (req "public_key" Tezos_crypto.Signature.Public_key.encoding) - (req "public_key_hash" Tezos_crypto.Signature.Public_key_hash.encoding) + (req "public_key" Signature.Public_key.encoding) + (req "public_key_hash" Signature.Public_key_hash.encoding) (req "secret_key_uri" string)) let pp_consensus_key fmt {alias; public_key_hash; _} = match alias with - | None -> - Format.fprintf - fmt - "%a" - Tezos_crypto.Signature.Public_key_hash.pp - public_key_hash + | None -> Format.fprintf fmt "%a" Signature.Public_key_hash.pp public_key_hash | Some alias -> Format.fprintf fmt "%s (%a)" alias - Tezos_crypto.Signature.Public_key_hash.pp + Signature.Public_key_hash.pp public_key_hash -type consensus_key_and_delegate = - consensus_key * Tezos_crypto.Signature.Public_key_hash.t +type consensus_key_and_delegate = consensus_key * Signature.Public_key_hash.t let consensus_key_and_delegate_encoding = let open Data_encoding in merge_objs consensus_key_encoding - (obj1 (req "delegate" Tezos_crypto.Signature.Public_key_hash.encoding)) + (obj1 (req "delegate" Signature.Public_key_hash.encoding)) let pp_consensus_key_and_delegate fmt (consensus_key, delegate) = - if - Tezos_crypto.Signature.Public_key_hash.equal - consensus_key.public_key_hash - delegate - then pp_consensus_key fmt consensus_key + if Signature.Public_key_hash.equal consensus_key.public_key_hash delegate then + pp_consensus_key fmt consensus_key else Format.fprintf fmt "%a@,on behalf of %a" pp_consensus_key consensus_key - Tezos_crypto.Signature.Public_key_hash.pp + Signature.Public_key_hash.pp delegate type validation_mode = Node | Local of Abstract_context_index.t @@ -250,7 +241,7 @@ module SlotMap : Map.S with type key = Slot.t = Map.Make (Slot) list of slots (i.e., a list of position indexes in the slot map, in other words the list of rounds when it will be the proposer), and its endorsing power. *) -type endorsing_slot = {slots : Slot.t list; endorsing_power : int} +type endorsing_slot = {first_slot : Slot.t; endorsing_power : int} (* FIXME: determine if the slot map should contain all slots or just the first one *) @@ -358,6 +349,9 @@ type state = { type t = state +let update_current_phase state new_phase = + {state with round_state = {state.round_state with current_phase = new_phase}} + type timeout_kind = | End_of_round of {ending_round : Round.t} | Time_to_bake_next_level of {at_round : Round.t} @@ -626,7 +620,7 @@ module DelegateSet = struct type t = consensus_key let compare {public_key_hash = pkh; _} {public_key_hash = pkh'; _} = - Tezos_crypto.Signature.Public_key_hash.compare pkh pkh' + Signature.Public_key_hash.compare pkh pkh' end) let find_pkh pkh s = @@ -634,8 +628,8 @@ module DelegateSet = struct try iter (fun ({public_key_hash; _} as delegate) -> - if Tezos_crypto.Signature.Public_key_hash.equal pkh public_key_hash - then raise (Found delegate) + if Signature.Public_key_hash.equal pkh public_key_hash then + raise (Found delegate) else ()) s ; None @@ -654,7 +648,12 @@ let compute_delegate_slots (cctxt : Protocol_client_context.full) List.fold_left (fun (own_map, all_map) slot -> let {Plugin.RPC.Validators.consensus_key; delegate; slots; _} = slot in - let endorsing_slot = {endorsing_power = List.length slots; slots} in + let endorsing_slot = + { + endorsing_power = List.length slots; + first_slot = Stdlib.List.hd slots; + } + in let all_map = List.fold_left (fun all_map slot -> SlotMap.add slot endorsing_slot all_map) @@ -788,13 +787,13 @@ let pp_elected_block fmt {proposal; endorsement_qc} = proposal.block (List.length endorsement_qc) -let pp_endorsing_slot fmt (consensus_key_and_delegate, {slots; endorsing_power}) - = +let pp_endorsing_slot fmt + (consensus_key_and_delegate, {first_slot; endorsing_power}) = Format.fprintf fmt - "slots: @[[%a]@],@ delegate: %a,@ endorsing_power: %d" - Format.(pp_print_list ~pp_sep:pp_print_space Slot.pp) - slots + "slots: @[first_slot: %a@],@ delegate: %a,@ endorsing_power: %d" + Slot.pp + first_slot pp_consensus_key_and_delegate consensus_key_and_delegate endorsing_power diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_state.mli b/src/proto_016_PtMumbai/lib_delegate/baking_state.mli index 34ce3743c4ad..fef6d3ccc336 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_state.mli +++ b/src/proto_016_PtMumbai/lib_delegate/baking_state.mli @@ -28,8 +28,8 @@ open Alpha_context type consensus_key = { alias : string option; - public_key : Tezos_crypto.Signature.public_key; - public_key_hash : Tezos_crypto.Signature.public_key_hash; + public_key : Signature.public_key; + public_key_hash : Signature.public_key_hash; secret_key_uri : Client_keys.sk_uri; } @@ -37,8 +37,7 @@ val consensus_key_encoding : consensus_key Data_encoding.t val pp_consensus_key : Format.formatter -> consensus_key -> unit -type consensus_key_and_delegate = - consensus_key * Tezos_crypto.Signature.Public_key_hash.t +type consensus_key_and_delegate = consensus_key * Signature.Public_key_hash.t val consensus_key_and_delegate_encoding : consensus_key_and_delegate Data_encoding.t @@ -97,7 +96,7 @@ val round_of_shell_header : Block_header.shell_header -> Round.t tzresult module SlotMap : Map.S with type key = Slot.t -type endorsing_slot = {slots : Slot.t list; endorsing_power : int} +type endorsing_slot = {first_slot : Slot.t; endorsing_power : int} type delegate_slots = { own_delegate_slots : (consensus_key_and_delegate * endorsing_slot) SlotMap.t; @@ -147,6 +146,8 @@ type state = { type t = state +val update_current_phase : t -> phase -> t + type timeout_kind = | End_of_round of {ending_round : Round.t} | Time_to_bake_next_level of {at_round : Round.t} diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_vdf.ml b/src/proto_016_PtMumbai/lib_delegate/baking_vdf.ml index eaad6ce638af..d6ce4c580274 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_vdf.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_vdf.ml @@ -156,7 +156,7 @@ let inject_vdf_revelation cctxt hash chain_id solution = ~solution () in - let bytes = Tezos_crypto.Signature.concat bytes Tezos_crypto.Signature.zero in + let bytes = Signature.concat bytes Signature.zero in Shell_services.Injection.operation cctxt ~chain bytes (* Checks if the VDF setup saved in the state is equal to the one computed diff --git a/src/proto_016_PtMumbai/lib_delegate/block_forge.ml b/src/proto_016_PtMumbai/lib_delegate/block_forge.ml index fedc2f80510b..d3be206ac6b6 100644 --- a/src/proto_016_PtMumbai/lib_delegate/block_forge.ml +++ b/src/proto_016_PtMumbai/lib_delegate/block_forge.ml @@ -57,7 +57,7 @@ let forge_faked_protocol_data ?(payload_hash = Block_payload_hash.zero) proof_of_work_nonce = Baking_pow.empty_proof_of_work_nonce; liquidity_baking_toggle_vote; }; - signature = Tezos_crypto.Signature.zero; + signature = Signature.zero; } let convert_operation (op : packed_operation) : Tezos_base.Operation.t = @@ -485,7 +485,7 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id let unsigned_block_header = { Block_header.shell = shell_header; - protocol_data = {contents; signature = Tezos_crypto.Signature.zero}; + protocol_data = {contents; signature = Signature.zero}; } in return {unsigned_block_header; operations} diff --git a/src/proto_016_PtMumbai/lib_delegate/client_baking_denunciation.ml b/src/proto_016_PtMumbai/lib_delegate/client_baking_denunciation.ml index cb6f6a02e66e..2ebc4af5961e 100644 --- a/src/proto_016_PtMumbai/lib_delegate/client_baking_denunciation.ml +++ b/src/proto_016_PtMumbai/lib_delegate/client_baking_denunciation.ml @@ -40,7 +40,7 @@ module HLevel = Hashtbl.Make (struct end) (* Blocks are associated to the delegates who baked them *) -module Delegate_Map = Map.Make (Tezos_crypto.Signature.Public_key_hash) +module Delegate_Map = Map.Make (Signature.Public_key_hash) (* (pre)endorsements are associated to the slot they are injected with; we rely on the fact that there is a unique canonical slot @@ -175,9 +175,7 @@ let process_consensus_op (type kind) cctxt ~op2 () >>=? fun bytes -> - let bytes = - Tezos_crypto.Signature.concat bytes Tezos_crypto.Signature.zero - in + let bytes = Signature.concat bytes Signature.zero in let double_op_detected, double_op_denounced = Events.( match op_kind with @@ -303,9 +301,7 @@ let process_block (cctxt : #Protocol_client_context.full) state ~bh2 () >>=? fun bytes -> - let bytes = - Tezos_crypto.Signature.concat bytes Tezos_crypto.Signature.zero - in + let bytes = Signature.concat bytes Signature.zero in Events.(emit double_baking_detected) () >>= fun () -> Shell_services.Injection.operation cctxt ~chain bytes >>=? fun op_hash -> diff --git a/src/proto_016_PtMumbai/lib_delegate/operation_selection.ml b/src/proto_016_PtMumbai/lib_delegate/operation_selection.ml index f08add33e4c6..cc33ddbfca71 100644 --- a/src/proto_016_PtMumbai/lib_delegate/operation_selection.ml +++ b/src/proto_016_PtMumbai/lib_delegate/operation_selection.ml @@ -57,9 +57,7 @@ module PrioritizedManagerSet = Set.Make (struct let compare {source; counter; weight; op; _} {source = source'; counter = counter'; weight = weight'; op = op'; _} = (* Be careful with the [compare] *) - let cmp_src = - Tezos_crypto.Signature.Public_key_hash.compare source source' - in + let cmp_src = Signature.Public_key_hash.compare source source' in if cmp_src = 0 then (* we want the smallest counter first *) let c = Manager_counter.compare counter counter' in diff --git a/src/proto_016_PtMumbai/lib_delegate/state_transitions.ml b/src/proto_016_PtMumbai/lib_delegate/state_transitions.ml index 6e8dd27e6284..d39c05dbc8cd 100644 --- a/src/proto_016_PtMumbai/lib_delegate/state_transitions.ml +++ b/src/proto_016_PtMumbai/lib_delegate/state_transitions.ml @@ -81,7 +81,7 @@ let make_consensus_list state proposal = SlotMap.fold (fun _slot (consensus_key_and_delegate, slots) acc -> ( consensus_key_and_delegate, - {slot = Stdlib.List.hd slots.slots; level; round; block_payload_hash} ) + {slot = slots.first_slot; level; round; block_payload_hash} ) :: acc) state.level_state.delegate_slots.own_delegate_slots [] @@ -90,16 +90,10 @@ let make_consensus_list state proposal = (* If we do not have any slots, we won't inject any operation but we will still participate to determine an elected block *) let make_preendorse_action state proposal = - let updated_state = - let round_state = - {state.round_state with current_phase = Awaiting_preendorsements} - in - {state with round_state} - in let preendorsements : (consensus_key_and_delegate * consensus_content) list = make_consensus_list state proposal in - Inject_preendorsements {preendorsements; updated_state} + Inject_preendorsements {preendorsements} let update_proposal state proposal = Events.(emit updating_latest_proposal proposal.block.hash) >>= fun () -> @@ -120,12 +114,13 @@ let preendorse state proposal = if Protocol_hash.(proposal.block.protocol <> proposal.block.next_protocol) then (* We do not preendorse the first transition block *) - let new_round_state = {state.round_state with current_phase = Idle} in - let new_state = {state with round_state = new_round_state} in + let new_state = update_current_phase state Idle in Lwt.return (new_state, Do_nothing) else Events.(emit attempting_preendorse_proposal proposal.block.hash) - >>= fun () -> Lwt.return (state, make_preendorse_action state proposal) + >>= fun () -> + let new_state = update_current_phase state Awaiting_preendorsements in + Lwt.return (new_state, make_preendorse_action state proposal) let extract_pqc state (new_proposal : proposal) = match new_proposal.block.prequorum with @@ -247,7 +242,13 @@ let rec handle_new_proposal state (new_proposal : proposal) = | Some {round; _} when Round.(locked_round.round < round) -> (* This PQC is above our locked_round, we can preendorse it *) preendorse new_state new_proposal - | _ -> Lwt.return (new_state, Do_nothing)) + | _ -> + (* We shouldn't preendorse this proposal, but we should at + least watch (pre)quorums events on it *) + let new_state = + update_current_phase new_state Awaiting_preendorsements + in + Lwt.return (new_state, Watch_proposal)) | None -> (* Otherwise, we did not lock on any payload, thus we can preendorse it *) @@ -413,10 +414,7 @@ let propose_fresh_block_action ~endorsements ?last_proposal let kind = Fresh operation_pool in Events.(emit proposing_fresh_block (delegate, round)) >>= fun () -> let block_to_bake = {predecessor; round; delegate; kind} in - let updated_state = - let new_round_state = {state.round_state with current_phase = Idle} in - {state with round_state = new_round_state} - in + let updated_state = update_current_phase state Idle in Lwt.return @@ Inject_block {block_to_bake; updated_state} let propose_block_action state delegate round (proposal : proposal) = @@ -503,10 +501,7 @@ let propose_block_action state delegate round (proposal : proposal) = let block_to_bake = {predecessor = proposal.predecessor; round; delegate; kind} in - let updated_state = - let new_round_state = {state.round_state with current_phase = Idle} in - {state with round_state = new_round_state} - in + let updated_state = update_current_phase state Idle in Lwt.return @@ Inject_block {block_to_bake; updated_state} let end_of_round state current_round = @@ -529,8 +524,7 @@ let end_of_round state current_round = (* We don't have any delegate that may propose a new block for this round -- We will wait for preendorsements when the next level block arrive. Meanwhile, we are idle *) - let new_round_state = {new_state.round_state with current_phase = Idle} in - let new_state = {state with round_state = new_round_state} in + let new_state = update_current_phase new_state Idle in do_nothing new_state | Some (delegate, _) -> let last_proposal = state.level_state.latest_proposal.block in @@ -588,20 +582,10 @@ let update_locked_round state round payload_hash = {state with level_state = new_level_state} let make_endorse_action state proposal = - let updated_state = - let new_round_state = - {state.round_state with current_phase = Awaiting_endorsements} - in - let new_state = {state with round_state = new_round_state} in - update_locked_round - new_state - proposal.block.round - proposal.block.payload_hash - in let endorsements : (consensus_key_and_delegate * consensus_content) list = make_consensus_list state proposal in - Inject_endorsements {endorsements; updated_state} + Inject_endorsements {endorsements} let prequorum_reached_when_awaiting_preendorsements state candidate preendorsements = @@ -642,6 +626,13 @@ let prequorum_reached_when_awaiting_preendorsements state candidate else state.level_state in let new_state = {state with level_state = new_level_state} in + let new_state = + update_locked_round + new_state + latest_proposal.block.round + latest_proposal.block.payload_hash + in + let new_state = update_current_phase new_state Awaiting_endorsements in Lwt.return (new_state, make_endorse_action new_state latest_proposal) let quorum_reached_when_waiting_endorsements state candidate endorsement_qc = @@ -674,6 +665,9 @@ let quorum_reached_when_waiting_endorsements state candidate endorsement_qc = (* Hypothesis: - The state is not to be modified outside this module + (NB: there are exceptions in Baking_actions: the corner cases + [update_to_level] and [synchronize_round] and + the hack used by [inject_block]) - new_proposal's received blocks are expected to belong to our current round diff --git a/src/proto_016_PtMumbai/lib_delegate/state_transitions.mli b/src/proto_016_PtMumbai/lib_delegate/state_transitions.mli index 9dde0efaf02b..adc584312329 100644 --- a/src/proto_016_PtMumbai/lib_delegate/state_transitions.mli +++ b/src/proto_016_PtMumbai/lib_delegate/state_transitions.mli @@ -23,6 +23,9 @@ (* *) (*****************************************************************************) +(** This module, and in particular the {!step} function, modifies the automaton + state, while {!Baking_actions} performs potentially failing side-effects. *) + open Protocol open Alpha_context open Baking_state diff --git a/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/mockup_simulator.ml b/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/mockup_simulator.ml index 4cadea890daa..346cea3ee712 100644 --- a/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/mockup_simulator.ml +++ b/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/mockup_simulator.ml @@ -842,7 +842,7 @@ let baker_process ~(delegates : Baking_state.consensus_key list) ~base_dir Lwt.pick [listener_process (); baker_process ()] >>=? fun () -> User_hooks.check_chain_on_success ~chain:state.chain -let genesis_protocol_data (baker_sk : Tezos_crypto.Signature.secret_key) +let genesis_protocol_data (baker_sk : Signature.secret_key) (predecessor_hash : Block_hash.t) (block_header : Block_header.shell_header) : Bytes.t = let proof_of_work_nonce = @@ -871,7 +871,7 @@ let genesis_protocol_data (baker_sk : Tezos_crypto.Signature.secret_key) (block_header, contents) in let signature = - Tezos_crypto.Signature.sign + Signature.sign ~watermark: Alpha_context.Block_header.(to_watermark (Block_header chain_id)) baker_sk @@ -887,7 +887,7 @@ let deduce_baker_sk (Protocol.Alpha_context.Parameters.bootstrap_account * Tezos_mockup_commands.Mockup_wallet.bootstrap_secret) list) (total_accounts : int) (level : int) : - Tezos_crypto.Signature.secret_key tzresult Lwt.t = + Signature.secret_key tzresult Lwt.t = (match (total_accounts, level) with | _, 0 -> return 0 (* apparently this doesn't really matter *) | _ -> @@ -902,8 +902,7 @@ let deduce_baker_sk |> WithExceptions.Option.get ~loc:__LOC__ in let secret_key = - Tezos_crypto.Signature.Secret_key.of_b58check_exn - (Uri.path (secret.sk_uri :> Uri.t)) + Signature.Secret_key.of_b58check_exn (Uri.path (secret.sk_uri :> Uri.t)) in return secret_key @@ -1078,8 +1077,7 @@ type config = { round0 : int64; round1 : int64; timeout : int; - delegate_selection : - (int32 * (int32 * Tezos_crypto.Signature.public_key_hash) list) list; + delegate_selection : (int32 * (int32 * Signature.public_key_hash) list) list; initial_seed : State_hash.t option; consensus_committee_size : int; consensus_threshold : int; @@ -1215,7 +1213,7 @@ let check_block_signature ~block_hash ~(block_header : Block_header.t) (block_header.shell, protocol_data.contents) in if - Tezos_crypto.Signature.check + Signature.check ~watermark: Alpha_context.Block_header.(to_watermark (Block_header chain_id)) public_key @@ -1227,7 +1225,7 @@ let check_block_signature ~block_hash ~(block_header : Block_header.t) "unexpected signature for %a; tried with %a@." Block_hash.pp block_hash - Tezos_crypto.Signature.Public_key.pp + Signature.Public_key.pp public_key type op_predicate = @@ -1262,7 +1260,7 @@ let op_is_signed_by ~public_key (op_hash : Operation_hash.t) Alpha_context.Operation.to_watermark (Endorsement chain_id) | Preendorsement _ -> Alpha_context.Operation.to_watermark (Preendorsement chain_id) - | _ -> Tezos_crypto.Signature.Generic_operation) + | _ -> Signature.Generic_operation) | _ -> failwith "unexpected contents in %a@." Operation_hash.pp op_hash) >>=? fun watermark -> match d.signature with @@ -1278,7 +1276,7 @@ let op_is_signed_by ~public_key (op_hash : Operation_hash.t) (op.shell, Contents_list d.contents) in return - (Tezos_crypto.Signature.check + (Signature.check ~watermark public_key signature diff --git a/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/mockup_simulator.mli b/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/mockup_simulator.mli index 2fc5f544597a..c01782653bdf 100644 --- a/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/mockup_simulator.mli +++ b/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/mockup_simulator.mli @@ -139,8 +139,7 @@ type config = { (** Maximal duration of the test. If the test takes longer to terminate it'll be aborted with an error. *) - delegate_selection : - (int32 * (int32 * Tezos_crypto.Signature.public_key_hash) list) list; + delegate_selection : (int32 * (int32 * Signature.public_key_hash) list) list; (** Desired selection of delegates per level/round *) initial_seed : State_hash.t option; (** Optional initial seed for protocol (used to control delegate selection) *) @@ -172,21 +171,21 @@ val default_config : config to the final result. *) val run : ?config:config -> (int * (module Hooks)) list -> unit tzresult Lwt.t -val bootstrap1 : Tezos_crypto.Signature.public_key +val bootstrap1 : Signature.public_key -val bootstrap2 : Tezos_crypto.Signature.public_key +val bootstrap2 : Signature.public_key -val bootstrap3 : Tezos_crypto.Signature.public_key +val bootstrap3 : Signature.public_key -val bootstrap4 : Tezos_crypto.Signature.public_key +val bootstrap4 : Signature.public_key -val bootstrap5 : Tezos_crypto.Signature.public_key +val bootstrap5 : Signature.public_key (** Check if a block header is signed by a given delegate. *) val check_block_signature : block_hash:Block_hash.t -> block_header:Block_header.t -> - public_key:Tezos_crypto.Signature.public_key -> + public_key:Signature.public_key -> unit tzresult Lwt.t (** A shortcut type for predicates on operations. *) @@ -216,8 +215,7 @@ val mempool_has_op_ref : unit tzresult Lwt.t (** Check if an operation is signed by the given delegate. *) -val op_is_signed_by : - public_key:Tezos_crypto.Signature.public_key -> op_predicate +val op_is_signed_by : public_key:Signature.public_key -> op_predicate (** Check that an operation is a preendorsement. *) val op_is_preendorsement : ?level:int32 -> ?round:int32 -> op_predicate diff --git a/src/proto_016_PtMumbai/lib_delegate/test/tenderbrute/lib/tenderbrute.ml b/src/proto_016_PtMumbai/lib_delegate/test/tenderbrute/lib/tenderbrute.ml index fdc0a6dc5abb..ce3f8bba6380 100644 --- a/src/proto_016_PtMumbai/lib_delegate/test/tenderbrute/lib/tenderbrute.ml +++ b/src/proto_016_PtMumbai/lib_delegate/test/tenderbrute/lib/tenderbrute.ml @@ -26,9 +26,7 @@ open Protocol type delegate_selection = - (Raw_level_repr.t - * (Round_repr.t * Tezos_crypto.Signature.public_key_hash) list) - list + (Raw_level_repr.t * (Round_repr.t * Signature.public_key_hash) list) list module LevelRoundMap = Map.Make (struct type t = Level_repr.t * Round_repr.t @@ -105,12 +103,8 @@ let check ctxt ~selection = Delegate_sampler.baking_rights_owner ctxt level ~round >|= Environment.wrap_tzresult >>=? fun (ctxt, _, pk) -> - if - not - (Tezos_crypto.Signature.Public_key_hash.equal - delegate - pk.delegate) - then raise Exit + if not (Signature.Public_key_hash.equal delegate pk.delegate) then + raise Exit else return ctxt) selection ctxt diff --git a/src/proto_016_PtMumbai/lib_delegate/test/tenderbrute/lib/tenderbrute.mli b/src/proto_016_PtMumbai/lib_delegate/test/tenderbrute/lib/tenderbrute.mli index c13cc23edb60..7b6915d2cfd2 100644 --- a/src/proto_016_PtMumbai/lib_delegate/test/tenderbrute/lib/tenderbrute.mli +++ b/src/proto_016_PtMumbai/lib_delegate/test/tenderbrute/lib/tenderbrute.mli @@ -29,9 +29,7 @@ open Protocol one can provide a public key hash that would be the proposer. All non- specified level and rounds are not constrained. *) type delegate_selection = - (Raw_level_repr.t - * (Round_repr.t * Tezos_crypto.Signature.public_key_hash) list) - list + (Raw_level_repr.t * (Round_repr.t * Signature.public_key_hash) list) list (** Brute-force an initial seed nonce for the desired delegate selection. When found, the seed nonce is returned as a byte sequence of size 32. If diff --git a/src/proto_016_PtMumbai/lib_delegate/test/tenderbrute/tenderbrute_main.ml b/src/proto_016_PtMumbai/lib_delegate/test/tenderbrute/tenderbrute_main.ml index 123db4e59248..f3558b1a1985 100644 --- a/src/proto_016_PtMumbai/lib_delegate/test/tenderbrute/tenderbrute_main.ml +++ b/src/proto_016_PtMumbai/lib_delegate/test/tenderbrute/tenderbrute_main.ml @@ -49,7 +49,7 @@ let delegate_encoding = case ~title:"Public key hash" (Tag 0) - Tezos_crypto.Signature.Public_key_hash.encoding + Signature.Public_key_hash.encoding (function `Pkh p -> Some p | _ -> None) (fun p -> `Pkh p); case diff --git a/src/proto_016_PtMumbai/lib_delegate/test/test_scenario.ml b/src/proto_016_PtMumbai/lib_delegate/test/test_scenario.ml index f10b60904c24..cb34cc8d3d63 100644 --- a/src/proto_016_PtMumbai/lib_delegate/test/test_scenario.ml +++ b/src/proto_016_PtMumbai/lib_delegate/test/test_scenario.ml @@ -1,14 +1,14 @@ open Mockup_simulator -let bootstrap1 = Tezos_crypto.Signature.Public_key.hash bootstrap1 +let bootstrap1 = Signature.Public_key.hash bootstrap1 -let bootstrap2 = Tezos_crypto.Signature.Public_key.hash bootstrap2 +let bootstrap2 = Signature.Public_key.hash bootstrap2 -let bootstrap3 = Tezos_crypto.Signature.Public_key.hash bootstrap3 +let bootstrap3 = Signature.Public_key.hash bootstrap3 -let bootstrap4 = Tezos_crypto.Signature.Public_key.hash bootstrap4 +let bootstrap4 = Signature.Public_key.hash bootstrap4 -let bootstrap5 = Tezos_crypto.Signature.Public_key.hash bootstrap5 +let bootstrap5 = Signature.Public_key.hash bootstrap5 let some_seed s = Some (Protocol.State_hash.of_b58check_exn s) @@ -401,6 +401,14 @@ let test_scenario_f1 () = let d_proposed_l1_r1 = ref false in let a_proposed_l2_r0 = ref false in let stop_on_event0 _ = !a_proposed_l2_r0 in + let pass = + (* This is to be sure that the proposal for round r arrives while + the baker is in round r, and not in round r-1. (See related + issue: + https://gitlab.com/tezos/tezos/-/issues/4143). Otherwise, the + test does not perform as expected. *) + Delay 0.5 + in let module Node_a_hooks : Hooks = struct include Default_hooks @@ -416,7 +424,7 @@ let test_scenario_f1 () = (a_proposed_l2_r0 := true ; return_unit) >>=? fun () -> - return (block_hash, block_header, operations, [Pass; Pass; Pass; Pass]) + return (block_hash, block_header, operations, [pass; pass; pass; pass]) | _ -> failwith "unexpected injection on the node A, level = %ld / round = %ld" @@ -455,7 +463,7 @@ let test_scenario_f1 () = (c_proposed_l1_r0 := true ; return_unit) >>=? fun () -> - return (block_hash, block_header, operations, [Pass; Pass; Pass; Pass]) + return (block_hash, block_header, operations, [pass; pass; pass; pass]) | _ -> failwith "unexpected injection on the node C, level = %ld / round = %ld" @@ -484,7 +492,7 @@ let test_scenario_f1 () = (d_proposed_l1_r1 := true ; return_unit) >>=? fun () -> - return (block_hash, block_header, operations, [Pass; Pass; Pass; Pass]) + return (block_hash, block_header, operations, [pass; pass; pass; pass]) | _ -> failwith "unexpected injection on the node D, level = %ld / round = %ld" @@ -1465,8 +1473,7 @@ let tests = tztest "scenario t1" `Quick test_scenario_t1; tztest "scenario t2" `Quick test_scenario_t2; tztest "scenario t3" `Quick test_scenario_t3; - (* See issue https://gitlab.com/nomadic-labs/tezos/-/issues/518 *) - (* tztest "scenario f1" `Quick test_scenario_f1; *) + tztest "scenario f1" `Quick test_scenario_f1; tztest "scenario f2" `Quick test_scenario_f2; tztest "scenario m1" `Quick test_scenario_m1; tztest "scenario m2" `Quick test_scenario_m2; -- GitLab From d7e481c53c1aee2eb6369bd76622db6f1b703212 Mon Sep 17 00:00:00 2001 From: Albin Coquereau Date: Mon, 13 Feb 2023 13:52:00 +0100 Subject: [PATCH 27/46] baker: apply operations on round greater than zero --- .../lib_delegate/baking_actions.ml | 11 +++++++++-- .../lib_delegate/baking_actions.mli | 5 +++++ src/proto_016_PtMumbai/lib_delegate/baking_lib.ml | 2 ++ .../lib_delegate/state_transitions.ml | 14 ++++++++++++-- src/proto_alpha/lib_delegate/baking_actions.ml | 11 +++++++++-- src/proto_alpha/lib_delegate/baking_actions.mli | 5 +++++ src/proto_alpha/lib_delegate/baking_lib.ml | 2 ++ src/proto_alpha/lib_delegate/state_transitions.ml | 14 ++++++++++++-- 8 files changed, 56 insertions(+), 8 deletions(-) diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_actions.ml b/src/proto_016_PtMumbai/lib_delegate/baking_actions.ml index cc6dfc5b54eb..588190fb1fa5 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_actions.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_actions.ml @@ -119,6 +119,7 @@ type block_to_bake = { round : Round.t; delegate : Baking_state.consensus_key_and_delegate; kind : block_kind; + force_apply : bool; } type action = @@ -214,7 +215,13 @@ let sign_block_header state proposer unsigned_block_header = return {Block_header.shell; protocol_data = {contents; signature}} let inject_block ~state_recorder state block_to_bake ~updated_state = - let {predecessor; round; delegate = (consensus_key, _) as delegate; kind} = + let { + predecessor; + round; + delegate = (consensus_key, _) as delegate; + kind; + force_apply; + } = block_to_bake in let cctxt = state.global_state.cctxt in @@ -306,7 +313,7 @@ let inject_block ~state_recorder state block_to_bake ~updated_state = ~payload_round ~liquidity_baking_toggle_vote ~user_activated_upgrades - ~force_apply:state.global_state.config.force_apply + ~force_apply state.global_state.config.fees simulation_mode simulation_kind diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_actions.mli b/src/proto_016_PtMumbai/lib_delegate/baking_actions.mli index 91b9bf772915..5b9715905497 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_actions.mli +++ b/src/proto_016_PtMumbai/lib_delegate/baking_actions.mli @@ -41,6 +41,11 @@ type block_to_bake = { round : Round.t; delegate : consensus_key_and_delegate; kind : block_kind; + force_apply : bool; + (** if true, while baking the block, try and apply the block and its + operations instead of only validating them. this can be permanently + set using the [--force-apply] flag (see [force_apply_switch_arg] in + [baking_commands.ml]). *) } type action = diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_lib.ml b/src/proto_016_PtMumbai/lib_delegate/baking_lib.ml index 5adbda789e44..ebb2cd35fc0c 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_lib.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_lib.ml @@ -235,6 +235,7 @@ let propose_at_next_level ~minimal_timestamp state = round = minimal_round; delegate; kind; + force_apply = state.global_state.config.force_apply; } in let state_recorder ~new_state = @@ -476,6 +477,7 @@ let baking_minimal_timestamp state = round = minimal_round; delegate; kind; + force_apply = state.global_state.config.force_apply; } in let state_recorder ~new_state = diff --git a/src/proto_016_PtMumbai/lib_delegate/state_transitions.ml b/src/proto_016_PtMumbai/lib_delegate/state_transitions.ml index d39c05dbc8cd..5cc55044263d 100644 --- a/src/proto_016_PtMumbai/lib_delegate/state_transitions.ml +++ b/src/proto_016_PtMumbai/lib_delegate/state_transitions.ml @@ -413,7 +413,12 @@ let propose_fresh_block_action ~endorsements ?last_proposal in let kind = Fresh operation_pool in Events.(emit proposing_fresh_block (delegate, round)) >>= fun () -> - let block_to_bake = {predecessor; round; delegate; kind} in + let force_apply = + state.global_state.config.force_apply || Round.(round <> zero) + (* This is used as a safety net by applying blocks on round > 0, in case + validation-only did not produce a correct round-0 block. *) + in + let block_to_bake = {predecessor; round; delegate; kind; force_apply} in let updated_state = update_current_phase state Idle in Lwt.return @@ Inject_block {block_to_bake; updated_state} @@ -498,8 +503,13 @@ let propose_block_action state delegate round (proposal : proposal) = let kind = Reproposal {consensus_operations; payload_hash; payload_round; payload} in + let force_apply = + true + (* This is used as a safety net by applying blocks on round > 0, in case + validation-only did not produce a correct round-0 block. *) + in let block_to_bake = - {predecessor = proposal.predecessor; round; delegate; kind} + {predecessor = proposal.predecessor; round; delegate; kind; force_apply} in let updated_state = update_current_phase state Idle in Lwt.return @@ Inject_block {block_to_bake; updated_state} diff --git a/src/proto_alpha/lib_delegate/baking_actions.ml b/src/proto_alpha/lib_delegate/baking_actions.ml index e05821d643bc..b24a54100308 100644 --- a/src/proto_alpha/lib_delegate/baking_actions.ml +++ b/src/proto_alpha/lib_delegate/baking_actions.ml @@ -119,6 +119,7 @@ type block_to_bake = { round : Round.t; delegate : Baking_state.consensus_key_and_delegate; kind : block_kind; + force_apply : bool; } type action = @@ -214,7 +215,13 @@ let sign_block_header state proposer unsigned_block_header = return {Block_header.shell; protocol_data = {contents; signature}} let inject_block ~state_recorder state block_to_bake ~updated_state = - let {predecessor; round; delegate = (consensus_key, _) as delegate; kind} = + let { + predecessor; + round; + delegate = (consensus_key, _) as delegate; + kind; + force_apply; + } = block_to_bake in Events.( @@ -311,7 +318,7 @@ let inject_block ~state_recorder state block_to_bake ~updated_state = ~payload_round ~liquidity_baking_toggle_vote ~user_activated_upgrades - ~force_apply:state.global_state.config.force_apply + ~force_apply state.global_state.config.fees simulation_mode simulation_kind diff --git a/src/proto_alpha/lib_delegate/baking_actions.mli b/src/proto_alpha/lib_delegate/baking_actions.mli index 91b9bf772915..5b9715905497 100644 --- a/src/proto_alpha/lib_delegate/baking_actions.mli +++ b/src/proto_alpha/lib_delegate/baking_actions.mli @@ -41,6 +41,11 @@ type block_to_bake = { round : Round.t; delegate : consensus_key_and_delegate; kind : block_kind; + force_apply : bool; + (** if true, while baking the block, try and apply the block and its + operations instead of only validating them. this can be permanently + set using the [--force-apply] flag (see [force_apply_switch_arg] in + [baking_commands.ml]). *) } type action = diff --git a/src/proto_alpha/lib_delegate/baking_lib.ml b/src/proto_alpha/lib_delegate/baking_lib.ml index 5adbda789e44..ebb2cd35fc0c 100644 --- a/src/proto_alpha/lib_delegate/baking_lib.ml +++ b/src/proto_alpha/lib_delegate/baking_lib.ml @@ -235,6 +235,7 @@ let propose_at_next_level ~minimal_timestamp state = round = minimal_round; delegate; kind; + force_apply = state.global_state.config.force_apply; } in let state_recorder ~new_state = @@ -476,6 +477,7 @@ let baking_minimal_timestamp state = round = minimal_round; delegate; kind; + force_apply = state.global_state.config.force_apply; } in let state_recorder ~new_state = diff --git a/src/proto_alpha/lib_delegate/state_transitions.ml b/src/proto_alpha/lib_delegate/state_transitions.ml index d39c05dbc8cd..5cc55044263d 100644 --- a/src/proto_alpha/lib_delegate/state_transitions.ml +++ b/src/proto_alpha/lib_delegate/state_transitions.ml @@ -413,7 +413,12 @@ let propose_fresh_block_action ~endorsements ?last_proposal in let kind = Fresh operation_pool in Events.(emit proposing_fresh_block (delegate, round)) >>= fun () -> - let block_to_bake = {predecessor; round; delegate; kind} in + let force_apply = + state.global_state.config.force_apply || Round.(round <> zero) + (* This is used as a safety net by applying blocks on round > 0, in case + validation-only did not produce a correct round-0 block. *) + in + let block_to_bake = {predecessor; round; delegate; kind; force_apply} in let updated_state = update_current_phase state Idle in Lwt.return @@ Inject_block {block_to_bake; updated_state} @@ -498,8 +503,13 @@ let propose_block_action state delegate round (proposal : proposal) = let kind = Reproposal {consensus_operations; payload_hash; payload_round; payload} in + let force_apply = + true + (* This is used as a safety net by applying blocks on round > 0, in case + validation-only did not produce a correct round-0 block. *) + in let block_to_bake = - {predecessor = proposal.predecessor; round; delegate; kind} + {predecessor = proposal.predecessor; round; delegate; kind; force_apply} in let updated_state = update_current_phase state Idle in Lwt.return @@ Inject_block {block_to_bake; updated_state} -- GitLab From dc77bc2e89087484d1cb018500745d9a42e7cb89 Mon Sep 17 00:00:00 2001 From: Albin Coquereau Date: Mon, 13 Feb 2023 10:19:16 +0100 Subject: [PATCH 28/46] Changes: add entry --- CHANGES.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index ad58e1ce5630..c398fcc8e77b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -196,6 +196,10 @@ Baker states in order to avoid unexpected crashes when the baker gets updated, or when a new protocol's baker starts. (MR :gl:`!7640`) +- Restored previous behaviour from :gl:`!7490` for blocks at round + greater than 0. Application-dependent checks are re-enabled for + re-proposal and fresh blocks at round greater than 0. + Accuser ------- -- GitLab From 62fe384c68678b5d1960757d4e764ba3062c15d6 Mon Sep 17 00:00:00 2001 From: vbot Date: Wed, 25 Jan 2023 17:53:09 +0100 Subject: [PATCH 29/46] Alpha/Baker: remove unused voting power field --- src/proto_alpha/lib_delegate/baking_lib.ml | 5 +-- .../lib_delegate/baking_scheduling.ml | 14 +++--- src/proto_alpha/lib_delegate/baking_state.ml | 44 +++++++------------ src/proto_alpha/lib_delegate/baking_state.mli | 10 +---- .../lib_delegate/operation_worker.ml | 9 +--- .../lib_delegate/operation_worker.mli | 7 +-- .../lib_delegate/state_transitions.ml | 7 ++- 7 files changed, 32 insertions(+), 64 deletions(-) diff --git a/src/proto_alpha/lib_delegate/baking_lib.ml b/src/proto_alpha/lib_delegate/baking_lib.ml index ebb2cd35fc0c..97905dcc5e59 100644 --- a/src/proto_alpha/lib_delegate/baking_lib.ml +++ b/src/proto_alpha/lib_delegate/baking_lib.ml @@ -301,7 +301,7 @@ let propose (cctxt : Protocol_client_context.full) ?minimal_fees | Some _ -> propose_at_next_level ~minimal_timestamp state | None -> ( match endorsement_quorum state with - | Some (voting_power, endorsement_qc) -> + | Some (_voting_power, endorsement_qc) -> let state = { state with @@ -323,8 +323,7 @@ let propose (cctxt : Protocol_client_context.full) ?minimal_fees let* state = State_transitions.step state - (Baking_state.Quorum_reached - (candidate, voting_power, endorsement_qc)) + (Baking_state.Quorum_reached (candidate, endorsement_qc)) >>= do_action (* this will register the elected block *) in diff --git a/src/proto_alpha/lib_delegate/baking_scheduling.ml b/src/proto_alpha/lib_delegate/baking_scheduling.ml index 96350fe3ba3e..5525a2b00431 100644 --- a/src/proto_alpha/lib_delegate/baking_scheduling.ml +++ b/src/proto_alpha/lib_delegate/baking_scheduling.ml @@ -187,18 +187,14 @@ let rec wait_next_event ~timeout loop_state = loop_state.last_future_block_event <- None ; return_some (New_proposal proposal) | `QC_reached - (Some - (Operation_worker.Prequorum_reached - (candidate, voting_power, preendorsement_qc))) -> + (Some (Operation_worker.Prequorum_reached (candidate, preendorsement_qc))) + -> loop_state.last_get_qc_event <- None ; - return_some - (Prequorum_reached (candidate, voting_power, preendorsement_qc)) + return_some (Prequorum_reached (candidate, preendorsement_qc)) | `QC_reached - (Some - (Operation_worker.Quorum_reached - (candidate, voting_power, endorsement_qc))) -> + (Some (Operation_worker.Quorum_reached (candidate, endorsement_qc))) -> loop_state.last_get_qc_event <- None ; - return_some (Quorum_reached (candidate, voting_power, endorsement_qc)) + return_some (Quorum_reached (candidate, endorsement_qc)) | `Timeout e -> return_some (Timeout e) (** From the current [state], the function returns an optional diff --git a/src/proto_alpha/lib_delegate/baking_state.ml b/src/proto_alpha/lib_delegate/baking_state.ml index 8d7b97748316..162a3f7fa3a4 100644 --- a/src/proto_alpha/lib_delegate/baking_state.ml +++ b/src/proto_alpha/lib_delegate/baking_state.ml @@ -376,18 +376,12 @@ let timeout_kind_encoding = (fun at_round -> Time_to_bake_next_level {at_round}); ] -type voting_power = int - type event = | New_proposal of proposal | Prequorum_reached of - Operation_worker.candidate - * voting_power - * Kind.preendorsement operation list + Operation_worker.candidate * Kind.preendorsement operation list | Quorum_reached of - Operation_worker.candidate - * voting_power - * Kind.endorsement operation list + Operation_worker.candidate * Kind.endorsement operation list | Timeout of timeout_kind let event_encoding = @@ -403,31 +397,28 @@ let event_encoding = case (Tag 1) ~title:"Prequorum_reached" - (tup3 + (tup2 Operation_worker.candidate_encoding - Data_encoding.int31 (Data_encoding.list (dynamic_size Operation.encoding))) (function - | Prequorum_reached (candidate, voting_power, ops) -> - Some (candidate, voting_power, List.map Operation.pack ops) + | Prequorum_reached (candidate, ops) -> + Some (candidate, List.map Operation.pack ops) | _ -> None) - (fun (candidate, voting_power, ops) -> + (fun (candidate, ops) -> Prequorum_reached - (candidate, voting_power, Operation_pool.filter_preendorsements ops)); + (candidate, Operation_pool.filter_preendorsements ops)); case (Tag 2) ~title:"Quorum_reached" - (tup3 + (tup2 Operation_worker.candidate_encoding - Data_encoding.int31 (Data_encoding.list (dynamic_size Operation.encoding))) (function - | Quorum_reached (candidate, voting_power, ops) -> - Some (candidate, voting_power, List.map Operation.pack ops) + | Quorum_reached (candidate, ops) -> + Some (candidate, List.map Operation.pack ops) | _ -> None) - (fun (candidate, voting_power, ops) -> - Quorum_reached - (candidate, voting_power, Operation_pool.filter_endorsements ops)); + (fun (candidate, ops) -> + Quorum_reached (candidate, Operation_pool.filter_endorsements ops)); case (Tag 3) ~title:"Timeout" @@ -884,23 +875,20 @@ let pp_event fmt = function "new proposal received: %a" pp_block_info proposal.block - | Prequorum_reached (candidate, voting_power, preendos) -> + | Prequorum_reached (candidate, preendos) -> Format.fprintf fmt - "pre-quorum reached with %d preendorsements (power: %d) for %a at \ - round %a" + "prequorum reached with %d preendorsements for %a at round %a" (List.length preendos) - voting_power Block_hash.pp candidate.Operation_worker.hash Round.pp candidate.round_watched - | Quorum_reached (candidate, voting_power, endos) -> + | Quorum_reached (candidate, endos) -> Format.fprintf fmt - "quorum reached with %d endorsements (power: %d) for %a at round %a" + "quorum reached with %d endorsements for %a at round %a" (List.length endos) - voting_power Block_hash.pp candidate.Operation_worker.hash Round.pp diff --git a/src/proto_alpha/lib_delegate/baking_state.mli b/src/proto_alpha/lib_delegate/baking_state.mli index fef6d3ccc336..45ea1d3392df 100644 --- a/src/proto_alpha/lib_delegate/baking_state.mli +++ b/src/proto_alpha/lib_delegate/baking_state.mli @@ -154,18 +154,12 @@ type timeout_kind = val timeout_kind_encoding : timeout_kind Data_encoding.t -type voting_power = int - type event = | New_proposal of proposal | Prequorum_reached of - Operation_worker.candidate - * voting_power - * Kind.preendorsement operation list + Operation_worker.candidate * Kind.preendorsement operation list | Quorum_reached of - Operation_worker.candidate - * voting_power - * Kind.endorsement operation list + Operation_worker.candidate * Kind.endorsement operation list | Timeout of timeout_kind val event_encoding : event Data_encoding.t diff --git a/src/proto_alpha/lib_delegate/operation_worker.ml b/src/proto_alpha/lib_delegate/operation_worker.ml index 66cec5723476..1e20efa00156 100644 --- a/src/proto_alpha/lib_delegate/operation_worker.ml +++ b/src/proto_alpha/lib_delegate/operation_worker.ml @@ -162,12 +162,9 @@ let candidate_encoding = (req "round_watched" Round.encoding) (req "payload_hash_watched" Block_payload_hash.encoding)) -type voting_power = int - type event = - | Prequorum_reached of - candidate * voting_power * Kind.preendorsement operation list - | Quorum_reached of candidate * voting_power * Kind.endorsement operation list + | Prequorum_reached of candidate * Kind.preendorsement operation list + | Quorum_reached of candidate * Kind.endorsement operation list type pqc_watched = { candidate_watched : candidate; @@ -310,7 +307,6 @@ let update_monitoring ?(should_lock = true) state ops = (Some (Prequorum_reached ( candidate_watched, - proposal_watched.current_voting_power, List.rev proposal_watched.preendorsements_received ))) ; (* Once the event has been emitted, we cancel the monitoring *) cancel_monitoring state ; @@ -370,7 +366,6 @@ let update_monitoring ?(should_lock = true) state ops = (Some (Quorum_reached ( candidate_watched, - proposal_watched.current_voting_power, List.rev proposal_watched.endorsements_received ))) ; (* Once the event has been emitted, we cancel the monitoring *) cancel_monitoring state ; diff --git a/src/proto_alpha/lib_delegate/operation_worker.mli b/src/proto_alpha/lib_delegate/operation_worker.mli index eecbc990f2d1..1c12e7356c37 100644 --- a/src/proto_alpha/lib_delegate/operation_worker.mli +++ b/src/proto_alpha/lib_delegate/operation_worker.mli @@ -41,12 +41,9 @@ type candidate = { val candidate_encoding : candidate Data_encoding.t -type voting_power = int - type event = - | Prequorum_reached of - candidate * voting_power * Kind.preendorsement operation list - | Quorum_reached of candidate * voting_power * Kind.endorsement operation list + | Prequorum_reached of candidate * Kind.preendorsement operation list + | Quorum_reached of candidate * Kind.endorsement operation list (** {1 Constructors}*) diff --git a/src/proto_alpha/lib_delegate/state_transitions.ml b/src/proto_alpha/lib_delegate/state_transitions.ml index 5cc55044263d..7d6fe18f2cba 100644 --- a/src/proto_alpha/lib_delegate/state_transitions.ml +++ b/src/proto_alpha/lib_delegate/state_transitions.ml @@ -719,14 +719,13 @@ let step (state : Baking_state.t) (event : Baking_state.event) : >>= fun () -> Events.(emit new_head_while_waiting_for_qc ()) >>= fun () -> handle_new_proposal state block_info - | ( Awaiting_preendorsements, - Prequorum_reached (candidate, _voting_power, preendorsement_qc) ) -> + | Awaiting_preendorsements, Prequorum_reached (candidate, preendorsement_qc) + -> prequorum_reached_when_awaiting_preendorsements state candidate preendorsement_qc - | ( Awaiting_endorsements, - Quorum_reached (candidate, _voting_power, endorsement_qc) ) -> + | Awaiting_endorsements, Quorum_reached (candidate, endorsement_qc) -> quorum_reached_when_waiting_endorsements state candidate endorsement_qc (* Unreachable cases *) | Idle, (Prequorum_reached _ | Quorum_reached _) -- GitLab From 3a7b7e281afe3b9b9fc0c434a8a6e6568ed69410 Mon Sep 17 00:00:00 2001 From: vbot Date: Thu, 19 Jan 2023 16:36:15 +0100 Subject: [PATCH 30/46] Alpha/Baker: rework RPC monitoring and expose valid blocks watch --- .../lib_delegate/baking_actions.ml | 15 + src/proto_alpha/lib_delegate/baking_events.ml | 16 +- src/proto_alpha/lib_delegate/baking_lib.ml | 16 +- src/proto_alpha/lib_delegate/baking_nonces.ml | 2 +- .../lib_delegate/baking_scheduling.ml | 12 +- .../lib_delegate/baking_simulator.ml | 25 +- .../lib_delegate/baking_simulator.mli | 1 + src/proto_alpha/lib_delegate/baking_state.ml | 90 ++--- src/proto_alpha/lib_delegate/baking_state.mli | 15 +- src/proto_alpha/lib_delegate/block_forge.ml | 47 +-- src/proto_alpha/lib_delegate/block_forge.mli | 2 + src/proto_alpha/lib_delegate/node_rpc.ml | 321 +++++++++++------- src/proto_alpha/lib_delegate/node_rpc.mli | 20 +- .../lib_delegate/state_transitions.ml | 7 +- 14 files changed, 323 insertions(+), 266 deletions(-) diff --git a/src/proto_alpha/lib_delegate/baking_actions.ml b/src/proto_alpha/lib_delegate/baking_actions.ml index b24a54100308..ffc740c5f5bd 100644 --- a/src/proto_alpha/lib_delegate/baking_actions.ml +++ b/src/proto_alpha/lib_delegate/baking_actions.ml @@ -215,6 +215,7 @@ let sign_block_header state proposer unsigned_block_header = return {Block_header.shell; protocol_data = {contents; signature}} let inject_block ~state_recorder state block_to_bake ~updated_state = + let open Lwt_result_syntax in let { predecessor; round; @@ -308,10 +309,24 @@ let inject_block ~state_recorder state block_to_bake ~updated_state = in Events.(emit vote_for_liquidity_baking_toggle) liquidity_baking_toggle_vote >>= fun () -> + let chain = `Hash state.global_state.chain_id in + let pred_block = `Hash (predecessor.hash, 0) in + let* pred_resulting_context_hash = + Shell_services.Blocks.resulting_context_hash + cctxt + ~chain + ~block:pred_block + () + in + let* pred_live_blocks = + Chain_services.Blocks.live_blocks cctxt ~chain ~block:pred_block () + in Block_forge.forge cctxt ~chain_id ~pred_info:predecessor + ~pred_live_blocks + ~pred_resulting_context_hash ~timestamp ~round ~seed_nonce_hash diff --git a/src/proto_alpha/lib_delegate/baking_events.ml b/src/proto_alpha/lib_delegate/baking_events.ml index 110ce3854366..6b6327721fe1 100644 --- a/src/proto_alpha/lib_delegate/baking_events.ml +++ b/src/proto_alpha/lib_delegate/baking_events.ml @@ -315,16 +315,14 @@ module Node_rpc = struct ~pp1:Error_monad.pp_print_trace ("trace", Error_monad.trace_encoding) - let raw_info = - declare_2 + let error_while_monitoring_valid_proposals = + declare_1 ~section - ~name:"raw_info" - ~level:Debug - ~msg:"raw info for {block_hash} at level {level}" - ~pp1:Block_hash.pp - ("block_hash", Block_hash.encoding) - ~pp2:pp_int32 - ("level", Data_encoding.int32) + ~name:"error_while_monitoring_valid_proposals" + ~level:Error + ~msg:"error while monitoring valid proposals {trace}" + ~pp1:Error_monad.pp_print_trace + ("trace", Error_monad.trace_encoding) end module Scheduling = struct diff --git a/src/proto_alpha/lib_delegate/baking_lib.ml b/src/proto_alpha/lib_delegate/baking_lib.ml index 97905dcc5e59..82297396c5d2 100644 --- a/src/proto_alpha/lib_delegate/baking_lib.ml +++ b/src/proto_alpha/lib_delegate/baking_lib.ml @@ -44,10 +44,10 @@ let create_state cctxt ?synchronize ?monitor_node_mempool ~config ~current_proposal delegates -let get_current_proposal cctxt = +let get_current_proposal cctxt ?cache () = let open Lwt_result_syntax in let* block_stream, _block_stream_stopper = - Node_rpc.monitor_proposals cctxt ~chain:cctxt#chain () + Node_rpc.monitor_heads cctxt ?cache ~chain:cctxt#chain () in Lwt_stream.peek block_stream >>= function | Some current_head -> return (block_stream, current_head) @@ -59,7 +59,8 @@ let preendorse (cctxt : Protocol_client_context.full) ?(force = false) delegates = let open State_transitions in let open Lwt_result_syntax in - let* _, current_proposal = get_current_proposal cctxt in + let cache = Baking_cache.Block_cache.create 10 in + let* _, current_proposal = get_current_proposal cctxt ~cache () in let config = Baking_configuration.make ~force () in let* state = create_state cctxt ~config ~current_proposal delegates in let proposal = state.level_state.latest_proposal in @@ -91,7 +92,8 @@ let preendorse (cctxt : Protocol_client_context.full) ?(force = false) delegates let endorse (cctxt : Protocol_client_context.full) ?(force = false) delegates = let open State_transitions in let open Lwt_result_syntax in - let* _, current_proposal = get_current_proposal cctxt in + let cache = Baking_cache.Block_cache.create 10 in + let* _, current_proposal = get_current_proposal cctxt ~cache () in let config = Baking_configuration.make ~force () in create_state cctxt ~config ~current_proposal delegates >>=? fun state -> let proposal = state.level_state.latest_proposal in @@ -283,7 +285,8 @@ let propose (cctxt : Protocol_client_context.full) ?minimal_fees ?minimal_nanotez_per_gas_unit ?minimal_nanotez_per_byte ?force_apply ?force ?(minimal_timestamp = false) ?extra_operations ?context_path delegates = let open Lwt_result_syntax in - let* _block_stream, current_proposal = get_current_proposal cctxt in + let cache = Baking_cache.Block_cache.create 10 in + let* _block_stream, current_proposal = get_current_proposal cctxt ~cache () in let config = Baking_configuration.make ?minimal_fees @@ -507,7 +510,8 @@ let bake (cctxt : Protocol_client_context.full) ?minimal_fees ?extra_operations () in - let* block_stream, current_proposal = get_current_proposal cctxt in + let cache = Baking_cache.Block_cache.create 10 in + let* block_stream, current_proposal = get_current_proposal cctxt ~cache () in let* state = create_state cctxt diff --git a/src/proto_alpha/lib_delegate/baking_nonces.ml b/src/proto_alpha/lib_delegate/baking_nonces.ml index fc1c9ada635a..c0c3d4d8a15d 100644 --- a/src/proto_alpha/lib_delegate/baking_nonces.ml +++ b/src/proto_alpha/lib_delegate/baking_nonces.ml @@ -255,7 +255,7 @@ let reveal_potential_nonces state new_proposal = let new_predecessor_hash = new_proposal.Baking_state.predecessor.hash in if Block_hash.(last_predecessor <> new_predecessor_hash) - && Protocol_hash.(new_proposal.predecessor.protocol = Protocol.hash) + && not (Baking_state.is_first_block_in_protocol new_proposal) then ( (* only try revealing nonces when the proposal's predecessor is a new one *) state.last_predecessor <- new_predecessor_hash ; diff --git a/src/proto_alpha/lib_delegate/baking_scheduling.ml b/src/proto_alpha/lib_delegate/baking_scheduling.ml index 5525a2b00431..3c1e970ce19c 100644 --- a/src/proto_alpha/lib_delegate/baking_scheduling.ml +++ b/src/proto_alpha/lib_delegate/baking_scheduling.ml @@ -207,8 +207,7 @@ let compute_next_round_time state = | None -> state.level_state.latest_proposal | Some {proposal; _} -> proposal in - if Protocol_hash.(proposal.predecessor.next_protocol <> Protocol.hash) then - None + if Baking_state.is_first_block_in_protocol proposal then None else match state.level_state.next_level_proposed_round with | Some _proposed_round -> @@ -611,11 +610,7 @@ let create_initial_state cctxt ?(synchronize = true) ~chain config ~chain >>=? fun next_level_delegate_slots -> let elected_block = - if - Protocol_hash.( - current_proposal.block.protocol <> Protocol.hash - && current_proposal.block.next_protocol = Protocol.hash) - then + if Baking_state.is_first_block_in_protocol current_proposal then (* If the last block is a protocol transition, we admit it as a final block *) Some {proposal = current_proposal; endorsement_qc = []} @@ -717,7 +712,8 @@ let run cctxt ?canceler ?(stop_on_event = fun _ -> false) ?(on_error = fun _ -> return_unit) ~chain config delegates = Shell_services.Chain.chain_id cctxt ~chain () >>=? fun chain_id -> perform_sanity_check cctxt ~chain_id >>=? fun () -> - Node_rpc.monitor_proposals cctxt ~chain () + let cache = Baking_cache.Block_cache.create 10 in + Node_rpc.monitor_heads cctxt ~cache ~chain () >>=? fun (block_stream, _block_stream_stopper) -> (Lwt_stream.get block_stream >>= function | Some current_head -> return current_head diff --git a/src/proto_alpha/lib_delegate/baking_simulator.ml b/src/proto_alpha/lib_delegate/baking_simulator.ml index 5bb1fa77808f..45dcaf283770 100644 --- a/src/proto_alpha/lib_delegate/baking_simulator.ml +++ b/src/proto_alpha/lib_delegate/baking_simulator.ml @@ -77,17 +77,11 @@ let check_context_consistency (abstract_index : Abstract_context_index.t) | false -> fail Invalid_context)) let begin_construction ~timestamp ~protocol_data ~force_apply - (abstract_index : Abstract_context_index.t) predecessor chain_id = + ~pred_resulting_context_hash (abstract_index : Abstract_context_index.t) + pred_block chain_id = protect (fun () -> - let { - Baking_state.shell = pred_shell; - hash = pred_hash; - resulting_context_hash; - _; - } = - predecessor - in - abstract_index.checkout_fun resulting_context_hash >>= function + let {Baking_state.shell = pred_shell; hash = pred_hash; _} = pred_block in + abstract_index.checkout_fun pred_resulting_context_hash >>= function | None -> fail Failed_to_checkout_context | Some context -> let header : Tezos_base.Block_header.shell_header = @@ -107,7 +101,7 @@ let begin_construction ~timestamp ~protocol_data ~force_apply let mode = Lifted_protocol.Construction { - predecessor_hash = predecessor.hash; + predecessor_hash = pred_hash; timestamp; block_header_data = protocol_data; } @@ -130,7 +124,14 @@ let begin_construction ~timestamp ~protocol_data ~force_apply else return_none) >>=? fun application_state -> let state = (validation_state, application_state) in - return {predecessor; context; state; rev_operations = []; header}) + return + { + predecessor = pred_block; + context; + state; + rev_operations = []; + header; + }) let ( let** ) x k = let open Lwt_result_syntax in diff --git a/src/proto_alpha/lib_delegate/baking_simulator.mli b/src/proto_alpha/lib_delegate/baking_simulator.mli index 959546fe6d21..c5155ac45ee0 100644 --- a/src/proto_alpha/lib_delegate/baking_simulator.mli +++ b/src/proto_alpha/lib_delegate/baking_simulator.mli @@ -52,6 +52,7 @@ val begin_construction : timestamp:Time.Protocol.t -> protocol_data:block_header_data -> force_apply:bool -> + pred_resulting_context_hash:Context_hash.t -> Abstract_context_index.t -> Baking_state.block_info -> Chain_id.t -> diff --git a/src/proto_alpha/lib_delegate/baking_state.ml b/src/proto_alpha/lib_delegate/baking_state.ml index 162a3f7fa3a4..2d96e3881533 100644 --- a/src/proto_alpha/lib_delegate/baking_state.ml +++ b/src/proto_alpha/lib_delegate/baking_state.ml @@ -103,16 +103,12 @@ type prequorum = { type block_info = { hash : Block_hash.t; shell : Block_header.shell_header; - resulting_context_hash : Context_hash.t; payload_hash : Block_payload_hash.t; payload_round : Round.t; round : Round.t; - protocol : Protocol_hash.t; - next_protocol : Protocol_hash.t; prequorum : prequorum option; quorum : Kind.endorsement operation list; payload : Operation_pool.payload; - live_blocks : Block_hash.Set.t; } type cache = { @@ -167,68 +163,48 @@ let block_info_encoding = (fun { hash; shell; - resulting_context_hash; payload_hash; payload_round; round; - protocol; - next_protocol; prequorum; quorum; payload; - live_blocks; } -> - ( ( hash, - shell, - resulting_context_hash, - payload_hash, - payload_round, - round, - protocol, - next_protocol, - prequorum, - List.map Operation.pack quorum ), - (payload, live_blocks) )) - (fun ( ( hash, - shell, - resulting_context_hash, - payload_hash, - payload_round, - round, - protocol, - next_protocol, - prequorum, - quorum ), - (payload, live_blocks) ) -> + ( hash, + shell, + payload_hash, + payload_round, + round, + prequorum, + List.map Operation.pack quorum, + payload )) + (fun ( hash, + shell, + payload_hash, + payload_round, + round, + prequorum, + quorum, + payload ) -> { hash; shell; - resulting_context_hash; payload_hash; payload_round; round; - protocol; - next_protocol; prequorum; quorum = List.filter_map Operation_pool.unpack_endorsement quorum; payload; - live_blocks; }) - (merge_objs - (obj10 - (req "hash" Block_hash.encoding) - (req "shell" Block_header.shell_header_encoding) - (req "resulting_context_hash" Context_hash.encoding) - (req "payload_hash" Block_payload_hash.encoding) - (req "payload_round" Round.encoding) - (req "round" Round.encoding) - (req "protocol" Protocol_hash.encoding) - (req "next_protocol" Protocol_hash.encoding) - (req "prequorum" (option prequorum_encoding)) - (req "quorum" (list (dynamic_size Operation.encoding)))) - (obj2 - (req "payload" Operation_pool.payload_encoding) - (req "live_blocks" Block_hash.Set.encoding))) + (obj8 + (req "hash" Block_hash.encoding) + (req "shell" Block_header.shell_header_encoding) + (req "payload_hash" Block_payload_hash.encoding) + (req "payload_round" Round.encoding) + (req "round" Round.encoding) + (req "prequorum" (option prequorum_encoding)) + (req "quorum" (list (dynamic_size Operation.encoding))) + (req "payload" Operation_pool.payload_encoding)) let round_of_shell_header shell_header = Environment.wrap_tzresult @@ -266,6 +242,9 @@ let proposal_encoding = (req "block" block_info_encoding) (req "predecessor" block_info_encoding)) +let is_first_block_in_protocol {block; predecessor; _} = + Compare.Int.(block.shell.proto_level <> predecessor.shell.proto_level) + type locked_round = {payload_hash : Block_payload_hash.t; round : Round.t} let locked_round_encoding = @@ -721,18 +700,15 @@ let pp_block_info fmt shell; payload_hash; round; - protocol; - next_protocol; prequorum; quorum; payload; - _; + payload_round; } = Format.fprintf fmt "@[Block:@ hash: %a@ payload_hash: %a@ level: %ld@ round: %a@ \ - protocol: %a@ next protocol: %a@ prequorum: %a@ quorum: %d endorsements@ \ - payload: %a@]" + prequorum: %a@ quorum: %d endorsements@ payload: %a@ payload round: %a@]" Block_hash.pp hash Block_payload_hash.pp_short @@ -740,15 +716,13 @@ let pp_block_info fmt shell.level Round.pp round - Protocol_hash.pp_short - protocol - Protocol_hash.pp_short - next_protocol (pp_option pp_prequorum) prequorum (List.length quorum) Operation_pool.pp_payload payload + Round.pp + payload_round let pp_proposal fmt {block; _} = pp_block_info fmt block diff --git a/src/proto_alpha/lib_delegate/baking_state.mli b/src/proto_alpha/lib_delegate/baking_state.mli index 45ea1d3392df..47596470a655 100644 --- a/src/proto_alpha/lib_delegate/baking_state.mli +++ b/src/proto_alpha/lib_delegate/baking_state.mli @@ -57,18 +57,12 @@ type prequorum = { type block_info = { hash : Block_hash.t; shell : Block_header.shell_header; - resulting_context_hash : Context_hash.t; payload_hash : Block_payload_hash.t; payload_round : Round.t; round : Round.t; - protocol : Protocol_hash.t; - next_protocol : Protocol_hash.t; prequorum : prequorum option; quorum : Kind.endorsement operation list; payload : Operation_pool.payload; - live_blocks : Block_hash.Set.t; - (** Set of live blocks for this block that is used to filter - old or too recent operations. *) } type cache = { @@ -108,6 +102,15 @@ type proposal = {block : block_info; predecessor : block_info} val proposal_encoding : proposal Data_encoding.t +(** Identify the first block of the protocol, ie. the block that + activates the current protocol. + + This block should be baked by the baker of the previous protocol + (that's why this same block is also referred to as the last block + of the previous protocol). It is always considered final and + therefore is not endorsed.*) +val is_first_block_in_protocol : proposal -> bool + type locked_round = {payload_hash : Block_payload_hash.t; round : Round.t} val locked_round_encoding : locked_round Data_encoding.t diff --git a/src/proto_alpha/lib_delegate/block_forge.ml b/src/proto_alpha/lib_delegate/block_forge.ml index d3be206ac6b6..01a06159e467 100644 --- a/src/proto_alpha/lib_delegate/block_forge.ml +++ b/src/proto_alpha/lib_delegate/block_forge.ml @@ -69,17 +69,19 @@ let convert_operation (op : packed_operation) : Tezos_base.Operation.t = op.protocol_data; } -(* [finalize_block_header ~shell_header ~validation_result ~operations_hash - ~pred_info ~round ~locked_round] updates the [shell_header] that was created - with dummy fields at the beginning of the block construction. It increments - the [level] and sets the actual [operations_hash], [fitness], - [validation_passes], and [context] (the predecessor resulting context hash). +(* [finalize_block_header] updates the [shell_header] that was created + with dummy fields at the beginning of the block construction. It + increments the [level] and sets the actual [operations_hash], + [fitness], [validation_passes], and [context] (the predecessor + resulting context hash). - When the operations from the block have been applied, the [fitness] is simply - retrieved from the [validation_result]. Otherwise, the [fitness] is computed - from the [round] and [locked_round] arguments. *) + When the operations from the block have been applied, the [fitness] + is simply retrieved from the [validation_result]. Otherwise, the + [fitness] is computed from the [round] and [locked_round] + arguments. *) let finalize_block_header ~shell_header ~validation_result ~operations_hash - ~(pred_info : Baking_state.block_info) ~round ~locked_round = + ~(pred_info : Baking_state.block_info) ~pred_resulting_context_hash ~round + ~locked_round = let open Lwt_result_syntax in let* fitness = match validation_result with @@ -108,7 +110,7 @@ let finalize_block_header ~shell_header ~validation_result ~operations_hash validation_passes; operations_hash; fitness; - context = pred_info.resulting_context_hash; + context = pred_resulting_context_hash; } in return header @@ -203,14 +205,15 @@ let filter_via_node ~chain_id ~fees_config ~hard_gas_limit_per_block [filter_via_node] is called to return these values. *) let filter_with_context ~chain_id ~fees_config ~hard_gas_limit_per_block ~faked_protocol_data ~user_activated_upgrades ~timestamp - ~(pred_info : Baking_state.block_info) ~force_apply ~round ~context_index - ~payload_round ~operation_pool cctxt = + ~(pred_info : Baking_state.block_info) ~pred_resulting_context_hash + ~force_apply ~round ~context_index ~payload_round ~operation_pool cctxt = let open Lwt_result_syntax in let* incremental = Baking_simulator.begin_construction ~timestamp ~protocol_data:faked_protocol_data ~force_apply + ~pred_resulting_context_hash context_index pred_info chain_id @@ -248,6 +251,7 @@ let filter_with_context ~chain_id ~fees_config ~hard_gas_limit_per_block ~validation_result ~operations_hash ~pred_info + ~pred_resulting_context_hash ~round ~locked_round:None in @@ -288,14 +292,16 @@ let apply_via_node ~chain_id ~faked_protocol_data ~timestamp consensus operations only from an [ordered_pool] via {!Operation_selection.filter_consensus_operations_only}. *) let apply_with_context ~chain_id ~faked_protocol_data ~user_activated_upgrades - ~timestamp ~(pred_info : Baking_state.block_info) ~force_apply ~round - ~ordered_pool ~context_index ~payload_hash cctxt = + ~timestamp ~(pred_info : Baking_state.block_info) + ~pred_resulting_context_hash ~force_apply ~round ~ordered_pool + ~context_index ~payload_hash cctxt = let open Lwt_result_syntax in let* incremental = Baking_simulator.begin_construction ~timestamp ~protocol_data:faked_protocol_data ~force_apply + ~pred_resulting_context_hash context_index pred_info chain_id @@ -358,6 +364,7 @@ let apply_with_context ~chain_id ~faked_protocol_data ~user_activated_upgrades ~validation_result ~operations_hash ~pred_info + ~pred_resulting_context_hash ~round ~locked_round:locked_round_when_no_validation_result in @@ -367,10 +374,10 @@ let apply_with_context ~chain_id ~faked_protocol_data ~user_activated_upgrades (* [forge] a new [unsigned_block] in accordance with [simulation_kind] and [simulation_mode] *) let forge (cctxt : #Protocol_client_context.full) ~chain_id - ~(pred_info : Baking_state.block_info) ~timestamp ~round - ~liquidity_baking_toggle_vote ~user_activated_upgrades fees_config - ~force_apply ~seed_nonce_hash ~payload_round simulation_mode simulation_kind - constants = + ~(pred_info : Baking_state.block_info) ~pred_resulting_context_hash + ~pred_live_blocks ~timestamp ~round ~liquidity_baking_toggle_vote + ~user_activated_upgrades fees_config ~force_apply ~seed_nonce_hash + ~payload_round simulation_mode simulation_kind constants = let open Lwt_result_syntax in let hard_gas_limit_per_block = constants.Constants.Parametric.hard_gas_limit_per_block @@ -382,7 +389,7 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id to our predecessor otherwise the node would reject the block. *) let filtered_pool = retain_live_operations_only - ~live_blocks:pred_info.live_blocks + ~live_blocks:pred_live_blocks operation_pool in Filter filtered_pool @@ -441,6 +448,7 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~user_activated_upgrades ~timestamp ~pred_info + ~pred_resulting_context_hash ~force_apply ~round ~context_index @@ -462,6 +470,7 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~user_activated_upgrades ~timestamp ~pred_info + ~pred_resulting_context_hash ~force_apply ~round ~ordered_pool diff --git a/src/proto_alpha/lib_delegate/block_forge.mli b/src/proto_alpha/lib_delegate/block_forge.mli index 77fb619a192f..7afc2ce9791f 100644 --- a/src/proto_alpha/lib_delegate/block_forge.mli +++ b/src/proto_alpha/lib_delegate/block_forge.mli @@ -44,6 +44,8 @@ val forge : #Protocol_client_context.full -> chain_id:Chain_id.t -> pred_info:Baking_state.block_info -> + pred_resulting_context_hash:Context_hash.t -> + pred_live_blocks:Block_hash.Set.t -> timestamp:Time.Protocol.t -> round:Round.t -> liquidity_baking_toggle_vote:Liquidity_baking.liquidity_baking_toggle_vote -> diff --git a/src/proto_alpha/lib_delegate/node_rpc.ml b/src/proto_alpha/lib_delegate/node_rpc.ml index a6d3d71cc15e..2f7e86e4fa7f 100644 --- a/src/proto_alpha/lib_delegate/node_rpc.ml +++ b/src/proto_alpha/lib_delegate/node_rpc.ml @@ -25,6 +25,8 @@ open Protocol open Alpha_context +open Baking_cache +open Baking_state module Block_services = Block_services.Make (Protocol) (Protocol) module Events = Baking_events.Node_rpc @@ -50,164 +52,219 @@ let preapply_block cctxt ~chain ~head ~timestamp ~protocol_data operations = let extract_prequorum preendorsements = match preendorsements with - | h :: _ as l -> + | h :: _ -> let ({protocol_data = {contents = Single (Preendorsement content); _}; _}) = (h : Kind.preendorsement Operation.t) in Some { - Baking_state.level = Raw_level.to_int32 content.level; + level = Raw_level.to_int32 content.level; round = content.round; block_payload_hash = content.block_payload_hash; - preendorsements = l; + preendorsements; } | _ -> None -let raw_info cctxt ~chain ~block_hash shell resulting_context_hash payload_hash - payload_round current_protocol next_protocol live_blocks = - Events.(emit raw_info (block_hash, shell.Tezos_base.Block_header.level)) - >>= fun () -> - let open Protocol_client_context in - let block = `Hash (block_hash, 0) in - let is_in_protocol = Protocol_hash.(current_protocol = Protocol.hash) in - (if is_in_protocol then - Alpha_block_services.Operations.operations cctxt ~chain ~block () - >>=? fun operations -> - let operations = - List.map - (fun l -> - List.map - (fun {Alpha_block_services.shell; protocol_data; _} -> - {Alpha_context.shell; protocol_data}) - l) - operations - in - match Operation_pool.extract_operations_of_list_list operations with - | None -> failwith "Unexpected operation list size" - | Some operations -> return operations - else - (* If we are not in the current protocol, do no consider operations *) - return (None, [], Operation_pool.empty_payload)) - >>=? fun (preendorsements, quorum, payload) -> - (match Baking_state.round_of_shell_header shell with - | Ok round -> ok round - | _ -> - (* this can occur if the protocol has just changed and the - previous protocol does not have a concept of round - (e.g. Genesis) *) - ok Round.zero) - >>?= fun round -> - let prequorum = Option.bind preendorsements extract_prequorum in +let info_of_header_and_ops ~in_protocol block_hash block_header operations = + let open Result_syntax in + let shell = block_header.Tezos_base.Block_header.shell in + let dummy_payload_hash = Block_payload_hash.zero in + let* round = + Environment.wrap_tzresult @@ Fitness.round_from_raw shell.fitness + in + let payload_hash, payload_round, prequorum, quorum, payload = + if not in_protocol then + (* The first block in the protocol is baked using the previous + protocol, the encodings might change. The baker's logic is to + consider final the first block of a new protocol and not + endorse it. Therefore, we do not need to have the correct + values here. *) + (dummy_payload_hash, Round.zero, None, [], Operation_pool.empty_payload) + else + let payload_hash, payload_round = + match + Data_encoding.Binary.of_bytes_opt + Protocol.block_header_data_encoding + block_header.protocol_data + with + | Some {contents = {payload_hash; payload_round; _}; _} -> + (payload_hash, payload_round) + | None -> assert false + in + let preendorsements, quorum, payload = + WithExceptions.Option.get + ~loc:__LOC__ + (Operation_pool.extract_operations_of_list_list operations) + in + let prequorum = Option.bind preendorsements extract_prequorum in + (payload_hash, payload_round, prequorum, quorum, payload) + in return { - Baking_state.hash = block_hash; + hash = block_hash; shell; - resulting_context_hash; payload_hash; payload_round; round; - protocol = current_protocol; - next_protocol; prequorum; quorum; payload; - live_blocks; } -let dummy_payload_hash = Block_payload_hash.zero +let compute_block_info cctxt ~in_protocol ?operations ~chain block_hash + block_header = + let open Lwt_result_syntax in + let* operations = + match operations with + | None when not in_protocol -> return_nil + | None -> + let open Protocol_client_context in + let* operations = + Alpha_block_services.Operations.operations + cctxt + ~chain + ~block:(`Hash (block_hash, 0)) + () + in + let packed_operations = + List.map + (fun l -> + List.map + (fun {Alpha_block_services.shell; protocol_data; _} -> + {Alpha_context.shell; protocol_data}) + l) + operations + in + return packed_operations + | Some operations -> + let parse_op (raw_op : Tezos_base.Operation.t) = + let protocol_data = + Data_encoding.Binary.of_bytes_exn + Operation.protocol_data_encoding + raw_op.proto + in + {shell = raw_op.shell; protocol_data} + in + protect @@ fun () -> return (List.map (List.map parse_op) operations) + in + let*? block_info = + info_of_header_and_ops ~in_protocol block_hash block_header operations + in + return block_info -let info cctxt ~chain ~block () = - let open Protocol_client_context in - (* Fails if the block's protocol is not the current one *) - Shell_services.Blocks.protocols cctxt ~chain ~block () - >>=? fun {current_protocol; next_protocol} -> - Shell_services.Blocks.resulting_context_hash cctxt ~chain ~block () - >>=? fun resulting_context_hash -> - (if Protocol_hash.(current_protocol <> Protocol.hash) then - Block_services.Header.shell_header cctxt ~chain ~block () >>=? fun shell -> - Chain_services.Blocks.Header.raw_protocol_data cctxt ~chain ~block () - >>=? fun protocol_data -> - let hash = - Tezos_base.Block_header.hash {Tezos_base.Block_header.shell; protocol_data} - in - (* /!\ We decode [protocol_data] with the current protocol's - encoding, while we should use the previous protocol's - [protocol_data] encoding. For now, this works because the - encoding has not changed. *) - let payload_hash, payload_round = - match - Data_encoding.Binary.of_bytes_opt - Protocol.block_header_data_encoding - protocol_data - with - | Some {contents = {payload_hash; payload_round; _}; _} -> - (payload_hash, payload_round) - | None -> (dummy_payload_hash, Round.zero) - in - return (hash, shell, resulting_context_hash, payload_hash, payload_round) - else - Alpha_block_services.header cctxt ~chain ~block () - >>=? fun {hash; shell; protocol_data; _} -> - return - ( hash, - shell, - resulting_context_hash, - protocol_data.contents.payload_hash, - protocol_data.contents.payload_round )) - >>=? fun (hash, shell, resulting_context_hash, payload_hash, payload_round) -> - (Chain_services.Blocks.live_blocks cctxt ~chain ~block () >>= function - | Error _ -> - (* The RPC might fail when a block's metadata is not available *) - Lwt.return Block_hash.Set.empty - | Ok live_blocks -> Lwt.return live_blocks) - >>= fun live_blocks -> - raw_info - cctxt - ~chain - ~block_hash:hash - shell - resulting_context_hash - payload_hash - payload_round - current_protocol - next_protocol - live_blocks +let proposal cctxt ?(cache : block_info Block_cache.t option) ?operations ~chain + block_hash (block_header : Tezos_base.Block_header.t) = + let open Lwt_result_syntax in + let predecessor_hash = block_header.shell.predecessor in + let pred_block = `Hash (predecessor_hash, 0) in + let predecessor_opt = + Option.bind cache (fun cache -> Block_cache.find_opt cache predecessor_hash) + in + let* is_proposal_in_protocol, predecessor = + match predecessor_opt with + | Some predecessor -> + return + ( predecessor.shell.proto_level = block_header.shell.proto_level, + predecessor ) + | None -> + let* { + current_protocol = pred_current_protocol; + next_protocol = pred_next_protocol; + } = + Shell_services.Blocks.protocols cctxt ~chain ~block:pred_block () + in + let is_proposal_in_protocol = + Protocol_hash.(pred_next_protocol = Protocol.hash) + in + let* predecessor = + let in_protocol = + Protocol_hash.(pred_current_protocol = Protocol.hash) + in + let* raw_header_b = + Shell_services.Blocks.raw_header cctxt ~chain ~block:pred_block () + in + let predecessor_header = + Data_encoding.Binary.of_bytes_exn + Tezos_base.Block_header.encoding + raw_header_b + in + compute_block_info + cctxt + ~in_protocol + ~chain + predecessor_hash + predecessor_header + in + Option.iter + (fun cache -> Block_cache.replace cache predecessor_hash predecessor) + cache ; + return (is_proposal_in_protocol, predecessor) + in + let block_opt = + Option.bind cache (fun cache -> Block_cache.find_opt cache block_hash) + in + let* block = + match block_opt with + | Some pi -> return pi + | None -> + let* pi = + compute_block_info + cctxt + ~in_protocol:is_proposal_in_protocol + ?operations + ~chain + block_hash + block_header + in + Option.iter (fun cache -> Block_cache.replace cache block_hash pi) cache ; + return pi + in + return {block; predecessor} -let find_in_cache_or_fetch cctxt ?cache ~chain block_hash = - let open Baking_cache in - let fetch () = info cctxt ~chain ~block:(`Hash (block_hash, 0)) () in - match cache with - | None -> fetch () - | Some block_cache -> ( - match Block_cache.find_opt block_cache block_hash with - | Some block_info -> return block_info - | None -> - fetch () >>=? fun block_info -> - Block_cache.replace block_cache block_hash block_info ; - return block_info) +let proposal cctxt ?cache ?operations ~chain block_hash block_header = + protect @@ fun () -> + proposal cctxt ?cache ?operations ~chain block_hash block_header -let proposal cctxt ?cache ~chain block_hash = - find_in_cache_or_fetch cctxt ~chain ?cache block_hash >>=? fun block -> - let predecessor_hash = block.shell.predecessor in - find_in_cache_or_fetch cctxt ~chain ?cache predecessor_hash - >>=? fun predecessor -> return {Baking_state.block; predecessor} +let monitor_valid_proposals cctxt ~chain ?cache () = + let open Lwt_result_syntax in + let next_protocols = [Protocol.hash] in + let* block_stream, stopper = + Monitor_services.validated_blocks cctxt ~chains:[chain] ~next_protocols () + in + let stream = + let map (_chain_id, block_hash, block_header, operations) = + let*! map_result = + proposal cctxt ?cache ~operations ~chain block_hash block_header + in + match map_result with + | Ok proposal -> Lwt.return_some proposal + | Error err -> + let*! () = Events.(emit error_while_monitoring_valid_proposals err) in + Lwt.return_none + in + Lwt_stream.filter_map_s map block_stream + in + return (stream, stopper) -let monitor_proposals cctxt ~chain () = - let cache = Baking_cache.Block_cache.create 100 in - Monitor_services.heads cctxt ~next_protocols:[Protocol.hash] chain - >>=? fun (block_stream, stopper) -> - return - ( Lwt_stream.filter_map_s - (fun (block_hash, _) -> - protect (fun () -> proposal cctxt ~cache ~chain block_hash) - >>= function - | Ok proposal -> Lwt.return_some proposal - | Error err -> - Events.(emit error_while_monitoring_heads err) >>= fun () -> - Lwt.return_none) - block_stream, - stopper ) +let monitor_heads cctxt ~chain ?cache () = + let open Lwt_result_syntax in + let next_protocols = [Protocol.hash] in + let* block_stream, stopper = + Monitor_services.heads cctxt ~next_protocols chain + in + let stream, stopper = + let map (block_hash, block_header) = + let*! map_result = proposal cctxt ?cache ~chain block_hash block_header in + match map_result with + | Ok proposal -> Lwt.return_some proposal + | Error err -> + let*! () = Events.(emit error_while_monitoring_heads err) in + Lwt.return_none + in + (Lwt_stream.filter_map_s map block_stream, stopper) + in + return (stream, stopper) let await_protocol_activation cctxt ~chain () = Monitor_services.heads cctxt ~next_protocols:[Protocol.hash] chain diff --git a/src/proto_alpha/lib_delegate/node_rpc.mli b/src/proto_alpha/lib_delegate/node_rpc.mli index 2dd0468ef6f1..6cd40df1760b 100644 --- a/src/proto_alpha/lib_delegate/node_rpc.mli +++ b/src/proto_alpha/lib_delegate/node_rpc.mli @@ -50,21 +50,19 @@ val preapply_block : (Tezos_base.Block_header.shell_header * error Preapply_result.t list) tzresult Lwt.t -(** Fetch a proposal from the node. - - @param cache is unset by default -*) -val proposal : - #Tezos_rpc.Context.simple -> - ?cache:Baking_state.block_info Baking_cache.Block_cache.t -> +(** Monitor validated blocks/proposals from the node. *) +val monitor_valid_proposals : + #Protocol_client_context.rpc_context -> chain:Shell_services.chain -> - Block_hash.t -> - Baking_state.proposal tzresult Lwt.t + ?cache:Baking_state.block_info Baking_cache.Block_cache.t -> + unit -> + (Baking_state.proposal Lwt_stream.t * (unit -> unit)) tzresult Lwt.t -(** Monitor proposals from the node.*) -val monitor_proposals : +(** Monitor heads from the node. *) +val monitor_heads : #Protocol_client_context.rpc_context -> chain:Shell_services.chain -> + ?cache:Baking_state.block_info Baking_cache.Block_cache.t -> unit -> (Baking_state.proposal Lwt_stream.t * (unit -> unit)) tzresult Lwt.t diff --git a/src/proto_alpha/lib_delegate/state_transitions.ml b/src/proto_alpha/lib_delegate/state_transitions.ml index 7d6fe18f2cba..43d07f528eb2 100644 --- a/src/proto_alpha/lib_delegate/state_transitions.ml +++ b/src/proto_alpha/lib_delegate/state_transitions.ml @@ -111,8 +111,7 @@ let may_update_proposal state (proposal : proposal) = else Lwt.return state let preendorse state proposal = - if Protocol_hash.(proposal.block.protocol <> proposal.block.next_protocol) - then + if Baking_state.is_first_block_in_protocol proposal then (* We do not preendorse the first transition block *) let new_state = update_current_phase state Idle in Lwt.return (new_state, Do_nothing) @@ -537,8 +536,8 @@ let end_of_round state current_round = let new_state = update_current_phase new_state Idle in do_nothing new_state | Some (delegate, _) -> - let last_proposal = state.level_state.latest_proposal.block in - if Protocol_hash.(last_proposal.protocol <> Protocol.hash) then + let latest_proposal = state.level_state.latest_proposal in + if Baking_state.is_first_block_in_protocol latest_proposal then (* Do not inject a block for the previous protocol! (Let the baker of the previous protocol do it.) *) do_nothing new_state -- GitLab From 469606e63bd60c7c26f781c8f0eea648e22fa580 Mon Sep 17 00:00:00 2001 From: vbot Date: Wed, 25 Jan 2023 14:16:43 +0100 Subject: [PATCH 31/46] Alpha/Baker: monitor valid blocks and trigger an associated event --- src/proto_alpha/lib_delegate/baking_lib.ml | 8 +- .../lib_delegate/baking_scheduling.ml | 116 ++++++++++++++---- .../lib_delegate/baking_scheduling.mli | 5 +- src/proto_alpha/lib_delegate/baking_state.ml | 29 +++-- src/proto_alpha/lib_delegate/baking_state.mli | 3 +- src/proto_alpha/lib_delegate/node_rpc.ml | 4 +- .../lib_delegate/state_transitions.ml | 9 +- .../lib_delegate/test/test_scenario.ml | 16 +-- 8 files changed, 140 insertions(+), 50 deletions(-) diff --git a/src/proto_alpha/lib_delegate/baking_lib.ml b/src/proto_alpha/lib_delegate/baking_lib.ml index 82297396c5d2..ae712266db5e 100644 --- a/src/proto_alpha/lib_delegate/baking_lib.ml +++ b/src/proto_alpha/lib_delegate/baking_lib.ml @@ -371,18 +371,18 @@ let propose (cctxt : Protocol_client_context.full) ?minimal_fees in return_unit -let bake_using_automaton config state block_stream = +let bake_using_automaton config state heads_stream = let open Lwt_result_syntax in let cctxt = state.global_state.cctxt in let* initial_event = first_automaton_event state in let current_level = state.level_state.latest_proposal.block.shell.level in let loop_state = Baking_scheduling.create_loop_state - block_stream + ~heads_stream state.global_state.operation_worker in let stop_on_next_level_block = function - | New_proposal proposal -> + | New_head_proposal proposal -> Compare.Int32.(proposal.block.shell.level >= Int32.succ current_level) | _ -> false in @@ -394,7 +394,7 @@ let bake_using_automaton config state block_stream = state initial_event >>=? function - | Some (New_proposal proposal) -> + | Some (New_head_proposal proposal) -> let*! () = cctxt#message "Block %a (%ld) injected" diff --git a/src/proto_alpha/lib_delegate/baking_scheduling.ml b/src/proto_alpha/lib_delegate/baking_scheduling.ml index 3c1e970ce19c..668218d08c96 100644 --- a/src/proto_alpha/lib_delegate/baking_scheduling.ml +++ b/src/proto_alpha/lib_delegate/baking_scheduling.ml @@ -28,33 +28,52 @@ module Events = Baking_events.Scheduling open Baking_state type loop_state = { - block_stream : Baking_state.proposal Lwt_stream.t; + heads_stream : Baking_state.proposal Lwt_stream.t; + get_valid_blocks_stream : Baking_state.proposal Lwt_stream.t Lwt.t; qc_stream : Operation_worker.event Lwt_stream.t; - future_block_stream : proposal Lwt_stream.t; - push_future_block : proposal -> unit; - mutable last_get_head_event : [`New_proposal of proposal option] Lwt.t option; + future_block_stream : + [`New_future_head of proposal | `New_future_valid_proposal of proposal] + Lwt_stream.t; + push_future_block : + [`New_future_head of proposal | `New_future_valid_proposal of proposal] -> + unit; + mutable last_get_head_event : + [`New_head_proposal of proposal option] Lwt.t option; + mutable last_get_valid_block_event : + [`New_valid_proposal of proposal option] Lwt.t option; mutable last_future_block_event : - [`New_future_block of Baking_state.proposal] Lwt.t option; + [`New_future_head of proposal | `New_future_valid_proposal of proposal] + Lwt.t + option; mutable last_get_qc_event : [`QC_reached of Operation_worker.event option] Lwt.t option; } type events = - [ `New_future_block of proposal - | `New_proposal of proposal option + [ `New_future_head of proposal + | `New_future_valid_proposal of proposal + | `New_valid_proposal of proposal option + | `New_head_proposal of proposal option | `QC_reached of Operation_worker.event option | `Termination | `Timeout of timeout_kind ] Lwt.t -let create_loop_state block_stream operation_worker = +let create_loop_state ?get_valid_blocks_stream ~heads_stream operation_worker = let future_block_stream, push_future_block = Lwt_stream.create () in + let get_valid_blocks_stream = + match get_valid_blocks_stream with + | None -> Lwt.return (Lwt_stream.create () |> fst) + | Some vbs_t -> vbs_t + in { - block_stream; + heads_stream; + get_valid_blocks_stream; qc_stream = Operation_worker.get_quorum_event_stream operation_worker; future_block_stream; push_future_block = (fun x -> push_future_block (Some x)); last_get_head_event = None; + last_get_valid_block_event = None; last_future_block_event = None; last_get_qc_event = None; } @@ -111,12 +130,24 @@ let rec wait_next_event ~timeout loop_state = match loop_state.last_get_head_event with | None -> let t = - Lwt_stream.get loop_state.block_stream >|= fun e -> `New_proposal e + Lwt_stream.get loop_state.heads_stream >|= fun e -> + `New_head_proposal e in loop_state.last_get_head_event <- Some t ; t | Some t -> t in + let get_valid_block_event () = + match loop_state.last_get_valid_block_event with + | None -> + let t = + loop_state.get_valid_blocks_stream >>= fun valid_blocks_stream -> + Lwt_stream.get valid_blocks_stream >|= fun e -> `New_valid_proposal e + in + loop_state.last_get_valid_block_event <- Some t ; + t + | Some t -> t + in let get_future_block_event () = (* n.b. we should also consume the available elements in the block_stream before starting baking. *) @@ -127,7 +158,7 @@ let rec wait_next_event ~timeout loop_state = | None -> (* unreachable, we never close the stream *) assert false - | Some proposal -> `New_future_block proposal + | Some future_proposal -> future_proposal in loop_state.last_future_block_event <- Some t ; t @@ -149,6 +180,7 @@ let rec wait_next_event ~timeout loop_state = [ terminated; (get_head_event () :> events); + (get_valid_block_event () :> events); (get_future_block_event () :> events); (get_qc_event () :> events); (timeout :> events); @@ -158,7 +190,11 @@ let rec wait_next_event ~timeout loop_state = | `Termination -> (* Exit the loop *) return_none - | `New_proposal None -> + | `New_valid_proposal None -> + (* Node connection lost *) + loop_state.last_get_valid_block_event <- None ; + fail Baking_errors.Node_connection_lost + | `New_head_proposal None -> (* Node connection lost *) loop_state.last_get_head_event <- None ; fail Baking_errors.Node_connection_lost @@ -166,7 +202,22 @@ let rec wait_next_event ~timeout loop_state = (* Not supposed to happen: exit the loop *) loop_state.last_get_qc_event <- None ; return_none - | `New_proposal (Some proposal) -> ( + | `New_valid_proposal (Some proposal) -> ( + loop_state.last_get_valid_block_event <- None ; + (* Is the block in the future? *) + match sleep_until proposal.block.shell.timestamp with + | Some waiter -> + (* If so, wait until its timestamp is reached before advertising it *) + Events.(emit proposal_in_the_future proposal.block.hash) >>= fun () -> + Lwt.dont_wait + (fun () -> + waiter >>= fun () -> + loop_state.push_future_block (`New_future_valid_proposal proposal) ; + Lwt.return_unit) + (fun _exn -> ()) ; + wait_next_event ~timeout loop_state + | None -> return_some (New_valid_proposal proposal)) + | `New_head_proposal (Some proposal) -> ( loop_state.last_get_head_event <- None ; (* Is the block in the future? *) match sleep_until proposal.block.shell.timestamp with @@ -176,16 +227,21 @@ let rec wait_next_event ~timeout loop_state = Lwt.dont_wait (fun () -> waiter >>= fun () -> - loop_state.push_future_block proposal ; + loop_state.push_future_block (`New_future_head proposal) ; Lwt.return_unit) (fun _exn -> ()) ; wait_next_event ~timeout loop_state - | None -> return_some (New_proposal proposal)) - | `New_future_block proposal -> + | None -> return_some (New_head_proposal proposal)) + | `New_future_head proposal -> + Events.(emit process_proposal_in_the_future proposal.block.hash) + >>= fun () -> + loop_state.last_future_block_event <- None ; + return_some (New_head_proposal proposal) + | `New_future_valid_proposal proposal -> Events.(emit process_proposal_in_the_future proposal.block.hash) >>= fun () -> loop_state.last_future_block_event <- None ; - return_some (New_proposal proposal) + return_some (New_valid_proposal proposal) | `QC_reached (Some (Operation_worker.Prequorum_reached (candidate, preendorsement_qc))) -> @@ -648,7 +704,7 @@ let compute_bootstrap_event state = = state.round_state.current_round) then (* If so, then trigger the new proposal event to possibly preendorse *) - ok @@ Baking_state.New_proposal state.level_state.latest_proposal + ok @@ Baking_state.New_head_proposal state.level_state.latest_proposal else (* Otherwise, trigger the end of round to check whether we need to propose at this level or not *) @@ -710,12 +766,13 @@ let perform_sanity_check cctxt ~chain_id = let run cctxt ?canceler ?(stop_on_event = fun _ -> false) ?(on_error = fun _ -> return_unit) ~chain config delegates = + let open Lwt_result_syntax in Shell_services.Chain.chain_id cctxt ~chain () >>=? fun chain_id -> perform_sanity_check cctxt ~chain_id >>=? fun () -> let cache = Baking_cache.Block_cache.create 10 in Node_rpc.monitor_heads cctxt ~cache ~chain () - >>=? fun (block_stream, _block_stream_stopper) -> - (Lwt_stream.get block_stream >>= function + >>=? fun (heads_stream, _block_stream_stopper) -> + (Lwt_stream.get heads_stream >>= function | Some current_head -> return current_head | None -> failwith "head stream unexpectedly ended") >>=? fun current_proposal -> @@ -734,7 +791,7 @@ let run cctxt ?canceler ?(stop_on_event = fun _ -> false) ~current_proposal delegates >>=? fun initial_state -> - let cloned_block_stream = Lwt_stream.clone block_stream in + let cloned_block_stream = Lwt_stream.clone heads_stream in Baking_nonces.start_revelation_worker cctxt initial_state.global_state.config.nonce @@ -748,9 +805,22 @@ let run cctxt ?canceler ?(stop_on_event = fun _ -> false) Lwt_canceler.cancel revelation_worker_canceler >>= fun _ -> Lwt.return_unit)) canceler ; - + (* FIXME: currently, the client streamed RPC call will hold until at + least one element is present in the stream. This is fixed by: + https://gitlab.com/nomadic-labs/resto/-/merge_requests/50. Until + then, we await the promise completion of the RPC call later + on. *) + let get_valid_blocks_stream = + let*! vbs = Node_rpc.monitor_valid_proposals cctxt ~cache ~chain () in + match vbs with + | Error _ -> Stdlib.failwith "Failed to get the validated blocks stream" + | Ok (vbs, _) -> Lwt.return vbs + in let loop_state = - create_loop_state block_stream initial_state.global_state.operation_worker + create_loop_state + ~get_valid_blocks_stream + ~heads_stream + initial_state.global_state.operation_worker in let on_error err = Events.(emit error_while_baking err) >>= fun () -> diff --git a/src/proto_alpha/lib_delegate/baking_scheduling.mli b/src/proto_alpha/lib_delegate/baking_scheduling.mli index 83e167cfcf34..ed09245e450d 100644 --- a/src/proto_alpha/lib_delegate/baking_scheduling.mli +++ b/src/proto_alpha/lib_delegate/baking_scheduling.mli @@ -29,7 +29,10 @@ open Protocol.Alpha_context type loop_state val create_loop_state : - proposal Lwt_stream.t -> Operation_worker.t -> loop_state + ?get_valid_blocks_stream:proposal Lwt_stream.t Lwt.t -> + heads_stream:proposal Lwt_stream.t -> + Operation_worker.t -> + loop_state val sleep_until : Time.Protocol.t -> unit Lwt.t option diff --git a/src/proto_alpha/lib_delegate/baking_state.ml b/src/proto_alpha/lib_delegate/baking_state.ml index 2d96e3881533..a953e87deeed 100644 --- a/src/proto_alpha/lib_delegate/baking_state.ml +++ b/src/proto_alpha/lib_delegate/baking_state.ml @@ -356,7 +356,8 @@ let timeout_kind_encoding = ] type event = - | New_proposal of proposal + | New_valid_proposal of proposal + | New_head_proposal of proposal | Prequorum_reached of Operation_worker.candidate * Kind.preendorsement operation list | Quorum_reached of @@ -369,12 +370,18 @@ let event_encoding = [ case (Tag 0) - ~title:"New_proposal" + ~title:"New_valid_proposal" proposal_encoding - (function New_proposal p -> Some p | _ -> None) - (fun p -> New_proposal p); + (function New_valid_proposal p -> Some p | _ -> None) + (fun p -> New_valid_proposal p); case (Tag 1) + ~title:"New_head_proposal" + proposal_encoding + (function New_head_proposal p -> Some p | _ -> None) + (fun p -> New_head_proposal p); + case + (Tag 2) ~title:"Prequorum_reached" (tup2 Operation_worker.candidate_encoding @@ -387,7 +394,7 @@ let event_encoding = Prequorum_reached (candidate, Operation_pool.filter_preendorsements ops)); case - (Tag 2) + (Tag 3) ~title:"Quorum_reached" (tup2 Operation_worker.candidate_encoding @@ -399,7 +406,7 @@ let event_encoding = (fun (candidate, ops) -> Quorum_reached (candidate, Operation_pool.filter_endorsements ops)); case - (Tag 3) + (Tag 4) ~title:"Timeout" timeout_kind_encoding (function Timeout tk -> Some tk | _ -> None) @@ -843,10 +850,16 @@ let pp_timeout_kind fmt = function Format.fprintf fmt "time to bake next level at round %a" Round.pp at_round let pp_event fmt = function - | New_proposal proposal -> + | New_valid_proposal proposal -> + Format.fprintf + fmt + "new valid proposal received: %a" + pp_block_info + proposal.block + | New_head_proposal proposal -> Format.fprintf fmt - "new proposal received: %a" + "new applied proposal received: %a" pp_block_info proposal.block | Prequorum_reached (candidate, preendos) -> diff --git a/src/proto_alpha/lib_delegate/baking_state.mli b/src/proto_alpha/lib_delegate/baking_state.mli index 47596470a655..d0b3785e3ec0 100644 --- a/src/proto_alpha/lib_delegate/baking_state.mli +++ b/src/proto_alpha/lib_delegate/baking_state.mli @@ -158,7 +158,8 @@ type timeout_kind = val timeout_kind_encoding : timeout_kind Data_encoding.t type event = - | New_proposal of proposal + | New_valid_proposal of proposal + | New_head_proposal of proposal | Prequorum_reached of Operation_worker.candidate * Kind.preendorsement operation list | Quorum_reached of diff --git a/src/proto_alpha/lib_delegate/node_rpc.ml b/src/proto_alpha/lib_delegate/node_rpc.ml index 2f7e86e4fa7f..b82cf6b8964a 100644 --- a/src/proto_alpha/lib_delegate/node_rpc.ml +++ b/src/proto_alpha/lib_delegate/node_rpc.ml @@ -253,7 +253,7 @@ let monitor_heads cctxt ~chain ?cache () = let* block_stream, stopper = Monitor_services.heads cctxt ~next_protocols chain in - let stream, stopper = + let stream = let map (block_hash, block_header) = let*! map_result = proposal cctxt ?cache ~chain block_hash block_header in match map_result with @@ -262,7 +262,7 @@ let monitor_heads cctxt ~chain ?cache () = let*! () = Events.(emit error_while_monitoring_heads err) in Lwt.return_none in - (Lwt_stream.filter_map_s map block_stream, stopper) + Lwt_stream.filter_map_s map block_stream in return (stream, stopper) diff --git a/src/proto_alpha/lib_delegate/state_transitions.ml b/src/proto_alpha/lib_delegate/state_transitions.ml index 43d07f528eb2..63559a80e202 100644 --- a/src/proto_alpha/lib_delegate/state_transitions.ml +++ b/src/proto_alpha/lib_delegate/state_transitions.ml @@ -699,7 +699,7 @@ let step (state : Baking_state.t) (event : Baking_state.event) : (* If it is time to bake the next level, stop everything currently going on and propose the next level block *) time_to_bake state at_round - | Idle, New_proposal block_info -> + | Idle, New_head_proposal block_info -> Events.( emit new_head @@ -707,8 +707,8 @@ let step (state : Baking_state.t) (event : Baking_state.event) : block_info.block.shell.level, block_info.block.round )) >>= fun () -> handle_new_proposal state block_info - | Awaiting_endorsements, New_proposal block_info - | Awaiting_preendorsements, New_proposal block_info -> + | Awaiting_endorsements, New_head_proposal block_info + | Awaiting_preendorsements, New_head_proposal block_info -> Events.( emit new_head @@ -732,3 +732,6 @@ let step (state : Baking_state.t) (event : Baking_state.event) : | Awaiting_endorsements, Prequorum_reached _ -> (* This cannot/should not happen *) do_nothing state + | _, New_valid_proposal _p -> + (* TODO: actually do something *) + do_nothing state diff --git a/src/proto_alpha/lib_delegate/test/test_scenario.ml b/src/proto_alpha/lib_delegate/test/test_scenario.ml index cb34cc8d3d63..86516ab4bb4f 100644 --- a/src/proto_alpha/lib_delegate/test/test_scenario.ml +++ b/src/proto_alpha/lib_delegate/test/test_scenario.ml @@ -24,7 +24,7 @@ let test_level_5 () = include Default_hooks let stop_on_event = function - | Baking_state.New_proposal {block; _} -> + | Baking_state.New_head_proposal {block; _} -> (* Stop the node as soon as we receive a proposal with a level higher than [level_to_reach]. *) block.shell.level > level_to_reach @@ -649,7 +649,7 @@ let test_scenario_m1 () = return (op_hash, op, propagation_vector) let stop_on_event = function - | Baking_state.New_proposal {block; _} -> block.shell.level > 4l + | Baking_state.New_head_proposal {block; _} -> block.shell.level > 4l | _ -> false end in let config = {default_config with timeout = 60} in @@ -679,7 +679,7 @@ let test_scenario_m2 () = include Default_hooks let stop_on_event = function - | Baking_state.New_proposal {block; _} -> block.shell.level > 5l + | Baking_state.New_head_proposal {block; _} -> block.shell.level > 5l | _ -> false end in let module Missing_node : Hooks = struct @@ -742,7 +742,7 @@ Scenario M3 let test_scenario_m3 () = let stop_on_event0 = function - | Baking_state.New_proposal {block; _} -> + | Baking_state.New_head_proposal {block; _} -> block.shell.level = 1l && Protocol.Alpha_context.Round.to_int32 block.round = 6l | _ -> false @@ -917,7 +917,7 @@ Scenario M5 let test_scenario_m5 () = let stop_on_event0 = function - | Baking_state.New_proposal {block; _} -> block.shell.level >= 2l + | Baking_state.New_head_proposal {block; _} -> block.shell.level >= 2l | _ -> false in let module Node_a_hooks : Hooks = struct @@ -1004,7 +1004,7 @@ Scenario M6 let test_scenario_m6 () = let b_proposal_2_1 = ref None in let stop_on_event0 = function - | Baking_state.New_proposal {block; _} -> block.shell.level > 4l + | Baking_state.New_head_proposal {block; _} -> block.shell.level > 4l | _ -> false in let module Node_a_hooks : Hooks = struct @@ -1134,7 +1134,7 @@ let test_scenario_m7 () = let c_received_2_1 = ref false in let d_received_2_1 = ref false in let stop_on_event0 = function - | Baking_state.New_proposal {block; _} -> block.shell.level > 4l + | Baking_state.New_head_proposal {block; _} -> block.shell.level > 4l | _ -> false in let check_chain_on_success0 node_label ~chain = @@ -1338,7 +1338,7 @@ Scenario M8 let test_scenario_m8 () = let b_proposal_2_0 = ref None in let stop_on_event0 = function - | Baking_state.New_proposal {block; _} -> block.shell.level > 4l + | Baking_state.New_head_proposal {block; _} -> block.shell.level > 4l | _ -> false in let on_inject_operation0 ~op_hash ~op = -- GitLab From e8b8dc075b5be126053d1e52fce0b4eecad8f1e7 Mon Sep 17 00:00:00 2001 From: vbot Date: Thu, 26 Jan 2023 16:34:20 +0100 Subject: [PATCH 32/46] Alpha/Baker: start preendorsing as soon as a valid block arrives --- src/proto_alpha/lib_delegate/baking_events.ml | 78 +++++++ .../lib_delegate/baking_scheduling.ml | 3 + src/proto_alpha/lib_delegate/baking_state.ml | 32 ++- src/proto_alpha/lib_delegate/baking_state.mli | 9 +- .../lib_delegate/operation_worker.ml | 9 +- .../lib_delegate/state_transitions.ml | 211 ++++++++++++++---- .../lib_delegate/state_transitions.mli | 6 +- .../lib_delegate/test/test_scenario.ml | 2 +- 8 files changed, 290 insertions(+), 60 deletions(-) diff --git a/src/proto_alpha/lib_delegate/baking_events.ml b/src/proto_alpha/lib_delegate/baking_events.ml index 6b6327721fe1..f4383307a31e 100644 --- a/src/proto_alpha/lib_delegate/baking_events.ml +++ b/src/proto_alpha/lib_delegate/baking_events.ml @@ -37,6 +37,19 @@ module State_transitions = struct let section = section @ ["transitions"] + let new_valid_proposal = + declare_3 + ~section + ~name:"new_valid_proposal" + ~level:Notice + ~msg:"received new proposal {block} at level {level}, round {round}" + ~pp1:Block_hash.pp + ("block", Block_hash.encoding) + ~pp2:pp_int32 + ("level", Data_encoding.int32) + ~pp3:Round.pp + ("round", Round.encoding) + let new_head = declare_3 ~section @@ -98,6 +111,63 @@ module State_transitions = struct ~msg:"received new head while waiting for a quorum" () + let applied_expected_proposal_received = + declare_1 + ~section + ~name:"applied_expected_proposal_received" + ~level:Info + ~msg:"received the expected application notice for {proposal}" + ~pp1:Block_hash.pp + ("proposal", Block_hash.encoding) + + let unexpected_new_head_while_waiting_for_application = + declare_0 + ~section + ~name:"unexpected_new_head_while_waiting_for_application" + ~level:Info + ~msg:"received new head while waiting for another proposal's application" + () + + let new_valid_proposal_while_waiting_for_qc = + declare_0 + ~section + ~name:"new_valid_proposal_while_waiting_for_qc" + ~level:Info + ~msg:"received new valid proposal while waiting for a quorum" + () + + let valid_proposal_received_after_application = + declare_0 + ~section + ~name:"valid_proposal_received_after_application" + ~level:Info + ~msg:"received valid proposal for a block already applied" + () + + let unexpected_pqc_while_waiting_for_application = + declare_2 + ~section + ~name:"unexpected_pqc_while_waiting_for_application" + ~level:Info + ~msg: + "received an unexpected prequorum for {prequorum} while waiting for \ + the proposal's {proposal} application" + ~pp1:Block_hash.pp + ("prequorum", Block_hash.encoding) + ~pp2:Block_hash.pp + ("proposal", Block_hash.encoding) + + let pqc_while_waiting_for_application = + declare_1 + ~section + ~name:"pqc_while_waiting_for_application" + ~level:Info + ~msg: + "received expected prequorum for {prequorum} while waiting for the \ + proposal's application" + ~pp1:Block_hash.pp + ("prequorum", Block_hash.encoding) + let unexpected_proposal_round = declare_2 ~section @@ -289,6 +359,14 @@ module State_transitions = struct ~pp2:Block_hash.pp ("expected_hash", Block_hash.encoding) + let handling_prequorum_on_non_applied_proposal = + declare_0 + ~section + ~name:"handling_prequorum_on_non_applied_proposal" + ~level:Error + ~msg:"Handling prequorum on a non-applied proposal" + () + let step_current_phase = declare_2 ~section diff --git a/src/proto_alpha/lib_delegate/baking_scheduling.ml b/src/proto_alpha/lib_delegate/baking_scheduling.ml index 668218d08c96..e974c5b0b0d0 100644 --- a/src/proto_alpha/lib_delegate/baking_scheduling.ml +++ b/src/proto_alpha/lib_delegate/baking_scheduling.ml @@ -676,6 +676,9 @@ let create_initial_state cctxt ?(synchronize = true) ~chain config { current_level = current_proposal.block.shell.level; latest_proposal = current_proposal; + is_latest_proposal_applied = + true (* this proposal is expected to be the current head *); + delayed_prequorum = None; locked_round = None; endorsable_payload = None; elected_block; diff --git a/src/proto_alpha/lib_delegate/baking_state.ml b/src/proto_alpha/lib_delegate/baking_state.ml index a953e87deeed..ccd2a6951bff 100644 --- a/src/proto_alpha/lib_delegate/baking_state.ml +++ b/src/proto_alpha/lib_delegate/baking_state.ml @@ -279,6 +279,9 @@ type elected_block = { type level_state = { current_level : int32; latest_proposal : proposal; + is_latest_proposal_applied : bool; + delayed_prequorum : + (Operation_worker.candidate * Kind.preendorsement operation list) option; (* Last proposal received where we injected an endorsement (thus we have seen 2f+1 preendorsements) *) locked_round : locked_round option; @@ -291,7 +294,11 @@ type level_state = { next_level_proposed_round : Round.t option; } -type phase = Idle | Awaiting_preendorsements | Awaiting_endorsements +type phase = + | Idle + | Awaiting_preendorsements + | Awaiting_application + | Awaiting_endorsements let phase_encoding = let open Data_encoding in @@ -311,9 +318,15 @@ let phase_encoding = (function Awaiting_preendorsements -> Some () | _ -> None) (fun () -> Awaiting_preendorsements); case - ~title:"Awaiting_endorsements" + ~title:"Awaiting_application" (Tag 2) unit + (function Awaiting_application -> Some () | _ -> None) + (fun () -> Awaiting_application); + case + ~title:"Awaiting_endorsements" + (Tag 3) + unit (function Awaiting_endorsements -> Some () | _ -> None) (fun () -> Awaiting_endorsements); ] @@ -789,6 +802,8 @@ let pp_level_state fmt { current_level; latest_proposal; + is_latest_proposal_applied; + delayed_prequorum; locked_round; endorsable_payload; elected_block; @@ -798,11 +813,13 @@ let pp_level_state fmt } = Format.fprintf fmt - "@[Level state:@ current level: %ld@ @[proposal:@ %a@]@ locked \ - round: %a@ endorsable payload: %a@ elected block: %a@ @[own delegate \ - slots:@ %a@]@ @[next level own delegate slots:@ %a@]@ next level \ - proposed round: %a@]" + "@[Level state:@ current level: %ld@ @[proposal (applied:%b, \ + delayed prequorum:%b):@ %a@]@ locked round: %a@ endorsable payload: %a@ \ + elected block: %a@ @[own delegate slots:@ %a@]@ @[next level \ + own delegate slots:@ %a@]@ next level proposed round: %a@]" current_level + is_latest_proposal_applied + (Option.is_some delayed_prequorum) pp_proposal latest_proposal (pp_option pp_locked_round) @@ -821,6 +838,7 @@ let pp_level_state fmt let pp_phase fmt = function | Idle -> Format.fprintf fmt "idle" | Awaiting_preendorsements -> Format.fprintf fmt "awaiting preendorsements" + | Awaiting_application -> Format.fprintf fmt "awaiting application" | Awaiting_endorsements -> Format.fprintf fmt "awaiting endorsements" let pp_round_state fmt {current_round; current_phase} = @@ -859,7 +877,7 @@ let pp_event fmt = function | New_head_proposal proposal -> Format.fprintf fmt - "new applied proposal received: %a" + "new head proposal received: %a" pp_block_info proposal.block | Prequorum_reached (candidate, preendos) -> diff --git a/src/proto_alpha/lib_delegate/baking_state.mli b/src/proto_alpha/lib_delegate/baking_state.mli index d0b3785e3ec0..02243593d8ac 100644 --- a/src/proto_alpha/lib_delegate/baking_state.mli +++ b/src/proto_alpha/lib_delegate/baking_state.mli @@ -127,6 +127,9 @@ type elected_block = { type level_state = { current_level : int32; latest_proposal : proposal; + is_latest_proposal_applied : bool; + delayed_prequorum : + (Operation_worker.candidate * Kind.preendorsement operation list) option; locked_round : locked_round option; endorsable_payload : endorsable_payload option; elected_block : elected_block option; @@ -135,7 +138,11 @@ type level_state = { next_level_proposed_round : Round.t option; } -type phase = Idle | Awaiting_preendorsements | Awaiting_endorsements +type phase = + | Idle + | Awaiting_preendorsements + | Awaiting_application + | Awaiting_endorsements val phase_encoding : phase Data_encoding.t diff --git a/src/proto_alpha/lib_delegate/operation_worker.ml b/src/proto_alpha/lib_delegate/operation_worker.ml index 1e20efa00156..9f54d826da64 100644 --- a/src/proto_alpha/lib_delegate/operation_worker.ml +++ b/src/proto_alpha/lib_delegate/operation_worker.ml @@ -23,13 +23,6 @@ (* *) (*****************************************************************************) -(* TODO: - add events + - running state introspection to recover/restart on failure - - Do we need a mutex ? -*) - open Protocol_client_context open Protocol open Alpha_context @@ -64,7 +57,7 @@ module Events = struct ~name:"pqc_reached" ~level:Debug ~msg: - "pre-quorum reached (voting power: {voting_power}, {preendorsements} \ + "prequorum reached (voting power: {voting_power}, {preendorsements} \ preendorsements)" ~pp1:pp_int ("voting_power", Data_encoding.int31) diff --git a/src/proto_alpha/lib_delegate/state_transitions.ml b/src/proto_alpha/lib_delegate/state_transitions.ml index 63559a80e202..79640027e695 100644 --- a/src/proto_alpha/lib_delegate/state_transitions.ml +++ b/src/proto_alpha/lib_delegate/state_transitions.ml @@ -95,19 +95,33 @@ let make_preendorse_action state proposal = in Inject_preendorsements {preendorsements} -let update_proposal state proposal = +let update_proposal ~is_proposal_applied state proposal = Events.(emit updating_latest_proposal proposal.block.hash) >>= fun () -> - let new_level_state = {state.level_state with latest_proposal = proposal} in + let prev_proposal = state.level_state.latest_proposal in + let is_latest_proposal_applied = + (* mark as applied if it is indeed applied or if this specific proposal was + already marked as applied *) + is_proposal_applied + || prev_proposal.block.hash = proposal.block.hash + && state.level_state.is_latest_proposal_applied + in + let new_level_state = + { + state.level_state with + is_latest_proposal_applied; + latest_proposal = proposal; + } + in Lwt.return {state with level_state = new_level_state} -let may_update_proposal state (proposal : proposal) = +let may_update_proposal ~is_proposal_applied state (proposal : proposal) = assert ( Compare.Int32.( state.level_state.latest_proposal.block.shell.level = proposal.block.shell.level)) ; if Round.(state.level_state.latest_proposal.block.round < proposal.block.round) - then update_proposal state proposal + then update_proposal ~is_proposal_applied state proposal else Lwt.return state let preendorse state proposal = @@ -118,7 +132,13 @@ let preendorse state proposal = else Events.(emit attempting_preendorse_proposal proposal.block.hash) >>= fun () -> - let new_state = update_current_phase state Awaiting_preendorsements in + let new_state = + (* We await for the block to be applied before updating its + locked values. *) + if state.level_state.is_latest_proposal_applied then + update_current_phase state Awaiting_preendorsements + else update_current_phase state Awaiting_application + in Lwt.return (new_state, make_preendorse_action state proposal) let extract_pqc state (new_proposal : proposal) = @@ -178,9 +198,35 @@ let may_update_endorsable_payload_with_internal_pqc state in {state with level_state = new_level_state} -let rec handle_new_proposal state (new_proposal : proposal) = +let may_update_is_latest_proposal_applied ~is_proposal_applied state + new_proposal = + let current_proposal = state.level_state.latest_proposal in + if + is_proposal_applied + && Block_hash.(current_proposal.block.hash = new_proposal.block.hash) + then + let new_level_state = + {state.level_state with is_latest_proposal_applied = true} + in + let new_state = {state with level_state = new_level_state} in + new_state + else state + +let has_already_been_handled state new_proposal = + let current_proposal = state.level_state.latest_proposal in + Block_hash.(current_proposal.block.hash = new_proposal.block.hash) + && state.level_state.is_latest_proposal_applied + +let rec handle_proposal ~is_proposal_applied state (new_proposal : proposal) = let current_level = state.level_state.current_level in let new_proposal_level = new_proposal.block.shell.level in + let current_proposal = state.level_state.latest_proposal in + let state = + may_update_is_latest_proposal_applied + ~is_proposal_applied + state + new_proposal + in if Compare.Int32.(current_level > new_proposal_level) then (* The baker is ahead, a reorg may have happened. Do nothing: wait for the node to send us the branch's head. This new head @@ -188,12 +234,11 @@ let rec handle_new_proposal state (new_proposal : proposal) = proposal and thus, its level should be at least the same as our current proposal's level. *) Events.(emit baker_is_ahead_of_node (current_level, new_proposal_level)) - >>= fun () -> Lwt.return (state, Do_nothing) + >>= fun () -> do_nothing state else if Compare.Int32.(current_level = new_proposal_level) then - (* The received head is a new proposal for the current level: - let's check if it's a valid one for us. *) - let current_proposal = state.level_state.latest_proposal in if + (* The received head is a new proposal for the current level: + let's check if it's a valid one for us. *) Block_hash.( current_proposal.predecessor.hash <> new_proposal.predecessor.hash) then @@ -201,7 +246,7 @@ let rec handle_new_proposal state (new_proposal : proposal) = emit new_proposal_is_on_another_branch (current_proposal.predecessor.hash, new_proposal.predecessor.hash)) - >>= fun () -> may_switch_branch state new_proposal + >>= fun () -> may_switch_branch ~is_proposal_applied state new_proposal else is_acceptable_proposal_for_current_level state new_proposal >>= function | Invalid -> @@ -216,15 +261,16 @@ let rec handle_new_proposal state (new_proposal : proposal) = (* The proposal is outdated: we update to be able to extract its included endorsements but we do not endorse it *) Events.(emit outdated_proposal new_proposal.block.hash) >>= fun () -> - may_update_proposal state new_proposal >>= fun state -> - do_nothing state + may_update_proposal ~is_proposal_applied state new_proposal + >>= fun state -> do_nothing state | Valid_proposal -> ( (* Valid_proposal => proposal.round = current_round *) (* Check whether we need to update our endorsable payload *) let new_state = may_update_endorsable_payload_with_internal_pqc state new_proposal in - may_update_proposal new_state new_proposal >>= fun new_state -> + may_update_proposal ~is_proposal_applied new_state new_proposal + >>= fun new_state -> (* The proposal is valid but maybe we already locked on a payload *) match new_state.level_state.locked_round with | Some locked_round -> ( @@ -242,18 +288,22 @@ let rec handle_new_proposal state (new_proposal : proposal) = (* This PQC is above our locked_round, we can preendorse it *) preendorse new_state new_proposal | _ -> - (* We shouldn't preendorse this proposal, but we should at - least watch (pre)quorums events on it *) - let new_state = - update_current_phase new_state Awaiting_preendorsements - in - Lwt.return (new_state, Watch_proposal)) + (* We shouldn't preendorse this proposal, but we + should at least watch (pre)quorums events on it + but only when it is applied otherwise we await + for the proposal to be applied. *) + if is_proposal_applied then + let new_state = + update_current_phase new_state Awaiting_preendorsements + in + Lwt.return (new_state, Watch_proposal) + else do_nothing new_state) | None -> (* Otherwise, we did not lock on any payload, thus we can preendorse it *) preendorse new_state new_proposal) else - (* new_proposal.level > current_level *) + (* Last case: new_proposal_level > current_level *) (* Possible scenarios: - we received a block for a next level - we received our own block @@ -267,6 +317,8 @@ let rec handle_new_proposal state (new_proposal : proposal) = { current_level = new_level; latest_proposal = new_proposal; + is_latest_proposal_applied = is_proposal_applied; + delayed_prequorum = None; (* Unlock values *) locked_round = None; endorsable_payload = None; @@ -278,19 +330,22 @@ let rec handle_new_proposal state (new_proposal : proposal) = in (* recursive call with the up-to-date state to handle the new level proposals *) - handle_new_proposal {state with level_state; round_state} new_proposal + handle_proposal + ~is_proposal_applied + {state with level_state; round_state} + new_proposal in let action = Update_to_level {new_level_proposal = new_proposal; compute_new_state} in Lwt.return (state, action) -and may_switch_branch state new_proposal = +and may_switch_branch ~is_proposal_applied state new_proposal = let switch_branch state = Events.(emit switching_branch ()) >>= fun () -> (* If we are on a different branch, we also need to update our [round_state] accordingly. - The recursive call to [handle_new_proposal] cannot end up + The recursive call to [handle_proposal] cannot end up with an invalid proposal as it's on a different branch, thus there is no need to backtrack to the former state as the new proposal must end up being the new [latest_proposal]. That's @@ -298,10 +353,11 @@ and may_switch_branch state new_proposal = let round_update = { Baking_actions.new_round_proposal = new_proposal; - handle_proposal = (fun state -> handle_new_proposal state new_proposal); + handle_proposal = + (fun state -> handle_proposal ~is_proposal_applied state new_proposal); } in - update_proposal state new_proposal >>= fun new_state -> + update_proposal ~is_proposal_applied state new_proposal >>= fun new_state -> (* TODO if the branch proposal is outdated, we should trigger an [End_of_round] to participate *) Lwt.return (new_state, Synchronize_round round_update) @@ -337,6 +393,25 @@ and may_switch_branch state new_proposal = Events.(emit branch_proposal_has_same_prequorum ()) >>= fun () -> do_nothing state +let may_register_early_prequorum state ((candidate, _) as received_prequorum) = + if + Block_hash.( + candidate.Operation_worker.hash + <> state.level_state.latest_proposal.block.hash) + then + Events.( + emit + unexpected_pqc_while_waiting_for_application + (candidate.hash, state.level_state.latest_proposal.block.hash)) + >>= fun () -> do_nothing state + else + Events.(emit pqc_while_waiting_for_application candidate.hash) >>= fun () -> + let new_level_state = + {state.level_state with delayed_prequorum = Some received_prequorum} + in + let new_state = {state with level_state = new_level_state} in + do_nothing new_state + (** In the association map [delegate_slots], the function returns an optional pair ([delegate], [endorsing_slot]) if for the current [round], the validator [delegate] has a endorsing slot. *) @@ -606,6 +681,9 @@ let prequorum_reached_when_awaiting_preendorsements state candidate unexpected_prequorum_received (candidate.hash, latest_proposal.block.hash)) >>= fun () -> do_nothing state + else if not state.level_state.is_latest_proposal_applied then + Events.(emit handling_prequorum_on_non_applied_proposal ()) >>= fun () -> + do_nothing state else let prequorum = { @@ -672,6 +750,24 @@ let quorum_reached_when_waiting_endorsements state candidate endorsement_qc = in do_nothing new_state +let handle_expected_applied_proposal (state : Baking_state.t) = + let new_level_state = + {state.level_state with is_latest_proposal_applied = true} + in + let new_state = {state with level_state = new_level_state} in + match new_state.level_state.delayed_prequorum with + | None -> + (* The application arrived before the prequorum: just wait for the prequorum. *) + let new_state = update_current_phase new_state Awaiting_preendorsements in + do_nothing new_state + | Some (candidate, preendorsement_qc) -> + (* The application arrived after the prequorum: handle the + prequorum received earlier. *) + prequorum_reached_when_awaiting_preendorsements + new_state + candidate + preendorsement_qc + (* Hypothesis: - The state is not to be modified outside this module (NB: there are exceptions in Baking_actions: the corner cases @@ -699,25 +795,60 @@ let step (state : Baking_state.t) (event : Baking_state.event) : (* If it is time to bake the next level, stop everything currently going on and propose the next level block *) time_to_bake state at_round - | Idle, New_head_proposal block_info -> + | Idle, New_head_proposal proposal -> Events.( emit new_head - ( block_info.block.hash, - block_info.block.shell.level, - block_info.block.round )) - >>= fun () -> handle_new_proposal state block_info - | Awaiting_endorsements, New_head_proposal block_info - | Awaiting_preendorsements, New_head_proposal block_info -> + (proposal.block.hash, proposal.block.shell.level, proposal.block.round)) + >>= fun () -> handle_proposal ~is_proposal_applied:true state proposal + | Awaiting_application, New_head_proposal proposal -> + if + Block_hash.( + state.level_state.latest_proposal.block.hash <> proposal.block.hash) + then + Events.( + emit + new_head + ( proposal.block.hash, + proposal.block.shell.level, + proposal.block.round )) + >>= fun () -> + Events.(emit unexpected_new_head_while_waiting_for_application ()) + >>= fun () -> handle_proposal ~is_proposal_applied:true state proposal + else + Events.(emit applied_expected_proposal_received proposal.block.hash) + >>= fun () -> handle_expected_applied_proposal state + | Awaiting_endorsements, New_head_proposal proposal + | Awaiting_preendorsements, New_head_proposal proposal -> Events.( emit new_head - ( block_info.block.hash, - block_info.block.shell.level, - block_info.block.round )) + (proposal.block.hash, proposal.block.shell.level, proposal.block.round)) >>= fun () -> Events.(emit new_head_while_waiting_for_qc ()) >>= fun () -> - handle_new_proposal state block_info + handle_proposal ~is_proposal_applied:true state proposal + | Idle, New_valid_proposal proposal -> + Events.( + emit + new_valid_proposal + (proposal.block.hash, proposal.block.shell.level, proposal.block.round)) + >>= fun () -> handle_proposal ~is_proposal_applied:false state proposal + | Awaiting_application, New_valid_proposal proposal + | Awaiting_endorsements, New_valid_proposal proposal + | Awaiting_preendorsements, New_valid_proposal proposal -> + Events.( + emit + new_valid_proposal + (proposal.block.hash, proposal.block.shell.level, proposal.block.round)) + >>= fun () -> + if has_already_been_handled state proposal then + Events.(emit valid_proposal_received_after_application ()) >>= fun () -> + do_nothing state + else + Events.(emit new_valid_proposal_while_waiting_for_qc ()) >>= fun () -> + handle_proposal ~is_proposal_applied:false state proposal + | Awaiting_application, Prequorum_reached (candidate, preendorsement_qc) -> + may_register_early_prequorum state (candidate, preendorsement_qc) | Awaiting_preendorsements, Prequorum_reached (candidate, preendorsement_qc) -> prequorum_reached_when_awaiting_preendorsements @@ -729,9 +860,7 @@ let step (state : Baking_state.t) (event : Baking_state.event) : (* Unreachable cases *) | Idle, (Prequorum_reached _ | Quorum_reached _) | Awaiting_preendorsements, Quorum_reached _ - | Awaiting_endorsements, Prequorum_reached _ -> + | Awaiting_endorsements, Prequorum_reached _ + | Awaiting_application, Quorum_reached _ -> (* This cannot/should not happen *) do_nothing state - | _, New_valid_proposal _p -> - (* TODO: actually do something *) - do_nothing state diff --git a/src/proto_alpha/lib_delegate/state_transitions.mli b/src/proto_alpha/lib_delegate/state_transitions.mli index adc584312329..596efad2100f 100644 --- a/src/proto_alpha/lib_delegate/state_transitions.mli +++ b/src/proto_alpha/lib_delegate/state_transitions.mli @@ -43,14 +43,16 @@ val make_consensus_list : val make_preendorse_action : state -> proposal -> action -val may_update_proposal : state -> proposal -> state Lwt.t +val may_update_proposal : + is_proposal_applied:bool -> state -> proposal -> state Lwt.t val preendorse : state -> proposal -> (state * action) Lwt.t val extract_pqc : state -> proposal -> (Kind.preendorsement operation list * Round.t) option -val handle_new_proposal : state -> proposal -> (state * action) Lwt.t +val handle_proposal : + is_proposal_applied:bool -> state -> proposal -> (state * action) Lwt.t val round_proposer : state -> diff --git a/src/proto_alpha/lib_delegate/test/test_scenario.ml b/src/proto_alpha/lib_delegate/test/test_scenario.ml index 86516ab4bb4f..afc642bf9457 100644 --- a/src/proto_alpha/lib_delegate/test/test_scenario.ml +++ b/src/proto_alpha/lib_delegate/test/test_scenario.ml @@ -734,7 +734,7 @@ Scenario M3 from other nodes only go to A. 3. The chain should not make progress. Since we have both bootstrap1 and bootstrap2 in delegate selection they have equal voting power. Therefore - it is necessary to have 2 votes for pre-quorums (which is achieved when A + it is necessary to have 2 votes for prequorums (which is achieved when A is proposing) and 2 votes for quorums (impossible because B has no way to obtain PQC and thus cannot send endorsements). -- GitLab From d80ee56879c53354c41dd08d0b3c802db68d65a9 Mon Sep 17 00:00:00 2001 From: vbot Date: Wed, 1 Feb 2023 11:53:03 +0100 Subject: [PATCH 33/46] Mockup: add newly introduced RPCs --- src/lib_mockup/local_services.ml | 40 ++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/lib_mockup/local_services.ml b/src/lib_mockup/local_services.ml index 9a6889d19a0c..bfb1e1765a4f 100644 --- a/src/lib_mockup/local_services.ml +++ b/src/lib_mockup/local_services.ml @@ -855,6 +855,26 @@ module Make (E : MENV) = struct write_context_callback operation_bytes) + let monitor_validated_blocks () = + Directory.register + Directory.empty + Tezos_shell_services.Monitor_services.S.validated_blocks + (fun () q () -> + let chain = + match List.hd q#chains with None -> `Main | Some chain -> chain + in + with_chain ~caller_name:"monitor validated blocks" chain (fun () -> + let block_header = + Block_header. + { + shell = E.rpc_context.block_header; + protocol_data = E.protocol_data; + } + in + let block_hash = E.rpc_context.block_hash in + Tezos_rpc.Answer.return + (E.chain_id, block_hash, block_header, [[]; []; []; []]))) + let monitor_heads () = Directory.register Directory.empty @@ -871,6 +891,24 @@ module Make (E : MENV) = struct let block_hash = E.rpc_context.block_hash in Tezos_rpc.Answer.return (block_hash, block_header))) + let raw_header () = + Directory.prefix + (Tezos_rpc.Path.prefix Chain_services.path Block_services.path) + (Directory.register + Directory.empty + E.Block_services.S.raw_header + (fun (((), chain), _block) () () -> + with_chain ~caller_name:"raw header" chain (fun () -> + let block_header_b = + Data_encoding.Binary.to_bytes_exn + Tezos_base.Block_header.encoding + { + shell = E.rpc_context.block_header; + protocol_data = E.protocol_data; + } + in + Tezos_rpc.Answer.return block_header_b))) + let header () = Directory.prefix (Tezos_rpc.Path.prefix Chain_services.path Block_services.path) @@ -962,7 +1000,9 @@ module Make (E : MENV) = struct |> merge (inject_block write_context_callback) |> merge (live_blocks ()) |> merge (preapply_block ()) + |> merge (monitor_validated_blocks ()) |> merge (monitor_heads ()) + |> merge (raw_header ()) |> merge (header ()) |> merge (operations ()) |> merge (resulting_context_hash ()) -- GitLab From d33c2415401a81dedee0073d596d694db88fa729 Mon Sep 17 00:00:00 2001 From: vbot Date: Tue, 31 Jan 2023 17:58:07 +0100 Subject: [PATCH 34/46] Alpha/Mockup: add the new necessary RPCs in the mockup baker --- .../test/mockup_simulator/faked_services.ml | 61 ++++++++++++++++--- .../test/mockup_simulator/mockup_simulator.ml | 58 +++++++++++++++++- .../mockup_simulator/mockup_simulator.mli | 9 +++ 3 files changed, 116 insertions(+), 12 deletions(-) diff --git a/src/proto_alpha/lib_delegate/test/mockup_simulator/faked_services.ml b/src/proto_alpha/lib_delegate/test/mockup_simulator/faked_services.ml index 624565b40889..e99d009738bf 100644 --- a/src/proto_alpha/lib_delegate/test/mockup_simulator/faked_services.ml +++ b/src/proto_alpha/lib_delegate/test/mockup_simulator/faked_services.ml @@ -8,7 +8,14 @@ module type Mocked_services_hooks = sig type mempool = Mockup.M.Block_services.Mempool.t (** The baker and endorser rely on this stream to be notified of new - blocks. *) + valid blocks. *) + val monitor_validated_blocks : + unit -> + (Chain_id.t * Block_hash.t * Block_header.t * Operation.t list list) + Tezos_rpc.Answer.stream + + (** The baker and endorser rely on this stream to be notified of new + heads. *) val monitor_heads : unit -> (Block_hash.t * Block_header.t) Tezos_rpc.Answer.stream @@ -16,6 +23,10 @@ module type Mocked_services_hooks = sig val protocols : Block_services.block -> Block_services.protocols tzresult Lwt.t + (** [raw_header] returns the byte encoded block header of the block + associated to the given block specification. *) + val raw_header : Block_services.block -> bytes tzresult Lwt.t + (** [header] returns the block header of the block associated to the given block specification. *) val header : @@ -112,6 +123,13 @@ end type hooks = (module Mocked_services_hooks) module Make (Hooks : Mocked_services_hooks) = struct + let monitor_validated_blocks = + Directory.gen_register0 + Directory.empty + Monitor_services.S.validated_blocks + (fun _next_protocol _ -> + Tezos_rpc.Answer.return_stream (Hooks.monitor_validated_blocks ())) + let monitor_heads = Directory.gen_register1 Directory.empty @@ -137,6 +155,14 @@ module Make (Hooks : Mocked_services_hooks) = struct Directory.register Directory.empty service (fun (_, block) () () -> Hooks.protocols block) + let raw_header = + Directory.prefix + (Tezos_rpc.Path.prefix Chain_services.path Block_services.path) + @@ Directory.register + Directory.empty + Mockup.M.Block_services.S.raw_header + (fun (((), _chain), block) _ _ -> Hooks.raw_header block) + let header = Directory.prefix (Tezos_rpc.Path.prefix Chain_services.path Block_services.path) @@ -281,15 +307,30 @@ module Make (Hooks : Mocked_services_hooks) = struct (fun (_, block) () () -> Hooks.raw_protocol_data block) let shell_directory chain_id = - let merge = Directory.merge in - Directory.empty |> merge monitor_heads |> merge protocols |> merge header - |> merge operations |> merge hash |> merge shell_header - |> merge resulting_context_hash - |> merge (chain chain_id) - |> merge inject_block |> merge inject_operation |> merge monitor_operations - |> merge list_blocks |> merge live_blocks |> merge raw_protocol_data - |> merge broadcast_block |> merge broadcast_operation - |> merge monitor_bootstrapped + List.fold_left + Directory.merge + Directory.empty + [ + monitor_validated_blocks; + monitor_heads; + protocols; + raw_header; + header; + operations; + hash; + shell_header; + resulting_context_hash; + chain chain_id; + inject_block; + inject_operation; + monitor_operations; + list_blocks; + live_blocks; + raw_protocol_data; + broadcast_block; + broadcast_operation; + monitor_bootstrapped; + ] let directory chain_id = let proto_directory = diff --git a/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.ml b/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.ml index 346cea3ee712..2c4d8230c19f 100644 --- a/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.ml +++ b/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.ml @@ -60,8 +60,13 @@ type state = { that functionality. *) ctxt_table : Tezos_protocol_environment.rpc_context Context_hash.Table.t; (** The context table allows us to look up rpc_context by its hash. *) + validated_blocks_pipe : + (Block_hash.t * Block_header.t * Operation.t list list) Lwt_pipe.Unbounded.t; + (** [validated_blocks_pipe] is used to implement the + [monitor_validated_blocks] RPC. *) heads_pipe : (Block_hash.t * Block_header.t) Lwt_pipe.Unbounded.t; - (** [heads_pipe] is used to implement the [monitor_heads] RPC. *) + (** [heads_pipe] is used to implement the [monitor_heads] + RPC. *) operations_pipe : (Operation_hash.t * Mockup.M.Protocol.operation) option Lwt_pipe.Unbounded.t; (** [operations_pipe] is used to implement the [operations_pipe] RPC. *) @@ -108,6 +113,12 @@ module type Hooks = sig tzresult Lwt.t + val on_new_validated_block : + block_hash:Block_hash.t -> + block_header:Block_header.t -> + operations:Operation.t list list -> + (Block_hash.t * Block_header.t * Operation.t list list) option Lwt.t + val on_new_head : block_hash:Block_hash.t -> block_header:Block_header.t -> @@ -238,11 +249,32 @@ let make_mocked_services_hooks (state : state) (user_hooks : (module Hooks)) : let module Impl : Faked_services.Mocked_services_hooks = struct type mempool = Mockup.M.Block_services.Mempool.t + let monitor_validated_blocks () = + let next () = + let rec pop_until_ok () = + Lwt_pipe.Unbounded.pop state.validated_blocks_pipe + >>= fun (block_hash, block_header, operations) -> + User_hooks.on_new_validated_block + ~block_hash + ~block_header + ~operations + >>= function + | None -> pop_until_ok () + | Some (hash, head, operations) -> + Lwt.return_some (chain_id, hash, head, operations) + in + pop_until_ok () + in + let shutdown () = () in + Tezos_rpc.Answer.{next; shutdown} + let monitor_heads () = let next () = let rec pop_until_ok () = Lwt_pipe.Unbounded.pop state.heads_pipe >>= fun (block_hash, block_header) -> + (* Sleep a 0.1s to simulate a block application delay *) + Lwt_unix.sleep 0.1 >>= fun () -> User_hooks.on_new_head ~block_hash ~block_header >>= function | None -> pop_until_ok () | Some head -> Lwt.return_some head @@ -295,6 +327,19 @@ let make_mocked_services_hooks (state : state) (user_hooks : (module Hooks)) : else Protocol.hash); } + let raw_header (block : Tezos_shell_services.Block_services.block) : + bytes tzresult Lwt.t = + locate_block state block >>=? fun x -> + let protocol_data = + Data_encoding.Binary.to_bytes_exn + Protocol.block_header_data_encoding + x.protocol_data + in + return + (Data_encoding.Binary.to_bytes_exn + Tezos_base.Block_header.encoding + {shell = x.rpc_context.block_header; protocol_data}) + let header (block : Tezos_shell_services.Block_services.block) : Mockup.M.Block_services.block_header tzresult Lwt.t = locate_block state block >>=? fun x -> @@ -716,6 +761,9 @@ let rec listener ~(user_hooks : (module Hooks)) ~state ~broadcast_pipe = process_block state block_hash block_header operations >>=? fun () -> User_hooks.check_chain_after_processing ~level ~round ~chain:state.chain >>=? fun () -> + Lwt_pipe.Unbounded.push + state.validated_blocks_pipe + (block_hash, block_header, operations) ; Lwt_pipe.Unbounded.push state.heads_pipe (block_hash, block_header) ; listener ~user_hooks ~state ~broadcast_pipe @@ -735,6 +783,7 @@ let create_fake_node_state ~i ~live_depth } in let chain0 = [genesis0] in + let validated_blocks_pipe = Lwt_pipe.Unbounded.create () in let heads_pipe = Lwt_pipe.Unbounded.create () in let operations_pipe = Lwt_pipe.Unbounded.create () in let genesis_block_true_hash = @@ -744,6 +793,8 @@ let create_fake_node_state ~i ~live_depth protocol_data = block_header0.protocol_data; } in + (* Only push genesis block as a new head, not a valid block: it is + the shell's semantics to not advertise "transition" blocks. *) Lwt_pipe.Unbounded.push heads_pipe (rpc_context0.block_hash, block_header0) ; return { @@ -768,6 +819,7 @@ let create_fake_node_state ~i ~live_depth .Block_header.context, rpc_context0 ); ]); + validated_blocks_pipe; heads_pipe; operations_pipe; streaming_operations = false; @@ -1026,7 +1078,6 @@ let make_genesis_context ~delegate_selection ~initial_seed ~round0 ~round1 in return (block_header, rpc_context) in - let level0_round0_duration = Protocol.Alpha_context.Round.round_duration round_durations @@ -1052,6 +1103,9 @@ module Default_hooks : Hooks = struct let on_inject_operation ~op_hash ~op = return (op_hash, op, default_propagation_vector) + let on_new_validated_block ~block_hash ~block_header ~operations = + Lwt.return (Some (block_hash, block_header, operations)) + let on_new_head ~block_hash ~block_header = Lwt.return (Some (block_hash, block_header)) diff --git a/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.mli b/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.mli index c01782653bdf..d9da9a19074f 100644 --- a/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.mli +++ b/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.mli @@ -74,6 +74,15 @@ module type Hooks = sig tzresult Lwt.t + (** This is called when a new validated block is going to be sent as + the response to a "monitor validated blocks" RPC call. Returning + [None] here terminates the process for the baker. *) + val on_new_validated_block : + block_hash:Block_hash.t -> + block_header:Block_header.t -> + operations:Operation.t list list -> + (Block_hash.t * Block_header.t * Operation.t list list) option Lwt.t + (** This is called when a new head is going to be sent as the response to a "monitor heads" RPC call. Returning [None] here terminates the process for the baker. *) -- GitLab From af1d1bbdf3b7a9cc49a25dbeccd24344f90ddecd Mon Sep 17 00:00:00 2001 From: vbot Date: Tue, 31 Jan 2023 17:59:33 +0100 Subject: [PATCH 35/46] Alpha/Mockup: lie on the genesis' proto level in mockup simulator --- src/proto_alpha/lib_client/mockup.ml | 11 ++++++- .../test/mockup_simulator/mockup_simulator.ml | 32 +++++++++++++++++-- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/proto_alpha/lib_client/mockup.ml b/src/proto_alpha/lib_client/mockup.ml index 62ed51f59076..31f3685e652c 100644 --- a/src/proto_alpha/lib_client/mockup.ml +++ b/src/proto_alpha/lib_client/mockup.ml @@ -250,6 +250,15 @@ module Forge = struct Bytes.create Constants.proof_of_work_nonce_size let make_shell ~level ~predecessor ~timestamp ~fitness ~operations_hash = + (* We initialize the [proto_level] at 1 in order to be able to + mimick a transition block in the baker. The baker distinguishes + the first block of a protocol by comparing a block and its + predecessor's proto level. If there is a difference, it must + mean that the block is a transition one. If we start at 0, we + cannot "hack" a transition block by decrementing the genesis + predecessor's protocol level because protocol levels are + encoded as uint8. *) + let proto_level = 1 in Tezos_base.Block_header. { level; @@ -257,7 +266,7 @@ module Forge = struct timestamp; fitness; operations_hash; - proto_level = 0; + proto_level; validation_passes = 0; context = Context_hash.zero; } diff --git a/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.ml b/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.ml index 2c4d8230c19f..82ae1d67c295 100644 --- a/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.ml +++ b/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.ml @@ -327,9 +327,36 @@ let make_mocked_services_hooks (state : state) (user_hooks : (module Hooks)) : else Protocol.hash); } + let may_lie_on_proto_level block x = + (* As for ../protocols, the baker distinguishes activation + blocks from "normal" blocks by comparing the [proto_level] of + the shell header and its predecessor. If the predecessor's + one is different, it must mean that we are considering an + activation block and must not endorse. Here, we do a bit of + hacking in order to return a different proto_level for the + predecessor of the genesis block which is considered as the + current protocol activation block. To perfectly mimic what is + supposed to happen, the first mocked up block created should + be made in the genesis protocol, however, it is not what's + done in the mockup mode. *) + let is_predecessor_of_genesis = + match block with + | `Hash (requested_hash, rel) -> + Int.equal rel 0 + && Block_hash.equal requested_hash genesis_predecessor_block_hash + | _ -> false + in + if is_predecessor_of_genesis then + { + x.rpc_context.block_header with + proto_level = pred x.rpc_context.block_header.proto_level; + } + else x.rpc_context.block_header + let raw_header (block : Tezos_shell_services.Block_services.block) : bytes tzresult Lwt.t = locate_block state block >>=? fun x -> + let shell = may_lie_on_proto_level block x in let protocol_data = Data_encoding.Binary.to_bytes_exn Protocol.block_header_data_encoding @@ -338,16 +365,17 @@ let make_mocked_services_hooks (state : state) (user_hooks : (module Hooks)) : return (Data_encoding.Binary.to_bytes_exn Tezos_base.Block_header.encoding - {shell = x.rpc_context.block_header; protocol_data}) + {shell; protocol_data}) let header (block : Tezos_shell_services.Block_services.block) : Mockup.M.Block_services.block_header tzresult Lwt.t = locate_block state block >>=? fun x -> + let shell = may_lie_on_proto_level block x in return { Mockup.M.Block_services.hash = x.rpc_context.block_hash; chain_id; - shell = x.rpc_context.block_header; + shell; protocol_data = x.protocol_data; } -- GitLab From b7d6c71f04b200d7a6c71c4dea1eab50748eef04 Mon Sep 17 00:00:00 2001 From: vbot Date: Tue, 31 Jan 2023 18:28:22 +0100 Subject: [PATCH 36/46] Alpha/Baker: add preendorsement on valid block test --- .../lib_delegate/test/test_scenario.ml | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/src/proto_alpha/lib_delegate/test/test_scenario.ml b/src/proto_alpha/lib_delegate/test/test_scenario.ml index afc642bf9457..693b84bdda01 100644 --- a/src/proto_alpha/lib_delegate/test/test_scenario.ml +++ b/src/proto_alpha/lib_delegate/test/test_scenario.ml @@ -54,6 +54,65 @@ let test_level_5 () = in run ~config [(3, (module Hooks)); (2, (module Hooks))] +let test_preendorse_on_valid () = + let level_to_reach = 2l in + let round_to_reach = 1l in + let module Hooks : Hooks = struct + include Default_hooks + + let on_new_head ~block_hash ~block_header = + (* Stop notifying heads on the level to reach, only notify that + it has been validated *) + if block_header.Block_header.shell.level < level_to_reach then + Lwt.return_some (block_hash, block_header) + else Lwt.return_none + + let seen_candidate = ref None + + let pqc_noticed = ref false + + let stop_on_event = function + | Baking_state.Prequorum_reached (candidate, _) -> + (* Register the PQC notice. *) + (match !seen_candidate with + | Some seen_candidate + when Block_hash.(candidate.hash = seen_candidate) -> + pqc_noticed := true + | _ -> ()) ; + false + | Baking_state.Quorum_reached (candidate, _) -> + (* Ensure that we never see a QC on the seen candidate. *) + (match !seen_candidate with + | Some seen_candidate + when Block_hash.(candidate.hash = seen_candidate) -> + Stdlib.failwith "Quorum occured on the seen candidate" + | _ -> ()) ; + false + | New_head_proposal {block; _} -> + (* Ensure that we never notice a new head at the level where + we are not supposed to. *) + if block.shell.level = level_to_reach then + Stdlib.failwith "Unexpected new head event" + else false + | New_valid_proposal {block; _} -> + (* Register the seen valid proposal candidate. *) + if + block.shell.level = level_to_reach + && Protocol.Alpha_context.Round.to_int32 block.round = 0l + then seen_candidate := Some block.hash ; + (* Stop the node when we reach level 2 / round 2. *) + block.shell.level = level_to_reach + && Protocol.Alpha_context.Round.to_int32 block.round >= round_to_reach + | _ -> false + + let check_chain_on_success ~chain:_ = + assert (!seen_candidate <> None) ; + assert !pqc_noticed ; + return_unit + end in + let config = {default_config with timeout = 10} in + run ~config [(1, (module Hooks))] + (* Scenario T1 @@ -1470,6 +1529,7 @@ let tests = let open Tezos_base_test_helpers.Tztest in [ tztest "reaches level 5" `Quick test_level_5; + tztest "cannot progress without new head" `Quick test_preendorse_on_valid; tztest "scenario t1" `Quick test_scenario_t1; tztest "scenario t2" `Quick test_scenario_t2; tztest "scenario t3" `Quick test_scenario_t3; -- GitLab From e114fb25b01759992a04fc673ff699023b397181 Mon Sep 17 00:00:00 2001 From: vbot Date: Thu, 2 Feb 2023 17:10:36 +0100 Subject: [PATCH 37/46] Alpha/Baker: refactor operations injection --- .../lib_delegate/baking_actions.ml | 29 +++++++------------ src/proto_alpha/lib_delegate/baking_events.ml | 11 +++++++ src/proto_alpha/lib_delegate/node_rpc.ml | 6 ++++ src/proto_alpha/lib_delegate/node_rpc.mli | 10 +++++++ 4 files changed, 38 insertions(+), 18 deletions(-) diff --git a/src/proto_alpha/lib_delegate/baking_actions.ml b/src/proto_alpha/lib_delegate/baking_actions.ml index ffc740c5f5bd..a6265ed0041a 100644 --- a/src/proto_alpha/lib_delegate/baking_actions.ml +++ b/src/proto_alpha/lib_delegate/baking_actions.ml @@ -430,18 +430,12 @@ let inject_preendorsements state ~preendorsements = (* TODO: add a RPC to inject multiple operations *) List.iter_ep (fun (delegate, operation) -> - let encoded_op = - Data_encoding.Binary.to_bytes_exn Operation.encoding operation - in protect ~on_error:(fun err -> Events.(emit failed_to_inject_preendorsement (delegate, err)) >>= fun () -> return_unit) (fun () -> - Shell_services.Injection.operation - cctxt - ~chain:(`Hash chain_id) - encoded_op + Node_rpc.inject_operation cctxt ~chain:(`Hash chain_id) operation >>=? fun oph -> Events.(emit preendorsement_injected (oph, delegate)) >>= fun () -> return_unit)) @@ -518,17 +512,16 @@ let inject_endorsements state ~endorsements = sign_endorsements state endorsements >>=? fun signed_operations -> (* TODO: add a RPC to inject multiple operations *) List.iter_ep - (fun (delegate, signed_operation) -> - let encoded_op = - Data_encoding.Binary.to_bytes_exn Operation.encoding signed_operation - in - Shell_services.Injection.operation - cctxt - ~chain:(`Hash chain_id) - encoded_op - >>=? fun oph -> - Events.(emit endorsement_injected (oph, delegate)) >>= fun () -> - return_unit) + (fun (delegate, operation) -> + protect + ~on_error:(fun err -> + Events.(emit failed_to_inject_endorsement (delegate, err)) + >>= fun () -> return_unit) + (fun () -> + Node_rpc.inject_operation cctxt ~chain:(`Hash chain_id) operation + >>=? fun oph -> + Events.(emit endorsement_injected (oph, delegate)) >>= fun () -> + return_unit)) signed_operations let prepare_waiting_for_quorum state = diff --git a/src/proto_alpha/lib_delegate/baking_events.ml b/src/proto_alpha/lib_delegate/baking_events.ml index f4383307a31e..9c639f399db1 100644 --- a/src/proto_alpha/lib_delegate/baking_events.ml +++ b/src/proto_alpha/lib_delegate/baking_events.ml @@ -602,6 +602,17 @@ module Actions = struct ~pp2:Error_monad.pp_print_trace ("trace", Error_monad.trace_encoding) + let failed_to_inject_endorsement = + declare_2 + ~section + ~name:"failed_to_inject_endorsement" + ~level:Error + ~msg:"failed to inject endorsement for {delegate} -- {trace}" + ~pp1:Baking_state.pp_consensus_key_and_delegate + ("delegate", Baking_state.consensus_key_and_delegate_encoding) + ~pp2:Error_monad.pp_print_trace + ("trace", Error_monad.trace_encoding) + let potential_double_baking = declare_2 ~section diff --git a/src/proto_alpha/lib_delegate/node_rpc.ml b/src/proto_alpha/lib_delegate/node_rpc.ml index b82cf6b8964a..8375db130c55 100644 --- a/src/proto_alpha/lib_delegate/node_rpc.ml +++ b/src/proto_alpha/lib_delegate/node_rpc.ml @@ -41,6 +41,12 @@ let inject_block cctxt ?(force = false) ~chain signed_block_header operations = signed_shell_header_bytes operations +let inject_operation cctxt ~chain operation = + let encoded_op = + Data_encoding.Binary.to_bytes_exn Operation.encoding operation + in + Shell_services.Injection.operation cctxt ~chain encoded_op + let preapply_block cctxt ~chain ~head ~timestamp ~protocol_data operations = Block_services.Helpers.Preapply.block cctxt diff --git a/src/proto_alpha/lib_delegate/node_rpc.mli b/src/proto_alpha/lib_delegate/node_rpc.mli index 6cd40df1760b..00977029991a 100644 --- a/src/proto_alpha/lib_delegate/node_rpc.mli +++ b/src/proto_alpha/lib_delegate/node_rpc.mli @@ -39,6 +39,16 @@ val inject_block : Tezos_base.Operation.t list list -> Block_hash.t tzresult Lwt.t +(** Inject an operation. + + @return operation hash of the newly injected operation +*) +val inject_operation : + #Protocol_client_context.full -> + chain:Shell_services.chain -> + packed_operation -> + Operation_hash.t tzresult Lwt.t + (** Preapply a block using the node validation mechanism.*) val preapply_block : #Protocol_client_context.full -> -- GitLab From 224b205eaf6dca33081c0bde402061bdda1e1575 Mon Sep 17 00:00:00 2001 From: vbot Date: Fri, 3 Feb 2023 10:21:41 +0100 Subject: [PATCH 38/46] Alpha/Baker: make operation and block injection async and forcing --- src/proto_alpha/lib_delegate/node_rpc.ml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/proto_alpha/lib_delegate/node_rpc.ml b/src/proto_alpha/lib_delegate/node_rpc.ml index 8375db130c55..754e96117c11 100644 --- a/src/proto_alpha/lib_delegate/node_rpc.ml +++ b/src/proto_alpha/lib_delegate/node_rpc.ml @@ -35,6 +35,7 @@ let inject_block cctxt ?(force = false) ~chain signed_block_header operations = Data_encoding.Binary.to_bytes_exn Block_header.encoding signed_block_header in Shell_services.Injection.block + ~async:true cctxt ~chain ~force @@ -45,7 +46,10 @@ let inject_operation cctxt ~chain operation = let encoded_op = Data_encoding.Binary.to_bytes_exn Operation.encoding operation in - Shell_services.Injection.operation cctxt ~chain encoded_op + (* FIXME: https://gitlab.com/tezos/tezos/-/issues/4875 + `Shell_services.Injection.operation` should be used instead once + the needed changes in the protocol are in place. *) + Shell_services.Injection.private_operation cctxt ~async:true ~chain encoded_op let preapply_block cctxt ~chain ~head ~timestamp ~protocol_data operations = Block_services.Helpers.Preapply.block -- GitLab From 284b49f06e7acd9d62977a19e312ce7c316f8422 Mon Sep 17 00:00:00 2001 From: vbot Date: Fri, 3 Feb 2023 11:55:02 +0100 Subject: [PATCH 39/46] Alpha/Baker: fix a concurrency bug canceling the PQC watch --- .../lib_delegate/operation_worker.ml | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/proto_alpha/lib_delegate/operation_worker.ml b/src/proto_alpha/lib_delegate/operation_worker.ml index 9f54d826da64..1568bd5bb901 100644 --- a/src/proto_alpha/lib_delegate/operation_worker.ml +++ b/src/proto_alpha/lib_delegate/operation_worker.ml @@ -247,6 +247,21 @@ let is_valid_consensus_content (candidate : candidate) consensus_content = let cancel_monitoring state = state.proposal_watched <- None +let reset_monitoring state = + Lwt_mutex.with_lock state.lock @@ fun () -> + match state.proposal_watched with + | None -> Lwt.return_unit + | Some (Pqc_watch pqc_watched) -> + pqc_watched.current_voting_power <- 0 ; + pqc_watched.preendorsements_count <- 0 ; + pqc_watched.preendorsements_received <- [] ; + Lwt.return_unit + | Some (Qc_watch qc_watched) -> + qc_watched.current_voting_power <- 0 ; + qc_watched.endorsements_count <- 0 ; + qc_watched.endorsements_received <- [] ; + Lwt.return_unit + let update_monitoring ?(should_lock = true) state ops = (if should_lock then Lwt_mutex.with_lock state.lock else fun f -> f ()) @@ fun () -> @@ -484,11 +499,10 @@ let create ?(monitor_node_operations = true) Lwt_stream.get operation_stream >>= function | None -> (* When the stream closes, it means a new head has been set, - we cancel the monitoring and flush current operations *) + we reset the monitoring and flush current operations *) Events.(emit end_of_stream ()) >>= fun () -> op_stream_stopper () ; - cancel_monitoring state ; - worker_loop () + reset_monitoring state >>= fun () -> worker_loop () | Some ops -> state.operation_pool <- Operation_pool.add_operations state.operation_pool ops ; -- GitLab From dd64f31edbda66cae18b648592cf4fd2c141431a Mon Sep 17 00:00:00 2001 From: vbot Date: Tue, 14 Feb 2023 14:50:41 +0100 Subject: [PATCH 40/46] Alpha/Baker: register preendorsements and reinject after application --- .../lib_delegate/baking_actions.ml | 28 +++++++++++++++++-- .../lib_delegate/baking_actions.mli | 3 +- src/proto_alpha/lib_delegate/baking_lib.ml | 1 + .../lib_delegate/baking_scheduling.ml | 1 + src/proto_alpha/lib_delegate/baking_state.ml | 10 +++++-- src/proto_alpha/lib_delegate/baking_state.mli | 1 + .../lib_delegate/state_transitions.ml | 21 ++++++++++++-- .../lib_delegate/state_transitions.mli | 2 -- 8 files changed, 56 insertions(+), 11 deletions(-) diff --git a/src/proto_alpha/lib_delegate/baking_actions.ml b/src/proto_alpha/lib_delegate/baking_actions.ml index a6265ed0041a..7eb0bc6dd90d 100644 --- a/src/proto_alpha/lib_delegate/baking_actions.ml +++ b/src/proto_alpha/lib_delegate/baking_actions.ml @@ -128,6 +128,7 @@ type action = | Inject_preendorsements of { preendorsements : (consensus_key_and_delegate * consensus_content) list; } + | Reinject_preendorsements of {preendorsements : packed_operation list} | Inject_endorsements of { endorsements : (consensus_key_and_delegate * consensus_content) list; } @@ -159,6 +160,7 @@ let pp_action fmt = function | Update_to_level _ -> Format.fprintf fmt "update to level" | Synchronize_round _ -> Format.fprintf fmt "synchronize round" | Watch_proposal -> Format.fprintf fmt "watch proposal" + | Reinject_preendorsements _ -> Format.fprintf fmt "reinject preendorsements" let generate_seed_nonce_hash config delegate level = if level.Level.expected_commitment then @@ -440,6 +442,16 @@ let inject_preendorsements state ~preendorsements = Events.(emit preendorsement_injected (oph, delegate)) >>= fun () -> return_unit)) signed_operations + >>=? fun () -> + (* Hackish way of registering injected preendorsements *) + let endorsements = List.map snd signed_operations in + if endorsements = [] then return state + else + let new_level_state = + {state.level_state with injected_preendorsements = Some endorsements} + in + let new_state = {state with level_state = new_level_state} in + return new_state let sign_endorsements state endorsements = let cctxt = state.global_state.cctxt in @@ -609,6 +621,16 @@ let update_to_level state level_update = compute_new_state ~current_round ~delegate_slots ~next_level_delegate_slots >>= return +let reinject_preendorsements state preendorsements = + let cctxt = state.global_state.cctxt in + let chain = `Hash state.global_state.chain_id in + List.iter_p + (fun operation -> + Node_rpc.inject_operation cctxt ~chain operation >>= fun _res -> + (* Ignore errors for now *) + Lwt.return_unit) + preendorsements + let synchronize_round state {new_round_proposal; handle_proposal} = Events.(emit synchronizing_round new_round_proposal.predecessor.hash) >>= fun () -> @@ -641,8 +663,8 @@ let rec perform_action ~state_recorder state (action : action) = | Inject_block {block_to_bake; updated_state} -> inject_block state ~state_recorder block_to_bake ~updated_state | Inject_preendorsements {preendorsements} -> - inject_preendorsements state ~preendorsements >>=? fun () -> - perform_action ~state_recorder state Watch_proposal + inject_preendorsements state ~preendorsements >>=? fun new_state -> + perform_action ~state_recorder new_state Watch_proposal | Inject_endorsements {endorsements} -> state_recorder ~new_state:state >>=? fun () -> inject_endorsements state ~endorsements >>=? fun () -> @@ -659,3 +681,5 @@ let rec perform_action ~state_recorder state (action : action) = (* We wait for preendorsements to trigger the [Prequorum_reached] event *) start_waiting_for_preendorsement_quorum state >>= fun () -> return state + | Reinject_preendorsements {preendorsements} -> + reinject_preendorsements state preendorsements >>= fun () -> return state diff --git a/src/proto_alpha/lib_delegate/baking_actions.mli b/src/proto_alpha/lib_delegate/baking_actions.mli index 5b9715905497..d9598b3b9796 100644 --- a/src/proto_alpha/lib_delegate/baking_actions.mli +++ b/src/proto_alpha/lib_delegate/baking_actions.mli @@ -54,6 +54,7 @@ type action = | Inject_preendorsements of { preendorsements : (consensus_key_and_delegate * consensus_content) list; } + | Reinject_preendorsements of {preendorsements : packed_operation list} | Inject_endorsements of { endorsements : (consensus_key_and_delegate * consensus_content) list; } @@ -93,7 +94,7 @@ val inject_block : val inject_preendorsements : state -> preendorsements:(consensus_key_and_delegate * consensus_content) list -> - unit tzresult Lwt.t + state tzresult Lwt.t val sign_endorsements : state -> diff --git a/src/proto_alpha/lib_delegate/baking_lib.ml b/src/proto_alpha/lib_delegate/baking_lib.ml index ae712266db5e..383e0f867739 100644 --- a/src/proto_alpha/lib_delegate/baking_lib.ml +++ b/src/proto_alpha/lib_delegate/baking_lib.ml @@ -88,6 +88,7 @@ let preendorse (cctxt : Protocol_client_context.full) ?(force = false) delegates (List.map fst consensus_list) in Baking_actions.inject_preendorsements state ~preendorsements:consensus_list + >>=? fun (_ignored_state : state) -> return_unit let endorse (cctxt : Protocol_client_context.full) ?(force = false) delegates = let open State_transitions in diff --git a/src/proto_alpha/lib_delegate/baking_scheduling.ml b/src/proto_alpha/lib_delegate/baking_scheduling.ml index e974c5b0b0d0..3389440a8469 100644 --- a/src/proto_alpha/lib_delegate/baking_scheduling.ml +++ b/src/proto_alpha/lib_delegate/baking_scheduling.ml @@ -679,6 +679,7 @@ let create_initial_state cctxt ?(synchronize = true) ~chain config is_latest_proposal_applied = true (* this proposal is expected to be the current head *); delayed_prequorum = None; + injected_preendorsements = None; locked_round = None; endorsable_payload = None; elected_block; diff --git a/src/proto_alpha/lib_delegate/baking_state.ml b/src/proto_alpha/lib_delegate/baking_state.ml index ccd2a6951bff..de069f210571 100644 --- a/src/proto_alpha/lib_delegate/baking_state.ml +++ b/src/proto_alpha/lib_delegate/baking_state.ml @@ -282,6 +282,7 @@ type level_state = { is_latest_proposal_applied : bool; delayed_prequorum : (Operation_worker.candidate * Kind.preendorsement operation list) option; + injected_preendorsements : packed_operation list option; (* Last proposal received where we injected an endorsement (thus we have seen 2f+1 preendorsements) *) locked_round : locked_round option; @@ -804,6 +805,7 @@ let pp_level_state fmt latest_proposal; is_latest_proposal_applied; delayed_prequorum; + injected_preendorsements; locked_round; endorsable_payload; elected_block; @@ -814,12 +816,14 @@ let pp_level_state fmt Format.fprintf fmt "@[Level state:@ current level: %ld@ @[proposal (applied:%b, \ - delayed prequorum:%b):@ %a@]@ locked round: %a@ endorsable payload: %a@ \ - elected block: %a@ @[own delegate slots:@ %a@]@ @[next level \ - own delegate slots:@ %a@]@ next level proposed round: %a@]" + delayed prequorum:%b, injected preendorsements: %d):@ %a@]@ locked round: \ + %a@ endorsable payload: %a@ elected block: %a@ @[own delegate \ + slots:@ %a@]@ @[next level own delegate slots:@ %a@]@ next level \ + proposed round: %a@]" current_level is_latest_proposal_applied (Option.is_some delayed_prequorum) + (match injected_preendorsements with None -> 0 | Some l -> List.length l) pp_proposal latest_proposal (pp_option pp_locked_round) diff --git a/src/proto_alpha/lib_delegate/baking_state.mli b/src/proto_alpha/lib_delegate/baking_state.mli index 02243593d8ac..cfa439d81738 100644 --- a/src/proto_alpha/lib_delegate/baking_state.mli +++ b/src/proto_alpha/lib_delegate/baking_state.mli @@ -130,6 +130,7 @@ type level_state = { is_latest_proposal_applied : bool; delayed_prequorum : (Operation_worker.candidate * Kind.preendorsement operation list) option; + injected_preendorsements : packed_operation list option; locked_round : locked_round option; endorsable_payload : endorsable_payload option; elected_block : elected_block option; diff --git a/src/proto_alpha/lib_delegate/state_transitions.ml b/src/proto_alpha/lib_delegate/state_transitions.ml index 79640027e695..ab8f6d35d7ba 100644 --- a/src/proto_alpha/lib_delegate/state_transitions.ml +++ b/src/proto_alpha/lib_delegate/state_transitions.ml @@ -319,6 +319,7 @@ let rec handle_proposal ~is_proposal_applied state (new_proposal : proposal) = latest_proposal = new_proposal; is_latest_proposal_applied = is_proposal_applied; delayed_prequorum = None; + injected_preendorsements = None; (* Unlock values *) locked_round = None; endorsable_payload = None; @@ -756,10 +757,24 @@ let handle_expected_applied_proposal (state : Baking_state.t) = in let new_state = {state with level_state = new_level_state} in match new_state.level_state.delayed_prequorum with - | None -> - (* The application arrived before the prequorum: just wait for the prequorum. *) + | None -> ( + (* The application arrived before the prequorum: wait for the prequorum. *) let new_state = update_current_phase new_state Awaiting_preendorsements in - do_nothing new_state + (* FIXME: https://gitlab.com/tezos/tezos/-/issues/4877 This + mechanism is only temporary and should be removed when the + protocol and prevalidator correctly accept early + preendorsements. *) + match new_state.level_state.injected_preendorsements with + | None -> do_nothing new_state + | Some preendorsements -> + let reinject_preendorsement_action = + Reinject_preendorsements {preendorsements} + in + let new_level_state = + {new_state.level_state with injected_preendorsements = None} + in + let new_state = {new_state with level_state = new_level_state} in + Lwt.return (new_state, reinject_preendorsement_action)) | Some (candidate, preendorsement_qc) -> (* The application arrived after the prequorum: handle the prequorum received earlier. *) diff --git a/src/proto_alpha/lib_delegate/state_transitions.mli b/src/proto_alpha/lib_delegate/state_transitions.mli index 596efad2100f..2fa02e3b2d8a 100644 --- a/src/proto_alpha/lib_delegate/state_transitions.mli +++ b/src/proto_alpha/lib_delegate/state_transitions.mli @@ -41,8 +41,6 @@ val is_acceptable_proposal_for_current_level : val make_consensus_list : state -> proposal -> (consensus_key_and_delegate * consensus_content) list -val make_preendorse_action : state -> proposal -> action - val may_update_proposal : is_proposal_applied:bool -> state -> proposal -> state Lwt.t -- GitLab From d89bc1e2d1a287a153d6ea92eb84a955dcb355f3 Mon Sep 17 00:00:00 2001 From: vbot Date: Thu, 26 Jan 2023 17:03:30 +0100 Subject: [PATCH 41/46] Mumbai/Baker: propagate changes --- src/proto_016_PtMumbai/lib_client/mockup.ml | 2 +- .../lib_delegate/baking_actions.ml | 72 ++-- .../lib_delegate/baking_actions.mli | 3 +- .../lib_delegate/baking_events.ml | 105 +++++- .../lib_delegate/baking_lib.ml | 30 +- .../lib_delegate/baking_nonces.ml | 2 +- .../lib_delegate/baking_scheduling.ml | 146 +++++--- .../lib_delegate/baking_scheduling.mli | 5 +- .../lib_delegate/baking_simulator.ml | 25 +- .../lib_delegate/baking_simulator.mli | 1 + .../lib_delegate/baking_state.ml | 193 +++++----- .../lib_delegate/baking_state.mli | 38 +- .../lib_delegate/block_forge.ml | 47 ++- .../lib_delegate/block_forge.mli | 2 + .../lib_delegate/node_rpc.ml | 331 +++++++++++------- .../lib_delegate/node_rpc.mli | 30 +- .../lib_delegate/operation_worker.ml | 38 +- .../lib_delegate/operation_worker.mli | 7 +- .../lib_delegate/state_transitions.ml | 237 ++++++++++--- .../lib_delegate/state_transitions.mli | 8 +- .../test/mockup_simulator/faked_services.ml | 61 +++- .../test/mockup_simulator/mockup_simulator.ml | 88 ++++- .../mockup_simulator/mockup_simulator.mli | 9 + .../lib_delegate/test/test_scenario.ml | 78 ++++- 24 files changed, 1089 insertions(+), 469 deletions(-) diff --git a/src/proto_016_PtMumbai/lib_client/mockup.ml b/src/proto_016_PtMumbai/lib_client/mockup.ml index 93e2c2690c0d..e82f935fb341 100644 --- a/src/proto_016_PtMumbai/lib_client/mockup.ml +++ b/src/proto_016_PtMumbai/lib_client/mockup.ml @@ -257,7 +257,7 @@ module Forge = struct timestamp; fitness; operations_hash; - proto_level = 0; + proto_level = 1; validation_passes = 0; context = Context_hash.zero; } diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_actions.ml b/src/proto_016_PtMumbai/lib_delegate/baking_actions.ml index 588190fb1fa5..6f911d234f28 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_actions.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_actions.ml @@ -128,6 +128,7 @@ type action = | Inject_preendorsements of { preendorsements : (consensus_key_and_delegate * consensus_content) list; } + | Reinject_preendorsements of {preendorsements : packed_operation list} | Inject_endorsements of { endorsements : (consensus_key_and_delegate * consensus_content) list; } @@ -159,6 +160,7 @@ let pp_action fmt = function | Update_to_level _ -> Format.fprintf fmt "update to level" | Synchronize_round _ -> Format.fprintf fmt "synchronize round" | Watch_proposal -> Format.fprintf fmt "watch proposal" + | Reinject_preendorsements _ -> Format.fprintf fmt "reinject preendorsements" let generate_seed_nonce_hash config delegate level = if level.Level.expected_commitment then @@ -215,6 +217,7 @@ let sign_block_header state proposer unsigned_block_header = return {Block_header.shell; protocol_data = {contents; signature}} let inject_block ~state_recorder state block_to_bake ~updated_state = + let open Lwt_result_syntax in let { predecessor; round; @@ -303,10 +306,24 @@ let inject_block ~state_recorder state block_to_bake ~updated_state = in Events.(emit vote_for_liquidity_baking_toggle) liquidity_baking_toggle_vote >>= fun () -> + let chain = `Hash state.global_state.chain_id in + let pred_block = `Hash (predecessor.hash, 0) in + let* pred_resulting_context_hash = + Shell_services.Blocks.resulting_context_hash + cctxt + ~chain + ~block:pred_block + () + in + let* pred_live_blocks = + Chain_services.Blocks.live_blocks cctxt ~chain ~block:pred_block () + in Block_forge.forge cctxt ~chain_id ~pred_info:predecessor + ~pred_live_blocks + ~pred_resulting_context_hash ~timestamp ~round ~seed_nonce_hash @@ -410,22 +427,26 @@ let inject_preendorsements state ~preendorsements = (* TODO: add a RPC to inject multiple operations *) List.iter_ep (fun (delegate, operation) -> - let encoded_op = - Data_encoding.Binary.to_bytes_exn Operation.encoding operation - in protect ~on_error:(fun err -> Events.(emit failed_to_inject_preendorsement (delegate, err)) >>= fun () -> return_unit) (fun () -> - Shell_services.Injection.operation - cctxt - ~chain:(`Hash chain_id) - encoded_op + Node_rpc.inject_operation cctxt ~chain:(`Hash chain_id) operation >>=? fun oph -> Events.(emit preendorsement_injected (oph, delegate)) >>= fun () -> return_unit)) signed_operations + >>=? fun () -> + (* Hackish way of registering injected preendorsements *) + let endorsements = List.map snd signed_operations in + if endorsements = [] then return state + else + let new_level_state = + {state.level_state with injected_preendorsements = Some endorsements} + in + let new_state = {state with level_state = new_level_state} in + return new_state let sign_endorsements state endorsements = let cctxt = state.global_state.cctxt in @@ -498,17 +519,16 @@ let inject_endorsements state ~endorsements = sign_endorsements state endorsements >>=? fun signed_operations -> (* TODO: add a RPC to inject multiple operations *) List.iter_ep - (fun (delegate, signed_operation) -> - let encoded_op = - Data_encoding.Binary.to_bytes_exn Operation.encoding signed_operation - in - Shell_services.Injection.operation - cctxt - ~chain:(`Hash chain_id) - encoded_op - >>=? fun oph -> - Events.(emit endorsement_injected (oph, delegate)) >>= fun () -> - return_unit) + (fun (delegate, operation) -> + protect + ~on_error:(fun err -> + Events.(emit failed_to_inject_endorsement (delegate, err)) + >>= fun () -> return_unit) + (fun () -> + Node_rpc.inject_operation cctxt ~chain:(`Hash chain_id) operation + >>=? fun oph -> + Events.(emit endorsement_injected (oph, delegate)) >>= fun () -> + return_unit)) signed_operations let prepare_waiting_for_quorum state = @@ -596,6 +616,16 @@ let update_to_level state level_update = compute_new_state ~current_round ~delegate_slots ~next_level_delegate_slots >>= return +let reinject_preendorsements state preendorsements = + let cctxt = state.global_state.cctxt in + let chain = `Hash state.global_state.chain_id in + List.iter_p + (fun operation -> + Node_rpc.inject_operation cctxt ~chain operation >>= fun _res -> + (* Ignore errors for now *) + Lwt.return_unit) + preendorsements + let synchronize_round state {new_round_proposal; handle_proposal} = Events.(emit synchronizing_round new_round_proposal.predecessor.hash) >>= fun () -> @@ -628,8 +658,8 @@ let rec perform_action ~state_recorder state (action : action) = | Inject_block {block_to_bake; updated_state} -> inject_block state ~state_recorder block_to_bake ~updated_state | Inject_preendorsements {preendorsements} -> - inject_preendorsements state ~preendorsements >>=? fun () -> - perform_action ~state_recorder state Watch_proposal + inject_preendorsements state ~preendorsements >>=? fun new_state -> + perform_action ~state_recorder new_state Watch_proposal | Inject_endorsements {endorsements} -> state_recorder ~new_state:state >>=? fun () -> inject_endorsements state ~endorsements >>=? fun () -> @@ -646,3 +676,5 @@ let rec perform_action ~state_recorder state (action : action) = (* We wait for preendorsements to trigger the [Prequorum_reached] event *) start_waiting_for_preendorsement_quorum state >>= fun () -> return state + | Reinject_preendorsements {preendorsements} -> + reinject_preendorsements state preendorsements >>= fun () -> return state diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_actions.mli b/src/proto_016_PtMumbai/lib_delegate/baking_actions.mli index 5b9715905497..d9598b3b9796 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_actions.mli +++ b/src/proto_016_PtMumbai/lib_delegate/baking_actions.mli @@ -54,6 +54,7 @@ type action = | Inject_preendorsements of { preendorsements : (consensus_key_and_delegate * consensus_content) list; } + | Reinject_preendorsements of {preendorsements : packed_operation list} | Inject_endorsements of { endorsements : (consensus_key_and_delegate * consensus_content) list; } @@ -93,7 +94,7 @@ val inject_block : val inject_preendorsements : state -> preendorsements:(consensus_key_and_delegate * consensus_content) list -> - unit tzresult Lwt.t + state tzresult Lwt.t val sign_endorsements : state -> diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_events.ml b/src/proto_016_PtMumbai/lib_delegate/baking_events.ml index 34cd55dec8ce..492985c2f89c 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_events.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_events.ml @@ -37,6 +37,19 @@ module State_transitions = struct let section = section @ ["transitions"] + let new_valid_proposal = + declare_3 + ~section + ~name:"new_valid_proposal" + ~level:Notice + ~msg:"received new proposal {block} at level {level}, round {round}" + ~pp1:Block_hash.pp + ("block", Block_hash.encoding) + ~pp2:pp_int32 + ("level", Data_encoding.int32) + ~pp3:Round.pp + ("round", Round.encoding) + let new_head = declare_3 ~section @@ -98,6 +111,63 @@ module State_transitions = struct ~msg:"received new head while waiting for a quorum" () + let applied_expected_proposal_received = + declare_1 + ~section + ~name:"applied_expected_proposal_received" + ~level:Info + ~msg:"received the expected application notice for {proposal}" + ~pp1:Block_hash.pp + ("proposal", Block_hash.encoding) + + let unexpected_new_head_while_waiting_for_application = + declare_0 + ~section + ~name:"unexpected_new_head_while_waiting_for_application" + ~level:Info + ~msg:"received new head while waiting for another proposal's application" + () + + let new_valid_proposal_while_waiting_for_qc = + declare_0 + ~section + ~name:"new_valid_proposal_while_waiting_for_qc" + ~level:Info + ~msg:"received new valid proposal while waiting for a quorum" + () + + let valid_proposal_received_after_application = + declare_0 + ~section + ~name:"valid_proposal_received_after_application" + ~level:Info + ~msg:"received valid proposal for a block already applied" + () + + let unexpected_pqc_while_waiting_for_application = + declare_2 + ~section + ~name:"unexpected_pqc_while_waiting_for_application" + ~level:Info + ~msg: + "received an unexpected prequorum for {prequorum} while waiting for \ + the proposal's {proposal} application" + ~pp1:Block_hash.pp + ("prequorum", Block_hash.encoding) + ~pp2:Block_hash.pp + ("proposal", Block_hash.encoding) + + let pqc_while_waiting_for_application = + declare_1 + ~section + ~name:"pqc_while_waiting_for_application" + ~level:Info + ~msg: + "received expected prequorum for {prequorum} while waiting for the \ + proposal's application" + ~pp1:Block_hash.pp + ("prequorum", Block_hash.encoding) + let unexpected_proposal_round = declare_2 ~section @@ -289,6 +359,14 @@ module State_transitions = struct ~pp2:Block_hash.pp ("expected_hash", Block_hash.encoding) + let handling_prequorum_on_non_applied_proposal = + declare_0 + ~section + ~name:"handling_prequorum_on_non_applied_proposal" + ~level:Error + ~msg:"Handling prequorum on a non-applied proposal" + () + let step_current_phase = declare_2 ~section @@ -315,16 +393,14 @@ module Node_rpc = struct ~pp1:Error_monad.pp_print_trace ("trace", Error_monad.trace_encoding) - let raw_info = - declare_2 + let error_while_monitoring_valid_proposals = + declare_1 ~section - ~name:"raw_info" - ~level:Debug - ~msg:"raw info for {block_hash} at level {level}" - ~pp1:Block_hash.pp - ("block_hash", Block_hash.encoding) - ~pp2:pp_int32 - ("level", Data_encoding.int32) + ~name:"error_while_monitoring_valid_proposals" + ~level:Error + ~msg:"error while monitoring valid proposals {trace}" + ~pp1:Error_monad.pp_print_trace + ("trace", Error_monad.trace_encoding) end module Scheduling = struct @@ -526,6 +602,17 @@ module Actions = struct ~pp2:Error_monad.pp_print_trace ("trace", Error_monad.trace_encoding) + let failed_to_inject_endorsement = + declare_2 + ~section + ~name:"failed_to_inject_endorsement" + ~level:Error + ~msg:"failed to inject endorsement for {delegate} -- {trace}" + ~pp1:Baking_state.pp_consensus_key_and_delegate + ("delegate", Baking_state.consensus_key_and_delegate_encoding) + ~pp2:Error_monad.pp_print_trace + ("trace", Error_monad.trace_encoding) + let potential_double_baking = declare_2 ~section diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_lib.ml b/src/proto_016_PtMumbai/lib_delegate/baking_lib.ml index ebb2cd35fc0c..383e0f867739 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_lib.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_lib.ml @@ -44,10 +44,10 @@ let create_state cctxt ?synchronize ?monitor_node_mempool ~config ~current_proposal delegates -let get_current_proposal cctxt = +let get_current_proposal cctxt ?cache () = let open Lwt_result_syntax in let* block_stream, _block_stream_stopper = - Node_rpc.monitor_proposals cctxt ~chain:cctxt#chain () + Node_rpc.monitor_heads cctxt ?cache ~chain:cctxt#chain () in Lwt_stream.peek block_stream >>= function | Some current_head -> return (block_stream, current_head) @@ -59,7 +59,8 @@ let preendorse (cctxt : Protocol_client_context.full) ?(force = false) delegates = let open State_transitions in let open Lwt_result_syntax in - let* _, current_proposal = get_current_proposal cctxt in + let cache = Baking_cache.Block_cache.create 10 in + let* _, current_proposal = get_current_proposal cctxt ~cache () in let config = Baking_configuration.make ~force () in let* state = create_state cctxt ~config ~current_proposal delegates in let proposal = state.level_state.latest_proposal in @@ -87,11 +88,13 @@ let preendorse (cctxt : Protocol_client_context.full) ?(force = false) delegates (List.map fst consensus_list) in Baking_actions.inject_preendorsements state ~preendorsements:consensus_list + >>=? fun (_ignored_state : state) -> return_unit let endorse (cctxt : Protocol_client_context.full) ?(force = false) delegates = let open State_transitions in let open Lwt_result_syntax in - let* _, current_proposal = get_current_proposal cctxt in + let cache = Baking_cache.Block_cache.create 10 in + let* _, current_proposal = get_current_proposal cctxt ~cache () in let config = Baking_configuration.make ~force () in create_state cctxt ~config ~current_proposal delegates >>=? fun state -> let proposal = state.level_state.latest_proposal in @@ -283,7 +286,8 @@ let propose (cctxt : Protocol_client_context.full) ?minimal_fees ?minimal_nanotez_per_gas_unit ?minimal_nanotez_per_byte ?force_apply ?force ?(minimal_timestamp = false) ?extra_operations ?context_path delegates = let open Lwt_result_syntax in - let* _block_stream, current_proposal = get_current_proposal cctxt in + let cache = Baking_cache.Block_cache.create 10 in + let* _block_stream, current_proposal = get_current_proposal cctxt ~cache () in let config = Baking_configuration.make ?minimal_fees @@ -301,7 +305,7 @@ let propose (cctxt : Protocol_client_context.full) ?minimal_fees | Some _ -> propose_at_next_level ~minimal_timestamp state | None -> ( match endorsement_quorum state with - | Some (voting_power, endorsement_qc) -> + | Some (_voting_power, endorsement_qc) -> let state = { state with @@ -323,8 +327,7 @@ let propose (cctxt : Protocol_client_context.full) ?minimal_fees let* state = State_transitions.step state - (Baking_state.Quorum_reached - (candidate, voting_power, endorsement_qc)) + (Baking_state.Quorum_reached (candidate, endorsement_qc)) >>= do_action (* this will register the elected block *) in @@ -369,18 +372,18 @@ let propose (cctxt : Protocol_client_context.full) ?minimal_fees in return_unit -let bake_using_automaton config state block_stream = +let bake_using_automaton config state heads_stream = let open Lwt_result_syntax in let cctxt = state.global_state.cctxt in let* initial_event = first_automaton_event state in let current_level = state.level_state.latest_proposal.block.shell.level in let loop_state = Baking_scheduling.create_loop_state - block_stream + ~heads_stream state.global_state.operation_worker in let stop_on_next_level_block = function - | New_proposal proposal -> + | New_head_proposal proposal -> Compare.Int32.(proposal.block.shell.level >= Int32.succ current_level) | _ -> false in @@ -392,7 +395,7 @@ let bake_using_automaton config state block_stream = state initial_event >>=? function - | Some (New_proposal proposal) -> + | Some (New_head_proposal proposal) -> let*! () = cctxt#message "Block %a (%ld) injected" @@ -508,7 +511,8 @@ let bake (cctxt : Protocol_client_context.full) ?minimal_fees ?extra_operations () in - let* block_stream, current_proposal = get_current_proposal cctxt in + let cache = Baking_cache.Block_cache.create 10 in + let* block_stream, current_proposal = get_current_proposal cctxt ~cache () in let* state = create_state cctxt diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_nonces.ml b/src/proto_016_PtMumbai/lib_delegate/baking_nonces.ml index fc1c9ada635a..c0c3d4d8a15d 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_nonces.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_nonces.ml @@ -255,7 +255,7 @@ let reveal_potential_nonces state new_proposal = let new_predecessor_hash = new_proposal.Baking_state.predecessor.hash in if Block_hash.(last_predecessor <> new_predecessor_hash) - && Protocol_hash.(new_proposal.predecessor.protocol = Protocol.hash) + && not (Baking_state.is_first_block_in_protocol new_proposal) then ( (* only try revealing nonces when the proposal's predecessor is a new one *) state.last_predecessor <- new_predecessor_hash ; diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_scheduling.ml b/src/proto_016_PtMumbai/lib_delegate/baking_scheduling.ml index 96350fe3ba3e..3389440a8469 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_scheduling.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_scheduling.ml @@ -28,33 +28,52 @@ module Events = Baking_events.Scheduling open Baking_state type loop_state = { - block_stream : Baking_state.proposal Lwt_stream.t; + heads_stream : Baking_state.proposal Lwt_stream.t; + get_valid_blocks_stream : Baking_state.proposal Lwt_stream.t Lwt.t; qc_stream : Operation_worker.event Lwt_stream.t; - future_block_stream : proposal Lwt_stream.t; - push_future_block : proposal -> unit; - mutable last_get_head_event : [`New_proposal of proposal option] Lwt.t option; + future_block_stream : + [`New_future_head of proposal | `New_future_valid_proposal of proposal] + Lwt_stream.t; + push_future_block : + [`New_future_head of proposal | `New_future_valid_proposal of proposal] -> + unit; + mutable last_get_head_event : + [`New_head_proposal of proposal option] Lwt.t option; + mutable last_get_valid_block_event : + [`New_valid_proposal of proposal option] Lwt.t option; mutable last_future_block_event : - [`New_future_block of Baking_state.proposal] Lwt.t option; + [`New_future_head of proposal | `New_future_valid_proposal of proposal] + Lwt.t + option; mutable last_get_qc_event : [`QC_reached of Operation_worker.event option] Lwt.t option; } type events = - [ `New_future_block of proposal - | `New_proposal of proposal option + [ `New_future_head of proposal + | `New_future_valid_proposal of proposal + | `New_valid_proposal of proposal option + | `New_head_proposal of proposal option | `QC_reached of Operation_worker.event option | `Termination | `Timeout of timeout_kind ] Lwt.t -let create_loop_state block_stream operation_worker = +let create_loop_state ?get_valid_blocks_stream ~heads_stream operation_worker = let future_block_stream, push_future_block = Lwt_stream.create () in + let get_valid_blocks_stream = + match get_valid_blocks_stream with + | None -> Lwt.return (Lwt_stream.create () |> fst) + | Some vbs_t -> vbs_t + in { - block_stream; + heads_stream; + get_valid_blocks_stream; qc_stream = Operation_worker.get_quorum_event_stream operation_worker; future_block_stream; push_future_block = (fun x -> push_future_block (Some x)); last_get_head_event = None; + last_get_valid_block_event = None; last_future_block_event = None; last_get_qc_event = None; } @@ -111,12 +130,24 @@ let rec wait_next_event ~timeout loop_state = match loop_state.last_get_head_event with | None -> let t = - Lwt_stream.get loop_state.block_stream >|= fun e -> `New_proposal e + Lwt_stream.get loop_state.heads_stream >|= fun e -> + `New_head_proposal e in loop_state.last_get_head_event <- Some t ; t | Some t -> t in + let get_valid_block_event () = + match loop_state.last_get_valid_block_event with + | None -> + let t = + loop_state.get_valid_blocks_stream >>= fun valid_blocks_stream -> + Lwt_stream.get valid_blocks_stream >|= fun e -> `New_valid_proposal e + in + loop_state.last_get_valid_block_event <- Some t ; + t + | Some t -> t + in let get_future_block_event () = (* n.b. we should also consume the available elements in the block_stream before starting baking. *) @@ -127,7 +158,7 @@ let rec wait_next_event ~timeout loop_state = | None -> (* unreachable, we never close the stream *) assert false - | Some proposal -> `New_future_block proposal + | Some future_proposal -> future_proposal in loop_state.last_future_block_event <- Some t ; t @@ -149,6 +180,7 @@ let rec wait_next_event ~timeout loop_state = [ terminated; (get_head_event () :> events); + (get_valid_block_event () :> events); (get_future_block_event () :> events); (get_qc_event () :> events); (timeout :> events); @@ -158,7 +190,11 @@ let rec wait_next_event ~timeout loop_state = | `Termination -> (* Exit the loop *) return_none - | `New_proposal None -> + | `New_valid_proposal None -> + (* Node connection lost *) + loop_state.last_get_valid_block_event <- None ; + fail Baking_errors.Node_connection_lost + | `New_head_proposal None -> (* Node connection lost *) loop_state.last_get_head_event <- None ; fail Baking_errors.Node_connection_lost @@ -166,7 +202,22 @@ let rec wait_next_event ~timeout loop_state = (* Not supposed to happen: exit the loop *) loop_state.last_get_qc_event <- None ; return_none - | `New_proposal (Some proposal) -> ( + | `New_valid_proposal (Some proposal) -> ( + loop_state.last_get_valid_block_event <- None ; + (* Is the block in the future? *) + match sleep_until proposal.block.shell.timestamp with + | Some waiter -> + (* If so, wait until its timestamp is reached before advertising it *) + Events.(emit proposal_in_the_future proposal.block.hash) >>= fun () -> + Lwt.dont_wait + (fun () -> + waiter >>= fun () -> + loop_state.push_future_block (`New_future_valid_proposal proposal) ; + Lwt.return_unit) + (fun _exn -> ()) ; + wait_next_event ~timeout loop_state + | None -> return_some (New_valid_proposal proposal)) + | `New_head_proposal (Some proposal) -> ( loop_state.last_get_head_event <- None ; (* Is the block in the future? *) match sleep_until proposal.block.shell.timestamp with @@ -176,29 +227,30 @@ let rec wait_next_event ~timeout loop_state = Lwt.dont_wait (fun () -> waiter >>= fun () -> - loop_state.push_future_block proposal ; + loop_state.push_future_block (`New_future_head proposal) ; Lwt.return_unit) (fun _exn -> ()) ; wait_next_event ~timeout loop_state - | None -> return_some (New_proposal proposal)) - | `New_future_block proposal -> + | None -> return_some (New_head_proposal proposal)) + | `New_future_head proposal -> + Events.(emit process_proposal_in_the_future proposal.block.hash) + >>= fun () -> + loop_state.last_future_block_event <- None ; + return_some (New_head_proposal proposal) + | `New_future_valid_proposal proposal -> Events.(emit process_proposal_in_the_future proposal.block.hash) >>= fun () -> loop_state.last_future_block_event <- None ; - return_some (New_proposal proposal) + return_some (New_valid_proposal proposal) | `QC_reached - (Some - (Operation_worker.Prequorum_reached - (candidate, voting_power, preendorsement_qc))) -> + (Some (Operation_worker.Prequorum_reached (candidate, preendorsement_qc))) + -> loop_state.last_get_qc_event <- None ; - return_some - (Prequorum_reached (candidate, voting_power, preendorsement_qc)) + return_some (Prequorum_reached (candidate, preendorsement_qc)) | `QC_reached - (Some - (Operation_worker.Quorum_reached - (candidate, voting_power, endorsement_qc))) -> + (Some (Operation_worker.Quorum_reached (candidate, endorsement_qc))) -> loop_state.last_get_qc_event <- None ; - return_some (Quorum_reached (candidate, voting_power, endorsement_qc)) + return_some (Quorum_reached (candidate, endorsement_qc)) | `Timeout e -> return_some (Timeout e) (** From the current [state], the function returns an optional @@ -211,8 +263,7 @@ let compute_next_round_time state = | None -> state.level_state.latest_proposal | Some {proposal; _} -> proposal in - if Protocol_hash.(proposal.predecessor.next_protocol <> Protocol.hash) then - None + if Baking_state.is_first_block_in_protocol proposal then None else match state.level_state.next_level_proposed_round with | Some _proposed_round -> @@ -615,11 +666,7 @@ let create_initial_state cctxt ?(synchronize = true) ~chain config ~chain >>=? fun next_level_delegate_slots -> let elected_block = - if - Protocol_hash.( - current_proposal.block.protocol <> Protocol.hash - && current_proposal.block.next_protocol = Protocol.hash) - then + if Baking_state.is_first_block_in_protocol current_proposal then (* If the last block is a protocol transition, we admit it as a final block *) Some {proposal = current_proposal; endorsement_qc = []} @@ -629,6 +676,10 @@ let create_initial_state cctxt ?(synchronize = true) ~chain config { current_level = current_proposal.block.shell.level; latest_proposal = current_proposal; + is_latest_proposal_applied = + true (* this proposal is expected to be the current head *); + delayed_prequorum = None; + injected_preendorsements = None; locked_round = None; endorsable_payload = None; elected_block; @@ -657,7 +708,7 @@ let compute_bootstrap_event state = = state.round_state.current_round) then (* If so, then trigger the new proposal event to possibly preendorse *) - ok @@ Baking_state.New_proposal state.level_state.latest_proposal + ok @@ Baking_state.New_head_proposal state.level_state.latest_proposal else (* Otherwise, trigger the end of round to check whether we need to propose at this level or not *) @@ -719,11 +770,13 @@ let perform_sanity_check cctxt ~chain_id = let run cctxt ?canceler ?(stop_on_event = fun _ -> false) ?(on_error = fun _ -> return_unit) ~chain config delegates = + let open Lwt_result_syntax in Shell_services.Chain.chain_id cctxt ~chain () >>=? fun chain_id -> perform_sanity_check cctxt ~chain_id >>=? fun () -> - Node_rpc.monitor_proposals cctxt ~chain () - >>=? fun (block_stream, _block_stream_stopper) -> - (Lwt_stream.get block_stream >>= function + let cache = Baking_cache.Block_cache.create 10 in + Node_rpc.monitor_heads cctxt ~cache ~chain () + >>=? fun (heads_stream, _block_stream_stopper) -> + (Lwt_stream.get heads_stream >>= function | Some current_head -> return current_head | None -> failwith "head stream unexpectedly ended") >>=? fun current_proposal -> @@ -742,7 +795,7 @@ let run cctxt ?canceler ?(stop_on_event = fun _ -> false) ~current_proposal delegates >>=? fun initial_state -> - let cloned_block_stream = Lwt_stream.clone block_stream in + let cloned_block_stream = Lwt_stream.clone heads_stream in Baking_nonces.start_revelation_worker cctxt initial_state.global_state.config.nonce @@ -756,9 +809,22 @@ let run cctxt ?canceler ?(stop_on_event = fun _ -> false) Lwt_canceler.cancel revelation_worker_canceler >>= fun _ -> Lwt.return_unit)) canceler ; - + (* FIXME: currently, the client streamed RPC call will hold until at + least one element is present in the stream. This is fixed by: + https://gitlab.com/nomadic-labs/resto/-/merge_requests/50. Until + then, we await the promise completion of the RPC call later + on. *) + let get_valid_blocks_stream = + let*! vbs = Node_rpc.monitor_valid_proposals cctxt ~cache ~chain () in + match vbs with + | Error _ -> Stdlib.failwith "Failed to get the validated blocks stream" + | Ok (vbs, _) -> Lwt.return vbs + in let loop_state = - create_loop_state block_stream initial_state.global_state.operation_worker + create_loop_state + ~get_valid_blocks_stream + ~heads_stream + initial_state.global_state.operation_worker in let on_error err = Events.(emit error_while_baking err) >>= fun () -> diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_scheduling.mli b/src/proto_016_PtMumbai/lib_delegate/baking_scheduling.mli index 83e167cfcf34..ed09245e450d 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_scheduling.mli +++ b/src/proto_016_PtMumbai/lib_delegate/baking_scheduling.mli @@ -29,7 +29,10 @@ open Protocol.Alpha_context type loop_state val create_loop_state : - proposal Lwt_stream.t -> Operation_worker.t -> loop_state + ?get_valid_blocks_stream:proposal Lwt_stream.t Lwt.t -> + heads_stream:proposal Lwt_stream.t -> + Operation_worker.t -> + loop_state val sleep_until : Time.Protocol.t -> unit Lwt.t option diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_simulator.ml b/src/proto_016_PtMumbai/lib_delegate/baking_simulator.ml index 5bb1fa77808f..45dcaf283770 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_simulator.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_simulator.ml @@ -77,17 +77,11 @@ let check_context_consistency (abstract_index : Abstract_context_index.t) | false -> fail Invalid_context)) let begin_construction ~timestamp ~protocol_data ~force_apply - (abstract_index : Abstract_context_index.t) predecessor chain_id = + ~pred_resulting_context_hash (abstract_index : Abstract_context_index.t) + pred_block chain_id = protect (fun () -> - let { - Baking_state.shell = pred_shell; - hash = pred_hash; - resulting_context_hash; - _; - } = - predecessor - in - abstract_index.checkout_fun resulting_context_hash >>= function + let {Baking_state.shell = pred_shell; hash = pred_hash; _} = pred_block in + abstract_index.checkout_fun pred_resulting_context_hash >>= function | None -> fail Failed_to_checkout_context | Some context -> let header : Tezos_base.Block_header.shell_header = @@ -107,7 +101,7 @@ let begin_construction ~timestamp ~protocol_data ~force_apply let mode = Lifted_protocol.Construction { - predecessor_hash = predecessor.hash; + predecessor_hash = pred_hash; timestamp; block_header_data = protocol_data; } @@ -130,7 +124,14 @@ let begin_construction ~timestamp ~protocol_data ~force_apply else return_none) >>=? fun application_state -> let state = (validation_state, application_state) in - return {predecessor; context; state; rev_operations = []; header}) + return + { + predecessor = pred_block; + context; + state; + rev_operations = []; + header; + }) let ( let** ) x k = let open Lwt_result_syntax in diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_simulator.mli b/src/proto_016_PtMumbai/lib_delegate/baking_simulator.mli index 959546fe6d21..c5155ac45ee0 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_simulator.mli +++ b/src/proto_016_PtMumbai/lib_delegate/baking_simulator.mli @@ -52,6 +52,7 @@ val begin_construction : timestamp:Time.Protocol.t -> protocol_data:block_header_data -> force_apply:bool -> + pred_resulting_context_hash:Context_hash.t -> Abstract_context_index.t -> Baking_state.block_info -> Chain_id.t -> diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_state.ml b/src/proto_016_PtMumbai/lib_delegate/baking_state.ml index 8d7b97748316..de069f210571 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_state.ml +++ b/src/proto_016_PtMumbai/lib_delegate/baking_state.ml @@ -103,16 +103,12 @@ type prequorum = { type block_info = { hash : Block_hash.t; shell : Block_header.shell_header; - resulting_context_hash : Context_hash.t; payload_hash : Block_payload_hash.t; payload_round : Round.t; round : Round.t; - protocol : Protocol_hash.t; - next_protocol : Protocol_hash.t; prequorum : prequorum option; quorum : Kind.endorsement operation list; payload : Operation_pool.payload; - live_blocks : Block_hash.Set.t; } type cache = { @@ -167,68 +163,48 @@ let block_info_encoding = (fun { hash; shell; - resulting_context_hash; payload_hash; payload_round; round; - protocol; - next_protocol; prequorum; quorum; payload; - live_blocks; } -> - ( ( hash, - shell, - resulting_context_hash, - payload_hash, - payload_round, - round, - protocol, - next_protocol, - prequorum, - List.map Operation.pack quorum ), - (payload, live_blocks) )) - (fun ( ( hash, - shell, - resulting_context_hash, - payload_hash, - payload_round, - round, - protocol, - next_protocol, - prequorum, - quorum ), - (payload, live_blocks) ) -> + ( hash, + shell, + payload_hash, + payload_round, + round, + prequorum, + List.map Operation.pack quorum, + payload )) + (fun ( hash, + shell, + payload_hash, + payload_round, + round, + prequorum, + quorum, + payload ) -> { hash; shell; - resulting_context_hash; payload_hash; payload_round; round; - protocol; - next_protocol; prequorum; quorum = List.filter_map Operation_pool.unpack_endorsement quorum; payload; - live_blocks; }) - (merge_objs - (obj10 - (req "hash" Block_hash.encoding) - (req "shell" Block_header.shell_header_encoding) - (req "resulting_context_hash" Context_hash.encoding) - (req "payload_hash" Block_payload_hash.encoding) - (req "payload_round" Round.encoding) - (req "round" Round.encoding) - (req "protocol" Protocol_hash.encoding) - (req "next_protocol" Protocol_hash.encoding) - (req "prequorum" (option prequorum_encoding)) - (req "quorum" (list (dynamic_size Operation.encoding)))) - (obj2 - (req "payload" Operation_pool.payload_encoding) - (req "live_blocks" Block_hash.Set.encoding))) + (obj8 + (req "hash" Block_hash.encoding) + (req "shell" Block_header.shell_header_encoding) + (req "payload_hash" Block_payload_hash.encoding) + (req "payload_round" Round.encoding) + (req "round" Round.encoding) + (req "prequorum" (option prequorum_encoding)) + (req "quorum" (list (dynamic_size Operation.encoding))) + (req "payload" Operation_pool.payload_encoding)) let round_of_shell_header shell_header = Environment.wrap_tzresult @@ -266,6 +242,9 @@ let proposal_encoding = (req "block" block_info_encoding) (req "predecessor" block_info_encoding)) +let is_first_block_in_protocol {block; predecessor; _} = + Compare.Int.(block.shell.proto_level <> predecessor.shell.proto_level) + type locked_round = {payload_hash : Block_payload_hash.t; round : Round.t} let locked_round_encoding = @@ -300,6 +279,10 @@ type elected_block = { type level_state = { current_level : int32; latest_proposal : proposal; + is_latest_proposal_applied : bool; + delayed_prequorum : + (Operation_worker.candidate * Kind.preendorsement operation list) option; + injected_preendorsements : packed_operation list option; (* Last proposal received where we injected an endorsement (thus we have seen 2f+1 preendorsements) *) locked_round : locked_round option; @@ -312,7 +295,11 @@ type level_state = { next_level_proposed_round : Round.t option; } -type phase = Idle | Awaiting_preendorsements | Awaiting_endorsements +type phase = + | Idle + | Awaiting_preendorsements + | Awaiting_application + | Awaiting_endorsements let phase_encoding = let open Data_encoding in @@ -332,9 +319,15 @@ let phase_encoding = (function Awaiting_preendorsements -> Some () | _ -> None) (fun () -> Awaiting_preendorsements); case - ~title:"Awaiting_endorsements" + ~title:"Awaiting_application" (Tag 2) unit + (function Awaiting_application -> Some () | _ -> None) + (fun () -> Awaiting_application); + case + ~title:"Awaiting_endorsements" + (Tag 3) + unit (function Awaiting_endorsements -> Some () | _ -> None) (fun () -> Awaiting_endorsements); ] @@ -376,18 +369,13 @@ let timeout_kind_encoding = (fun at_round -> Time_to_bake_next_level {at_round}); ] -type voting_power = int - type event = - | New_proposal of proposal + | New_valid_proposal of proposal + | New_head_proposal of proposal | Prequorum_reached of - Operation_worker.candidate - * voting_power - * Kind.preendorsement operation list + Operation_worker.candidate * Kind.preendorsement operation list | Quorum_reached of - Operation_worker.candidate - * voting_power - * Kind.endorsement operation list + Operation_worker.candidate * Kind.endorsement operation list | Timeout of timeout_kind let event_encoding = @@ -396,40 +384,43 @@ let event_encoding = [ case (Tag 0) - ~title:"New_proposal" + ~title:"New_valid_proposal" proposal_encoding - (function New_proposal p -> Some p | _ -> None) - (fun p -> New_proposal p); + (function New_valid_proposal p -> Some p | _ -> None) + (fun p -> New_valid_proposal p); case (Tag 1) + ~title:"New_head_proposal" + proposal_encoding + (function New_head_proposal p -> Some p | _ -> None) + (fun p -> New_head_proposal p); + case + (Tag 2) ~title:"Prequorum_reached" - (tup3 + (tup2 Operation_worker.candidate_encoding - Data_encoding.int31 (Data_encoding.list (dynamic_size Operation.encoding))) (function - | Prequorum_reached (candidate, voting_power, ops) -> - Some (candidate, voting_power, List.map Operation.pack ops) + | Prequorum_reached (candidate, ops) -> + Some (candidate, List.map Operation.pack ops) | _ -> None) - (fun (candidate, voting_power, ops) -> + (fun (candidate, ops) -> Prequorum_reached - (candidate, voting_power, Operation_pool.filter_preendorsements ops)); + (candidate, Operation_pool.filter_preendorsements ops)); case - (Tag 2) + (Tag 3) ~title:"Quorum_reached" - (tup3 + (tup2 Operation_worker.candidate_encoding - Data_encoding.int31 (Data_encoding.list (dynamic_size Operation.encoding))) (function - | Quorum_reached (candidate, voting_power, ops) -> - Some (candidate, voting_power, List.map Operation.pack ops) + | Quorum_reached (candidate, ops) -> + Some (candidate, List.map Operation.pack ops) | _ -> None) - (fun (candidate, voting_power, ops) -> - Quorum_reached - (candidate, voting_power, Operation_pool.filter_endorsements ops)); + (fun (candidate, ops) -> + Quorum_reached (candidate, Operation_pool.filter_endorsements ops)); case - (Tag 3) + (Tag 4) ~title:"Timeout" timeout_kind_encoding (function Timeout tk -> Some tk | _ -> None) @@ -730,18 +721,15 @@ let pp_block_info fmt shell; payload_hash; round; - protocol; - next_protocol; prequorum; quorum; payload; - _; + payload_round; } = Format.fprintf fmt "@[Block:@ hash: %a@ payload_hash: %a@ level: %ld@ round: %a@ \ - protocol: %a@ next protocol: %a@ prequorum: %a@ quorum: %d endorsements@ \ - payload: %a@]" + prequorum: %a@ quorum: %d endorsements@ payload: %a@ payload round: %a@]" Block_hash.pp hash Block_payload_hash.pp_short @@ -749,15 +737,13 @@ let pp_block_info fmt shell.level Round.pp round - Protocol_hash.pp_short - protocol - Protocol_hash.pp_short - next_protocol (pp_option pp_prequorum) prequorum (List.length quorum) Operation_pool.pp_payload payload + Round.pp + payload_round let pp_proposal fmt {block; _} = pp_block_info fmt block @@ -817,6 +803,9 @@ let pp_level_state fmt { current_level; latest_proposal; + is_latest_proposal_applied; + delayed_prequorum; + injected_preendorsements; locked_round; endorsable_payload; elected_block; @@ -826,11 +815,15 @@ let pp_level_state fmt } = Format.fprintf fmt - "@[Level state:@ current level: %ld@ @[proposal:@ %a@]@ locked \ - round: %a@ endorsable payload: %a@ elected block: %a@ @[own delegate \ + "@[Level state:@ current level: %ld@ @[proposal (applied:%b, \ + delayed prequorum:%b, injected preendorsements: %d):@ %a@]@ locked round: \ + %a@ endorsable payload: %a@ elected block: %a@ @[own delegate \ slots:@ %a@]@ @[next level own delegate slots:@ %a@]@ next level \ proposed round: %a@]" current_level + is_latest_proposal_applied + (Option.is_some delayed_prequorum) + (match injected_preendorsements with None -> 0 | Some l -> List.length l) pp_proposal latest_proposal (pp_option pp_locked_round) @@ -849,6 +842,7 @@ let pp_level_state fmt let pp_phase fmt = function | Idle -> Format.fprintf fmt "idle" | Awaiting_preendorsements -> Format.fprintf fmt "awaiting preendorsements" + | Awaiting_application -> Format.fprintf fmt "awaiting application" | Awaiting_endorsements -> Format.fprintf fmt "awaiting endorsements" let pp_round_state fmt {current_round; current_phase} = @@ -878,29 +872,32 @@ let pp_timeout_kind fmt = function Format.fprintf fmt "time to bake next level at round %a" Round.pp at_round let pp_event fmt = function - | New_proposal proposal -> + | New_valid_proposal proposal -> + Format.fprintf + fmt + "new valid proposal received: %a" + pp_block_info + proposal.block + | New_head_proposal proposal -> Format.fprintf fmt - "new proposal received: %a" + "new head proposal received: %a" pp_block_info proposal.block - | Prequorum_reached (candidate, voting_power, preendos) -> + | Prequorum_reached (candidate, preendos) -> Format.fprintf fmt - "pre-quorum reached with %d preendorsements (power: %d) for %a at \ - round %a" + "prequorum reached with %d preendorsements for %a at round %a" (List.length preendos) - voting_power Block_hash.pp candidate.Operation_worker.hash Round.pp candidate.round_watched - | Quorum_reached (candidate, voting_power, endos) -> + | Quorum_reached (candidate, endos) -> Format.fprintf fmt - "quorum reached with %d endorsements (power: %d) for %a at round %a" + "quorum reached with %d endorsements for %a at round %a" (List.length endos) - voting_power Block_hash.pp candidate.Operation_worker.hash Round.pp diff --git a/src/proto_016_PtMumbai/lib_delegate/baking_state.mli b/src/proto_016_PtMumbai/lib_delegate/baking_state.mli index fef6d3ccc336..cfa439d81738 100644 --- a/src/proto_016_PtMumbai/lib_delegate/baking_state.mli +++ b/src/proto_016_PtMumbai/lib_delegate/baking_state.mli @@ -57,18 +57,12 @@ type prequorum = { type block_info = { hash : Block_hash.t; shell : Block_header.shell_header; - resulting_context_hash : Context_hash.t; payload_hash : Block_payload_hash.t; payload_round : Round.t; round : Round.t; - protocol : Protocol_hash.t; - next_protocol : Protocol_hash.t; prequorum : prequorum option; quorum : Kind.endorsement operation list; payload : Operation_pool.payload; - live_blocks : Block_hash.Set.t; - (** Set of live blocks for this block that is used to filter - old or too recent operations. *) } type cache = { @@ -108,6 +102,15 @@ type proposal = {block : block_info; predecessor : block_info} val proposal_encoding : proposal Data_encoding.t +(** Identify the first block of the protocol, ie. the block that + activates the current protocol. + + This block should be baked by the baker of the previous protocol + (that's why this same block is also referred to as the last block + of the previous protocol). It is always considered final and + therefore is not endorsed.*) +val is_first_block_in_protocol : proposal -> bool + type locked_round = {payload_hash : Block_payload_hash.t; round : Round.t} val locked_round_encoding : locked_round Data_encoding.t @@ -124,6 +127,10 @@ type elected_block = { type level_state = { current_level : int32; latest_proposal : proposal; + is_latest_proposal_applied : bool; + delayed_prequorum : + (Operation_worker.candidate * Kind.preendorsement operation list) option; + injected_preendorsements : packed_operation list option; locked_round : locked_round option; endorsable_payload : endorsable_payload option; elected_block : elected_block option; @@ -132,7 +139,11 @@ type level_state = { next_level_proposed_round : Round.t option; } -type phase = Idle | Awaiting_preendorsements | Awaiting_endorsements +type phase = + | Idle + | Awaiting_preendorsements + | Awaiting_application + | Awaiting_endorsements val phase_encoding : phase Data_encoding.t @@ -154,18 +165,13 @@ type timeout_kind = val timeout_kind_encoding : timeout_kind Data_encoding.t -type voting_power = int - type event = - | New_proposal of proposal + | New_valid_proposal of proposal + | New_head_proposal of proposal | Prequorum_reached of - Operation_worker.candidate - * voting_power - * Kind.preendorsement operation list + Operation_worker.candidate * Kind.preendorsement operation list | Quorum_reached of - Operation_worker.candidate - * voting_power - * Kind.endorsement operation list + Operation_worker.candidate * Kind.endorsement operation list | Timeout of timeout_kind val event_encoding : event Data_encoding.t diff --git a/src/proto_016_PtMumbai/lib_delegate/block_forge.ml b/src/proto_016_PtMumbai/lib_delegate/block_forge.ml index d3be206ac6b6..01a06159e467 100644 --- a/src/proto_016_PtMumbai/lib_delegate/block_forge.ml +++ b/src/proto_016_PtMumbai/lib_delegate/block_forge.ml @@ -69,17 +69,19 @@ let convert_operation (op : packed_operation) : Tezos_base.Operation.t = op.protocol_data; } -(* [finalize_block_header ~shell_header ~validation_result ~operations_hash - ~pred_info ~round ~locked_round] updates the [shell_header] that was created - with dummy fields at the beginning of the block construction. It increments - the [level] and sets the actual [operations_hash], [fitness], - [validation_passes], and [context] (the predecessor resulting context hash). +(* [finalize_block_header] updates the [shell_header] that was created + with dummy fields at the beginning of the block construction. It + increments the [level] and sets the actual [operations_hash], + [fitness], [validation_passes], and [context] (the predecessor + resulting context hash). - When the operations from the block have been applied, the [fitness] is simply - retrieved from the [validation_result]. Otherwise, the [fitness] is computed - from the [round] and [locked_round] arguments. *) + When the operations from the block have been applied, the [fitness] + is simply retrieved from the [validation_result]. Otherwise, the + [fitness] is computed from the [round] and [locked_round] + arguments. *) let finalize_block_header ~shell_header ~validation_result ~operations_hash - ~(pred_info : Baking_state.block_info) ~round ~locked_round = + ~(pred_info : Baking_state.block_info) ~pred_resulting_context_hash ~round + ~locked_round = let open Lwt_result_syntax in let* fitness = match validation_result with @@ -108,7 +110,7 @@ let finalize_block_header ~shell_header ~validation_result ~operations_hash validation_passes; operations_hash; fitness; - context = pred_info.resulting_context_hash; + context = pred_resulting_context_hash; } in return header @@ -203,14 +205,15 @@ let filter_via_node ~chain_id ~fees_config ~hard_gas_limit_per_block [filter_via_node] is called to return these values. *) let filter_with_context ~chain_id ~fees_config ~hard_gas_limit_per_block ~faked_protocol_data ~user_activated_upgrades ~timestamp - ~(pred_info : Baking_state.block_info) ~force_apply ~round ~context_index - ~payload_round ~operation_pool cctxt = + ~(pred_info : Baking_state.block_info) ~pred_resulting_context_hash + ~force_apply ~round ~context_index ~payload_round ~operation_pool cctxt = let open Lwt_result_syntax in let* incremental = Baking_simulator.begin_construction ~timestamp ~protocol_data:faked_protocol_data ~force_apply + ~pred_resulting_context_hash context_index pred_info chain_id @@ -248,6 +251,7 @@ let filter_with_context ~chain_id ~fees_config ~hard_gas_limit_per_block ~validation_result ~operations_hash ~pred_info + ~pred_resulting_context_hash ~round ~locked_round:None in @@ -288,14 +292,16 @@ let apply_via_node ~chain_id ~faked_protocol_data ~timestamp consensus operations only from an [ordered_pool] via {!Operation_selection.filter_consensus_operations_only}. *) let apply_with_context ~chain_id ~faked_protocol_data ~user_activated_upgrades - ~timestamp ~(pred_info : Baking_state.block_info) ~force_apply ~round - ~ordered_pool ~context_index ~payload_hash cctxt = + ~timestamp ~(pred_info : Baking_state.block_info) + ~pred_resulting_context_hash ~force_apply ~round ~ordered_pool + ~context_index ~payload_hash cctxt = let open Lwt_result_syntax in let* incremental = Baking_simulator.begin_construction ~timestamp ~protocol_data:faked_protocol_data ~force_apply + ~pred_resulting_context_hash context_index pred_info chain_id @@ -358,6 +364,7 @@ let apply_with_context ~chain_id ~faked_protocol_data ~user_activated_upgrades ~validation_result ~operations_hash ~pred_info + ~pred_resulting_context_hash ~round ~locked_round:locked_round_when_no_validation_result in @@ -367,10 +374,10 @@ let apply_with_context ~chain_id ~faked_protocol_data ~user_activated_upgrades (* [forge] a new [unsigned_block] in accordance with [simulation_kind] and [simulation_mode] *) let forge (cctxt : #Protocol_client_context.full) ~chain_id - ~(pred_info : Baking_state.block_info) ~timestamp ~round - ~liquidity_baking_toggle_vote ~user_activated_upgrades fees_config - ~force_apply ~seed_nonce_hash ~payload_round simulation_mode simulation_kind - constants = + ~(pred_info : Baking_state.block_info) ~pred_resulting_context_hash + ~pred_live_blocks ~timestamp ~round ~liquidity_baking_toggle_vote + ~user_activated_upgrades fees_config ~force_apply ~seed_nonce_hash + ~payload_round simulation_mode simulation_kind constants = let open Lwt_result_syntax in let hard_gas_limit_per_block = constants.Constants.Parametric.hard_gas_limit_per_block @@ -382,7 +389,7 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id to our predecessor otherwise the node would reject the block. *) let filtered_pool = retain_live_operations_only - ~live_blocks:pred_info.live_blocks + ~live_blocks:pred_live_blocks operation_pool in Filter filtered_pool @@ -441,6 +448,7 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~user_activated_upgrades ~timestamp ~pred_info + ~pred_resulting_context_hash ~force_apply ~round ~context_index @@ -462,6 +470,7 @@ let forge (cctxt : #Protocol_client_context.full) ~chain_id ~user_activated_upgrades ~timestamp ~pred_info + ~pred_resulting_context_hash ~force_apply ~round ~ordered_pool diff --git a/src/proto_016_PtMumbai/lib_delegate/block_forge.mli b/src/proto_016_PtMumbai/lib_delegate/block_forge.mli index 77fb619a192f..7afc2ce9791f 100644 --- a/src/proto_016_PtMumbai/lib_delegate/block_forge.mli +++ b/src/proto_016_PtMumbai/lib_delegate/block_forge.mli @@ -44,6 +44,8 @@ val forge : #Protocol_client_context.full -> chain_id:Chain_id.t -> pred_info:Baking_state.block_info -> + pred_resulting_context_hash:Context_hash.t -> + pred_live_blocks:Block_hash.Set.t -> timestamp:Time.Protocol.t -> round:Round.t -> liquidity_baking_toggle_vote:Liquidity_baking.liquidity_baking_toggle_vote -> diff --git a/src/proto_016_PtMumbai/lib_delegate/node_rpc.ml b/src/proto_016_PtMumbai/lib_delegate/node_rpc.ml index a6d3d71cc15e..754e96117c11 100644 --- a/src/proto_016_PtMumbai/lib_delegate/node_rpc.ml +++ b/src/proto_016_PtMumbai/lib_delegate/node_rpc.ml @@ -25,6 +25,8 @@ open Protocol open Alpha_context +open Baking_cache +open Baking_state module Block_services = Block_services.Make (Protocol) (Protocol) module Events = Baking_events.Node_rpc @@ -33,12 +35,22 @@ let inject_block cctxt ?(force = false) ~chain signed_block_header operations = Data_encoding.Binary.to_bytes_exn Block_header.encoding signed_block_header in Shell_services.Injection.block + ~async:true cctxt ~chain ~force signed_shell_header_bytes operations +let inject_operation cctxt ~chain operation = + let encoded_op = + Data_encoding.Binary.to_bytes_exn Operation.encoding operation + in + (* FIXME: https://gitlab.com/tezos/tezos/-/issues/4875 + `Shell_services.Injection.operation` should be used instead once + the needed changes in the protocol are in place. *) + Shell_services.Injection.private_operation cctxt ~async:true ~chain encoded_op + let preapply_block cctxt ~chain ~head ~timestamp ~protocol_data operations = Block_services.Helpers.Preapply.block cctxt @@ -50,164 +62,219 @@ let preapply_block cctxt ~chain ~head ~timestamp ~protocol_data operations = let extract_prequorum preendorsements = match preendorsements with - | h :: _ as l -> + | h :: _ -> let ({protocol_data = {contents = Single (Preendorsement content); _}; _}) = (h : Kind.preendorsement Operation.t) in Some { - Baking_state.level = Raw_level.to_int32 content.level; + level = Raw_level.to_int32 content.level; round = content.round; block_payload_hash = content.block_payload_hash; - preendorsements = l; + preendorsements; } | _ -> None -let raw_info cctxt ~chain ~block_hash shell resulting_context_hash payload_hash - payload_round current_protocol next_protocol live_blocks = - Events.(emit raw_info (block_hash, shell.Tezos_base.Block_header.level)) - >>= fun () -> - let open Protocol_client_context in - let block = `Hash (block_hash, 0) in - let is_in_protocol = Protocol_hash.(current_protocol = Protocol.hash) in - (if is_in_protocol then - Alpha_block_services.Operations.operations cctxt ~chain ~block () - >>=? fun operations -> - let operations = - List.map - (fun l -> - List.map - (fun {Alpha_block_services.shell; protocol_data; _} -> - {Alpha_context.shell; protocol_data}) - l) - operations - in - match Operation_pool.extract_operations_of_list_list operations with - | None -> failwith "Unexpected operation list size" - | Some operations -> return operations - else - (* If we are not in the current protocol, do no consider operations *) - return (None, [], Operation_pool.empty_payload)) - >>=? fun (preendorsements, quorum, payload) -> - (match Baking_state.round_of_shell_header shell with - | Ok round -> ok round - | _ -> - (* this can occur if the protocol has just changed and the - previous protocol does not have a concept of round - (e.g. Genesis) *) - ok Round.zero) - >>?= fun round -> - let prequorum = Option.bind preendorsements extract_prequorum in +let info_of_header_and_ops ~in_protocol block_hash block_header operations = + let open Result_syntax in + let shell = block_header.Tezos_base.Block_header.shell in + let dummy_payload_hash = Block_payload_hash.zero in + let* round = + Environment.wrap_tzresult @@ Fitness.round_from_raw shell.fitness + in + let payload_hash, payload_round, prequorum, quorum, payload = + if not in_protocol then + (* The first block in the protocol is baked using the previous + protocol, the encodings might change. The baker's logic is to + consider final the first block of a new protocol and not + endorse it. Therefore, we do not need to have the correct + values here. *) + (dummy_payload_hash, Round.zero, None, [], Operation_pool.empty_payload) + else + let payload_hash, payload_round = + match + Data_encoding.Binary.of_bytes_opt + Protocol.block_header_data_encoding + block_header.protocol_data + with + | Some {contents = {payload_hash; payload_round; _}; _} -> + (payload_hash, payload_round) + | None -> assert false + in + let preendorsements, quorum, payload = + WithExceptions.Option.get + ~loc:__LOC__ + (Operation_pool.extract_operations_of_list_list operations) + in + let prequorum = Option.bind preendorsements extract_prequorum in + (payload_hash, payload_round, prequorum, quorum, payload) + in return { - Baking_state.hash = block_hash; + hash = block_hash; shell; - resulting_context_hash; payload_hash; payload_round; round; - protocol = current_protocol; - next_protocol; prequorum; quorum; payload; - live_blocks; } -let dummy_payload_hash = Block_payload_hash.zero +let compute_block_info cctxt ~in_protocol ?operations ~chain block_hash + block_header = + let open Lwt_result_syntax in + let* operations = + match operations with + | None when not in_protocol -> return_nil + | None -> + let open Protocol_client_context in + let* operations = + Alpha_block_services.Operations.operations + cctxt + ~chain + ~block:(`Hash (block_hash, 0)) + () + in + let packed_operations = + List.map + (fun l -> + List.map + (fun {Alpha_block_services.shell; protocol_data; _} -> + {Alpha_context.shell; protocol_data}) + l) + operations + in + return packed_operations + | Some operations -> + let parse_op (raw_op : Tezos_base.Operation.t) = + let protocol_data = + Data_encoding.Binary.of_bytes_exn + Operation.protocol_data_encoding + raw_op.proto + in + {shell = raw_op.shell; protocol_data} + in + protect @@ fun () -> return (List.map (List.map parse_op) operations) + in + let*? block_info = + info_of_header_and_ops ~in_protocol block_hash block_header operations + in + return block_info -let info cctxt ~chain ~block () = - let open Protocol_client_context in - (* Fails if the block's protocol is not the current one *) - Shell_services.Blocks.protocols cctxt ~chain ~block () - >>=? fun {current_protocol; next_protocol} -> - Shell_services.Blocks.resulting_context_hash cctxt ~chain ~block () - >>=? fun resulting_context_hash -> - (if Protocol_hash.(current_protocol <> Protocol.hash) then - Block_services.Header.shell_header cctxt ~chain ~block () >>=? fun shell -> - Chain_services.Blocks.Header.raw_protocol_data cctxt ~chain ~block () - >>=? fun protocol_data -> - let hash = - Tezos_base.Block_header.hash {Tezos_base.Block_header.shell; protocol_data} - in - (* /!\ We decode [protocol_data] with the current protocol's - encoding, while we should use the previous protocol's - [protocol_data] encoding. For now, this works because the - encoding has not changed. *) - let payload_hash, payload_round = - match - Data_encoding.Binary.of_bytes_opt - Protocol.block_header_data_encoding - protocol_data - with - | Some {contents = {payload_hash; payload_round; _}; _} -> - (payload_hash, payload_round) - | None -> (dummy_payload_hash, Round.zero) - in - return (hash, shell, resulting_context_hash, payload_hash, payload_round) - else - Alpha_block_services.header cctxt ~chain ~block () - >>=? fun {hash; shell; protocol_data; _} -> - return - ( hash, - shell, - resulting_context_hash, - protocol_data.contents.payload_hash, - protocol_data.contents.payload_round )) - >>=? fun (hash, shell, resulting_context_hash, payload_hash, payload_round) -> - (Chain_services.Blocks.live_blocks cctxt ~chain ~block () >>= function - | Error _ -> - (* The RPC might fail when a block's metadata is not available *) - Lwt.return Block_hash.Set.empty - | Ok live_blocks -> Lwt.return live_blocks) - >>= fun live_blocks -> - raw_info - cctxt - ~chain - ~block_hash:hash - shell - resulting_context_hash - payload_hash - payload_round - current_protocol - next_protocol - live_blocks +let proposal cctxt ?(cache : block_info Block_cache.t option) ?operations ~chain + block_hash (block_header : Tezos_base.Block_header.t) = + let open Lwt_result_syntax in + let predecessor_hash = block_header.shell.predecessor in + let pred_block = `Hash (predecessor_hash, 0) in + let predecessor_opt = + Option.bind cache (fun cache -> Block_cache.find_opt cache predecessor_hash) + in + let* is_proposal_in_protocol, predecessor = + match predecessor_opt with + | Some predecessor -> + return + ( predecessor.shell.proto_level = block_header.shell.proto_level, + predecessor ) + | None -> + let* { + current_protocol = pred_current_protocol; + next_protocol = pred_next_protocol; + } = + Shell_services.Blocks.protocols cctxt ~chain ~block:pred_block () + in + let is_proposal_in_protocol = + Protocol_hash.(pred_next_protocol = Protocol.hash) + in + let* predecessor = + let in_protocol = + Protocol_hash.(pred_current_protocol = Protocol.hash) + in + let* raw_header_b = + Shell_services.Blocks.raw_header cctxt ~chain ~block:pred_block () + in + let predecessor_header = + Data_encoding.Binary.of_bytes_exn + Tezos_base.Block_header.encoding + raw_header_b + in + compute_block_info + cctxt + ~in_protocol + ~chain + predecessor_hash + predecessor_header + in + Option.iter + (fun cache -> Block_cache.replace cache predecessor_hash predecessor) + cache ; + return (is_proposal_in_protocol, predecessor) + in + let block_opt = + Option.bind cache (fun cache -> Block_cache.find_opt cache block_hash) + in + let* block = + match block_opt with + | Some pi -> return pi + | None -> + let* pi = + compute_block_info + cctxt + ~in_protocol:is_proposal_in_protocol + ?operations + ~chain + block_hash + block_header + in + Option.iter (fun cache -> Block_cache.replace cache block_hash pi) cache ; + return pi + in + return {block; predecessor} -let find_in_cache_or_fetch cctxt ?cache ~chain block_hash = - let open Baking_cache in - let fetch () = info cctxt ~chain ~block:(`Hash (block_hash, 0)) () in - match cache with - | None -> fetch () - | Some block_cache -> ( - match Block_cache.find_opt block_cache block_hash with - | Some block_info -> return block_info - | None -> - fetch () >>=? fun block_info -> - Block_cache.replace block_cache block_hash block_info ; - return block_info) +let proposal cctxt ?cache ?operations ~chain block_hash block_header = + protect @@ fun () -> + proposal cctxt ?cache ?operations ~chain block_hash block_header -let proposal cctxt ?cache ~chain block_hash = - find_in_cache_or_fetch cctxt ~chain ?cache block_hash >>=? fun block -> - let predecessor_hash = block.shell.predecessor in - find_in_cache_or_fetch cctxt ~chain ?cache predecessor_hash - >>=? fun predecessor -> return {Baking_state.block; predecessor} +let monitor_valid_proposals cctxt ~chain ?cache () = + let open Lwt_result_syntax in + let next_protocols = [Protocol.hash] in + let* block_stream, stopper = + Monitor_services.validated_blocks cctxt ~chains:[chain] ~next_protocols () + in + let stream = + let map (_chain_id, block_hash, block_header, operations) = + let*! map_result = + proposal cctxt ?cache ~operations ~chain block_hash block_header + in + match map_result with + | Ok proposal -> Lwt.return_some proposal + | Error err -> + let*! () = Events.(emit error_while_monitoring_valid_proposals err) in + Lwt.return_none + in + Lwt_stream.filter_map_s map block_stream + in + return (stream, stopper) -let monitor_proposals cctxt ~chain () = - let cache = Baking_cache.Block_cache.create 100 in - Monitor_services.heads cctxt ~next_protocols:[Protocol.hash] chain - >>=? fun (block_stream, stopper) -> - return - ( Lwt_stream.filter_map_s - (fun (block_hash, _) -> - protect (fun () -> proposal cctxt ~cache ~chain block_hash) - >>= function - | Ok proposal -> Lwt.return_some proposal - | Error err -> - Events.(emit error_while_monitoring_heads err) >>= fun () -> - Lwt.return_none) - block_stream, - stopper ) +let monitor_heads cctxt ~chain ?cache () = + let open Lwt_result_syntax in + let next_protocols = [Protocol.hash] in + let* block_stream, stopper = + Monitor_services.heads cctxt ~next_protocols chain + in + let stream = + let map (block_hash, block_header) = + let*! map_result = proposal cctxt ?cache ~chain block_hash block_header in + match map_result with + | Ok proposal -> Lwt.return_some proposal + | Error err -> + let*! () = Events.(emit error_while_monitoring_heads err) in + Lwt.return_none + in + Lwt_stream.filter_map_s map block_stream + in + return (stream, stopper) let await_protocol_activation cctxt ~chain () = Monitor_services.heads cctxt ~next_protocols:[Protocol.hash] chain diff --git a/src/proto_016_PtMumbai/lib_delegate/node_rpc.mli b/src/proto_016_PtMumbai/lib_delegate/node_rpc.mli index 2dd0468ef6f1..00977029991a 100644 --- a/src/proto_016_PtMumbai/lib_delegate/node_rpc.mli +++ b/src/proto_016_PtMumbai/lib_delegate/node_rpc.mli @@ -39,6 +39,16 @@ val inject_block : Tezos_base.Operation.t list list -> Block_hash.t tzresult Lwt.t +(** Inject an operation. + + @return operation hash of the newly injected operation +*) +val inject_operation : + #Protocol_client_context.full -> + chain:Shell_services.chain -> + packed_operation -> + Operation_hash.t tzresult Lwt.t + (** Preapply a block using the node validation mechanism.*) val preapply_block : #Protocol_client_context.full -> @@ -50,21 +60,19 @@ val preapply_block : (Tezos_base.Block_header.shell_header * error Preapply_result.t list) tzresult Lwt.t -(** Fetch a proposal from the node. - - @param cache is unset by default -*) -val proposal : - #Tezos_rpc.Context.simple -> - ?cache:Baking_state.block_info Baking_cache.Block_cache.t -> +(** Monitor validated blocks/proposals from the node. *) +val monitor_valid_proposals : + #Protocol_client_context.rpc_context -> chain:Shell_services.chain -> - Block_hash.t -> - Baking_state.proposal tzresult Lwt.t + ?cache:Baking_state.block_info Baking_cache.Block_cache.t -> + unit -> + (Baking_state.proposal Lwt_stream.t * (unit -> unit)) tzresult Lwt.t -(** Monitor proposals from the node.*) -val monitor_proposals : +(** Monitor heads from the node. *) +val monitor_heads : #Protocol_client_context.rpc_context -> chain:Shell_services.chain -> + ?cache:Baking_state.block_info Baking_cache.Block_cache.t -> unit -> (Baking_state.proposal Lwt_stream.t * (unit -> unit)) tzresult Lwt.t diff --git a/src/proto_016_PtMumbai/lib_delegate/operation_worker.ml b/src/proto_016_PtMumbai/lib_delegate/operation_worker.ml index 66cec5723476..1568bd5bb901 100644 --- a/src/proto_016_PtMumbai/lib_delegate/operation_worker.ml +++ b/src/proto_016_PtMumbai/lib_delegate/operation_worker.ml @@ -23,13 +23,6 @@ (* *) (*****************************************************************************) -(* TODO: - add events + - running state introspection to recover/restart on failure - - Do we need a mutex ? -*) - open Protocol_client_context open Protocol open Alpha_context @@ -64,7 +57,7 @@ module Events = struct ~name:"pqc_reached" ~level:Debug ~msg: - "pre-quorum reached (voting power: {voting_power}, {preendorsements} \ + "prequorum reached (voting power: {voting_power}, {preendorsements} \ preendorsements)" ~pp1:pp_int ("voting_power", Data_encoding.int31) @@ -162,12 +155,9 @@ let candidate_encoding = (req "round_watched" Round.encoding) (req "payload_hash_watched" Block_payload_hash.encoding)) -type voting_power = int - type event = - | Prequorum_reached of - candidate * voting_power * Kind.preendorsement operation list - | Quorum_reached of candidate * voting_power * Kind.endorsement operation list + | Prequorum_reached of candidate * Kind.preendorsement operation list + | Quorum_reached of candidate * Kind.endorsement operation list type pqc_watched = { candidate_watched : candidate; @@ -257,6 +247,21 @@ let is_valid_consensus_content (candidate : candidate) consensus_content = let cancel_monitoring state = state.proposal_watched <- None +let reset_monitoring state = + Lwt_mutex.with_lock state.lock @@ fun () -> + match state.proposal_watched with + | None -> Lwt.return_unit + | Some (Pqc_watch pqc_watched) -> + pqc_watched.current_voting_power <- 0 ; + pqc_watched.preendorsements_count <- 0 ; + pqc_watched.preendorsements_received <- [] ; + Lwt.return_unit + | Some (Qc_watch qc_watched) -> + qc_watched.current_voting_power <- 0 ; + qc_watched.endorsements_count <- 0 ; + qc_watched.endorsements_received <- [] ; + Lwt.return_unit + let update_monitoring ?(should_lock = true) state ops = (if should_lock then Lwt_mutex.with_lock state.lock else fun f -> f ()) @@ fun () -> @@ -310,7 +315,6 @@ let update_monitoring ?(should_lock = true) state ops = (Some (Prequorum_reached ( candidate_watched, - proposal_watched.current_voting_power, List.rev proposal_watched.preendorsements_received ))) ; (* Once the event has been emitted, we cancel the monitoring *) cancel_monitoring state ; @@ -370,7 +374,6 @@ let update_monitoring ?(should_lock = true) state ops = (Some (Quorum_reached ( candidate_watched, - proposal_watched.current_voting_power, List.rev proposal_watched.endorsements_received ))) ; (* Once the event has been emitted, we cancel the monitoring *) cancel_monitoring state ; @@ -496,11 +499,10 @@ let create ?(monitor_node_operations = true) Lwt_stream.get operation_stream >>= function | None -> (* When the stream closes, it means a new head has been set, - we cancel the monitoring and flush current operations *) + we reset the monitoring and flush current operations *) Events.(emit end_of_stream ()) >>= fun () -> op_stream_stopper () ; - cancel_monitoring state ; - worker_loop () + reset_monitoring state >>= fun () -> worker_loop () | Some ops -> state.operation_pool <- Operation_pool.add_operations state.operation_pool ops ; diff --git a/src/proto_016_PtMumbai/lib_delegate/operation_worker.mli b/src/proto_016_PtMumbai/lib_delegate/operation_worker.mli index eecbc990f2d1..1c12e7356c37 100644 --- a/src/proto_016_PtMumbai/lib_delegate/operation_worker.mli +++ b/src/proto_016_PtMumbai/lib_delegate/operation_worker.mli @@ -41,12 +41,9 @@ type candidate = { val candidate_encoding : candidate Data_encoding.t -type voting_power = int - type event = - | Prequorum_reached of - candidate * voting_power * Kind.preendorsement operation list - | Quorum_reached of candidate * voting_power * Kind.endorsement operation list + | Prequorum_reached of candidate * Kind.preendorsement operation list + | Quorum_reached of candidate * Kind.endorsement operation list (** {1 Constructors}*) diff --git a/src/proto_016_PtMumbai/lib_delegate/state_transitions.ml b/src/proto_016_PtMumbai/lib_delegate/state_transitions.ml index 5cc55044263d..ab8f6d35d7ba 100644 --- a/src/proto_016_PtMumbai/lib_delegate/state_transitions.ml +++ b/src/proto_016_PtMumbai/lib_delegate/state_transitions.ml @@ -95,31 +95,50 @@ let make_preendorse_action state proposal = in Inject_preendorsements {preendorsements} -let update_proposal state proposal = +let update_proposal ~is_proposal_applied state proposal = Events.(emit updating_latest_proposal proposal.block.hash) >>= fun () -> - let new_level_state = {state.level_state with latest_proposal = proposal} in + let prev_proposal = state.level_state.latest_proposal in + let is_latest_proposal_applied = + (* mark as applied if it is indeed applied or if this specific proposal was + already marked as applied *) + is_proposal_applied + || prev_proposal.block.hash = proposal.block.hash + && state.level_state.is_latest_proposal_applied + in + let new_level_state = + { + state.level_state with + is_latest_proposal_applied; + latest_proposal = proposal; + } + in Lwt.return {state with level_state = new_level_state} -let may_update_proposal state (proposal : proposal) = +let may_update_proposal ~is_proposal_applied state (proposal : proposal) = assert ( Compare.Int32.( state.level_state.latest_proposal.block.shell.level = proposal.block.shell.level)) ; if Round.(state.level_state.latest_proposal.block.round < proposal.block.round) - then update_proposal state proposal + then update_proposal ~is_proposal_applied state proposal else Lwt.return state let preendorse state proposal = - if Protocol_hash.(proposal.block.protocol <> proposal.block.next_protocol) - then + if Baking_state.is_first_block_in_protocol proposal then (* We do not preendorse the first transition block *) let new_state = update_current_phase state Idle in Lwt.return (new_state, Do_nothing) else Events.(emit attempting_preendorse_proposal proposal.block.hash) >>= fun () -> - let new_state = update_current_phase state Awaiting_preendorsements in + let new_state = + (* We await for the block to be applied before updating its + locked values. *) + if state.level_state.is_latest_proposal_applied then + update_current_phase state Awaiting_preendorsements + else update_current_phase state Awaiting_application + in Lwt.return (new_state, make_preendorse_action state proposal) let extract_pqc state (new_proposal : proposal) = @@ -179,9 +198,35 @@ let may_update_endorsable_payload_with_internal_pqc state in {state with level_state = new_level_state} -let rec handle_new_proposal state (new_proposal : proposal) = +let may_update_is_latest_proposal_applied ~is_proposal_applied state + new_proposal = + let current_proposal = state.level_state.latest_proposal in + if + is_proposal_applied + && Block_hash.(current_proposal.block.hash = new_proposal.block.hash) + then + let new_level_state = + {state.level_state with is_latest_proposal_applied = true} + in + let new_state = {state with level_state = new_level_state} in + new_state + else state + +let has_already_been_handled state new_proposal = + let current_proposal = state.level_state.latest_proposal in + Block_hash.(current_proposal.block.hash = new_proposal.block.hash) + && state.level_state.is_latest_proposal_applied + +let rec handle_proposal ~is_proposal_applied state (new_proposal : proposal) = let current_level = state.level_state.current_level in let new_proposal_level = new_proposal.block.shell.level in + let current_proposal = state.level_state.latest_proposal in + let state = + may_update_is_latest_proposal_applied + ~is_proposal_applied + state + new_proposal + in if Compare.Int32.(current_level > new_proposal_level) then (* The baker is ahead, a reorg may have happened. Do nothing: wait for the node to send us the branch's head. This new head @@ -189,12 +234,11 @@ let rec handle_new_proposal state (new_proposal : proposal) = proposal and thus, its level should be at least the same as our current proposal's level. *) Events.(emit baker_is_ahead_of_node (current_level, new_proposal_level)) - >>= fun () -> Lwt.return (state, Do_nothing) + >>= fun () -> do_nothing state else if Compare.Int32.(current_level = new_proposal_level) then - (* The received head is a new proposal for the current level: - let's check if it's a valid one for us. *) - let current_proposal = state.level_state.latest_proposal in if + (* The received head is a new proposal for the current level: + let's check if it's a valid one for us. *) Block_hash.( current_proposal.predecessor.hash <> new_proposal.predecessor.hash) then @@ -202,7 +246,7 @@ let rec handle_new_proposal state (new_proposal : proposal) = emit new_proposal_is_on_another_branch (current_proposal.predecessor.hash, new_proposal.predecessor.hash)) - >>= fun () -> may_switch_branch state new_proposal + >>= fun () -> may_switch_branch ~is_proposal_applied state new_proposal else is_acceptable_proposal_for_current_level state new_proposal >>= function | Invalid -> @@ -217,15 +261,16 @@ let rec handle_new_proposal state (new_proposal : proposal) = (* The proposal is outdated: we update to be able to extract its included endorsements but we do not endorse it *) Events.(emit outdated_proposal new_proposal.block.hash) >>= fun () -> - may_update_proposal state new_proposal >>= fun state -> - do_nothing state + may_update_proposal ~is_proposal_applied state new_proposal + >>= fun state -> do_nothing state | Valid_proposal -> ( (* Valid_proposal => proposal.round = current_round *) (* Check whether we need to update our endorsable payload *) let new_state = may_update_endorsable_payload_with_internal_pqc state new_proposal in - may_update_proposal new_state new_proposal >>= fun new_state -> + may_update_proposal ~is_proposal_applied new_state new_proposal + >>= fun new_state -> (* The proposal is valid but maybe we already locked on a payload *) match new_state.level_state.locked_round with | Some locked_round -> ( @@ -243,18 +288,22 @@ let rec handle_new_proposal state (new_proposal : proposal) = (* This PQC is above our locked_round, we can preendorse it *) preendorse new_state new_proposal | _ -> - (* We shouldn't preendorse this proposal, but we should at - least watch (pre)quorums events on it *) - let new_state = - update_current_phase new_state Awaiting_preendorsements - in - Lwt.return (new_state, Watch_proposal)) + (* We shouldn't preendorse this proposal, but we + should at least watch (pre)quorums events on it + but only when it is applied otherwise we await + for the proposal to be applied. *) + if is_proposal_applied then + let new_state = + update_current_phase new_state Awaiting_preendorsements + in + Lwt.return (new_state, Watch_proposal) + else do_nothing new_state) | None -> (* Otherwise, we did not lock on any payload, thus we can preendorse it *) preendorse new_state new_proposal) else - (* new_proposal.level > current_level *) + (* Last case: new_proposal_level > current_level *) (* Possible scenarios: - we received a block for a next level - we received our own block @@ -268,6 +317,9 @@ let rec handle_new_proposal state (new_proposal : proposal) = { current_level = new_level; latest_proposal = new_proposal; + is_latest_proposal_applied = is_proposal_applied; + delayed_prequorum = None; + injected_preendorsements = None; (* Unlock values *) locked_round = None; endorsable_payload = None; @@ -279,19 +331,22 @@ let rec handle_new_proposal state (new_proposal : proposal) = in (* recursive call with the up-to-date state to handle the new level proposals *) - handle_new_proposal {state with level_state; round_state} new_proposal + handle_proposal + ~is_proposal_applied + {state with level_state; round_state} + new_proposal in let action = Update_to_level {new_level_proposal = new_proposal; compute_new_state} in Lwt.return (state, action) -and may_switch_branch state new_proposal = +and may_switch_branch ~is_proposal_applied state new_proposal = let switch_branch state = Events.(emit switching_branch ()) >>= fun () -> (* If we are on a different branch, we also need to update our [round_state] accordingly. - The recursive call to [handle_new_proposal] cannot end up + The recursive call to [handle_proposal] cannot end up with an invalid proposal as it's on a different branch, thus there is no need to backtrack to the former state as the new proposal must end up being the new [latest_proposal]. That's @@ -299,10 +354,11 @@ and may_switch_branch state new_proposal = let round_update = { Baking_actions.new_round_proposal = new_proposal; - handle_proposal = (fun state -> handle_new_proposal state new_proposal); + handle_proposal = + (fun state -> handle_proposal ~is_proposal_applied state new_proposal); } in - update_proposal state new_proposal >>= fun new_state -> + update_proposal ~is_proposal_applied state new_proposal >>= fun new_state -> (* TODO if the branch proposal is outdated, we should trigger an [End_of_round] to participate *) Lwt.return (new_state, Synchronize_round round_update) @@ -338,6 +394,25 @@ and may_switch_branch state new_proposal = Events.(emit branch_proposal_has_same_prequorum ()) >>= fun () -> do_nothing state +let may_register_early_prequorum state ((candidate, _) as received_prequorum) = + if + Block_hash.( + candidate.Operation_worker.hash + <> state.level_state.latest_proposal.block.hash) + then + Events.( + emit + unexpected_pqc_while_waiting_for_application + (candidate.hash, state.level_state.latest_proposal.block.hash)) + >>= fun () -> do_nothing state + else + Events.(emit pqc_while_waiting_for_application candidate.hash) >>= fun () -> + let new_level_state = + {state.level_state with delayed_prequorum = Some received_prequorum} + in + let new_state = {state with level_state = new_level_state} in + do_nothing new_state + (** In the association map [delegate_slots], the function returns an optional pair ([delegate], [endorsing_slot]) if for the current [round], the validator [delegate] has a endorsing slot. *) @@ -537,8 +612,8 @@ let end_of_round state current_round = let new_state = update_current_phase new_state Idle in do_nothing new_state | Some (delegate, _) -> - let last_proposal = state.level_state.latest_proposal.block in - if Protocol_hash.(last_proposal.protocol <> Protocol.hash) then + let latest_proposal = state.level_state.latest_proposal in + if Baking_state.is_first_block_in_protocol latest_proposal then (* Do not inject a block for the previous protocol! (Let the baker of the previous protocol do it.) *) do_nothing new_state @@ -607,6 +682,9 @@ let prequorum_reached_when_awaiting_preendorsements state candidate unexpected_prequorum_received (candidate.hash, latest_proposal.block.hash)) >>= fun () -> do_nothing state + else if not state.level_state.is_latest_proposal_applied then + Events.(emit handling_prequorum_on_non_applied_proposal ()) >>= fun () -> + do_nothing state else let prequorum = { @@ -673,6 +751,38 @@ let quorum_reached_when_waiting_endorsements state candidate endorsement_qc = in do_nothing new_state +let handle_expected_applied_proposal (state : Baking_state.t) = + let new_level_state = + {state.level_state with is_latest_proposal_applied = true} + in + let new_state = {state with level_state = new_level_state} in + match new_state.level_state.delayed_prequorum with + | None -> ( + (* The application arrived before the prequorum: wait for the prequorum. *) + let new_state = update_current_phase new_state Awaiting_preendorsements in + (* FIXME: https://gitlab.com/tezos/tezos/-/issues/4877 This + mechanism is only temporary and should be removed when the + protocol and prevalidator correctly accept early + preendorsements. *) + match new_state.level_state.injected_preendorsements with + | None -> do_nothing new_state + | Some preendorsements -> + let reinject_preendorsement_action = + Reinject_preendorsements {preendorsements} + in + let new_level_state = + {new_state.level_state with injected_preendorsements = None} + in + let new_state = {new_state with level_state = new_level_state} in + Lwt.return (new_state, reinject_preendorsement_action)) + | Some (candidate, preendorsement_qc) -> + (* The application arrived after the prequorum: handle the + prequorum received earlier. *) + prequorum_reached_when_awaiting_preendorsements + new_state + candidate + preendorsement_qc + (* Hypothesis: - The state is not to be modified outside this module (NB: there are exceptions in Baking_actions: the corner cases @@ -700,37 +810,72 @@ let step (state : Baking_state.t) (event : Baking_state.event) : (* If it is time to bake the next level, stop everything currently going on and propose the next level block *) time_to_bake state at_round - | Idle, New_proposal block_info -> + | Idle, New_head_proposal proposal -> Events.( emit new_head - ( block_info.block.hash, - block_info.block.shell.level, - block_info.block.round )) - >>= fun () -> handle_new_proposal state block_info - | Awaiting_endorsements, New_proposal block_info - | Awaiting_preendorsements, New_proposal block_info -> + (proposal.block.hash, proposal.block.shell.level, proposal.block.round)) + >>= fun () -> handle_proposal ~is_proposal_applied:true state proposal + | Awaiting_application, New_head_proposal proposal -> + if + Block_hash.( + state.level_state.latest_proposal.block.hash <> proposal.block.hash) + then + Events.( + emit + new_head + ( proposal.block.hash, + proposal.block.shell.level, + proposal.block.round )) + >>= fun () -> + Events.(emit unexpected_new_head_while_waiting_for_application ()) + >>= fun () -> handle_proposal ~is_proposal_applied:true state proposal + else + Events.(emit applied_expected_proposal_received proposal.block.hash) + >>= fun () -> handle_expected_applied_proposal state + | Awaiting_endorsements, New_head_proposal proposal + | Awaiting_preendorsements, New_head_proposal proposal -> Events.( emit new_head - ( block_info.block.hash, - block_info.block.shell.level, - block_info.block.round )) + (proposal.block.hash, proposal.block.shell.level, proposal.block.round)) >>= fun () -> Events.(emit new_head_while_waiting_for_qc ()) >>= fun () -> - handle_new_proposal state block_info - | ( Awaiting_preendorsements, - Prequorum_reached (candidate, _voting_power, preendorsement_qc) ) -> + handle_proposal ~is_proposal_applied:true state proposal + | Idle, New_valid_proposal proposal -> + Events.( + emit + new_valid_proposal + (proposal.block.hash, proposal.block.shell.level, proposal.block.round)) + >>= fun () -> handle_proposal ~is_proposal_applied:false state proposal + | Awaiting_application, New_valid_proposal proposal + | Awaiting_endorsements, New_valid_proposal proposal + | Awaiting_preendorsements, New_valid_proposal proposal -> + Events.( + emit + new_valid_proposal + (proposal.block.hash, proposal.block.shell.level, proposal.block.round)) + >>= fun () -> + if has_already_been_handled state proposal then + Events.(emit valid_proposal_received_after_application ()) >>= fun () -> + do_nothing state + else + Events.(emit new_valid_proposal_while_waiting_for_qc ()) >>= fun () -> + handle_proposal ~is_proposal_applied:false state proposal + | Awaiting_application, Prequorum_reached (candidate, preendorsement_qc) -> + may_register_early_prequorum state (candidate, preendorsement_qc) + | Awaiting_preendorsements, Prequorum_reached (candidate, preendorsement_qc) + -> prequorum_reached_when_awaiting_preendorsements state candidate preendorsement_qc - | ( Awaiting_endorsements, - Quorum_reached (candidate, _voting_power, endorsement_qc) ) -> + | Awaiting_endorsements, Quorum_reached (candidate, endorsement_qc) -> quorum_reached_when_waiting_endorsements state candidate endorsement_qc (* Unreachable cases *) | Idle, (Prequorum_reached _ | Quorum_reached _) | Awaiting_preendorsements, Quorum_reached _ - | Awaiting_endorsements, Prequorum_reached _ -> + | Awaiting_endorsements, Prequorum_reached _ + | Awaiting_application, Quorum_reached _ -> (* This cannot/should not happen *) do_nothing state diff --git a/src/proto_016_PtMumbai/lib_delegate/state_transitions.mli b/src/proto_016_PtMumbai/lib_delegate/state_transitions.mli index adc584312329..2fa02e3b2d8a 100644 --- a/src/proto_016_PtMumbai/lib_delegate/state_transitions.mli +++ b/src/proto_016_PtMumbai/lib_delegate/state_transitions.mli @@ -41,16 +41,16 @@ val is_acceptable_proposal_for_current_level : val make_consensus_list : state -> proposal -> (consensus_key_and_delegate * consensus_content) list -val make_preendorse_action : state -> proposal -> action - -val may_update_proposal : state -> proposal -> state Lwt.t +val may_update_proposal : + is_proposal_applied:bool -> state -> proposal -> state Lwt.t val preendorse : state -> proposal -> (state * action) Lwt.t val extract_pqc : state -> proposal -> (Kind.preendorsement operation list * Round.t) option -val handle_new_proposal : state -> proposal -> (state * action) Lwt.t +val handle_proposal : + is_proposal_applied:bool -> state -> proposal -> (state * action) Lwt.t val round_proposer : state -> diff --git a/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/faked_services.ml b/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/faked_services.ml index 624565b40889..e99d009738bf 100644 --- a/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/faked_services.ml +++ b/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/faked_services.ml @@ -8,7 +8,14 @@ module type Mocked_services_hooks = sig type mempool = Mockup.M.Block_services.Mempool.t (** The baker and endorser rely on this stream to be notified of new - blocks. *) + valid blocks. *) + val monitor_validated_blocks : + unit -> + (Chain_id.t * Block_hash.t * Block_header.t * Operation.t list list) + Tezos_rpc.Answer.stream + + (** The baker and endorser rely on this stream to be notified of new + heads. *) val monitor_heads : unit -> (Block_hash.t * Block_header.t) Tezos_rpc.Answer.stream @@ -16,6 +23,10 @@ module type Mocked_services_hooks = sig val protocols : Block_services.block -> Block_services.protocols tzresult Lwt.t + (** [raw_header] returns the byte encoded block header of the block + associated to the given block specification. *) + val raw_header : Block_services.block -> bytes tzresult Lwt.t + (** [header] returns the block header of the block associated to the given block specification. *) val header : @@ -112,6 +123,13 @@ end type hooks = (module Mocked_services_hooks) module Make (Hooks : Mocked_services_hooks) = struct + let monitor_validated_blocks = + Directory.gen_register0 + Directory.empty + Monitor_services.S.validated_blocks + (fun _next_protocol _ -> + Tezos_rpc.Answer.return_stream (Hooks.monitor_validated_blocks ())) + let monitor_heads = Directory.gen_register1 Directory.empty @@ -137,6 +155,14 @@ module Make (Hooks : Mocked_services_hooks) = struct Directory.register Directory.empty service (fun (_, block) () () -> Hooks.protocols block) + let raw_header = + Directory.prefix + (Tezos_rpc.Path.prefix Chain_services.path Block_services.path) + @@ Directory.register + Directory.empty + Mockup.M.Block_services.S.raw_header + (fun (((), _chain), block) _ _ -> Hooks.raw_header block) + let header = Directory.prefix (Tezos_rpc.Path.prefix Chain_services.path Block_services.path) @@ -281,15 +307,30 @@ module Make (Hooks : Mocked_services_hooks) = struct (fun (_, block) () () -> Hooks.raw_protocol_data block) let shell_directory chain_id = - let merge = Directory.merge in - Directory.empty |> merge monitor_heads |> merge protocols |> merge header - |> merge operations |> merge hash |> merge shell_header - |> merge resulting_context_hash - |> merge (chain chain_id) - |> merge inject_block |> merge inject_operation |> merge monitor_operations - |> merge list_blocks |> merge live_blocks |> merge raw_protocol_data - |> merge broadcast_block |> merge broadcast_operation - |> merge monitor_bootstrapped + List.fold_left + Directory.merge + Directory.empty + [ + monitor_validated_blocks; + monitor_heads; + protocols; + raw_header; + header; + operations; + hash; + shell_header; + resulting_context_hash; + chain chain_id; + inject_block; + inject_operation; + monitor_operations; + list_blocks; + live_blocks; + raw_protocol_data; + broadcast_block; + broadcast_operation; + monitor_bootstrapped; + ] let directory chain_id = let proto_directory = diff --git a/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/mockup_simulator.ml b/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/mockup_simulator.ml index 346cea3ee712..82ae1d67c295 100644 --- a/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/mockup_simulator.ml +++ b/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/mockup_simulator.ml @@ -60,8 +60,13 @@ type state = { that functionality. *) ctxt_table : Tezos_protocol_environment.rpc_context Context_hash.Table.t; (** The context table allows us to look up rpc_context by its hash. *) + validated_blocks_pipe : + (Block_hash.t * Block_header.t * Operation.t list list) Lwt_pipe.Unbounded.t; + (** [validated_blocks_pipe] is used to implement the + [monitor_validated_blocks] RPC. *) heads_pipe : (Block_hash.t * Block_header.t) Lwt_pipe.Unbounded.t; - (** [heads_pipe] is used to implement the [monitor_heads] RPC. *) + (** [heads_pipe] is used to implement the [monitor_heads] + RPC. *) operations_pipe : (Operation_hash.t * Mockup.M.Protocol.operation) option Lwt_pipe.Unbounded.t; (** [operations_pipe] is used to implement the [operations_pipe] RPC. *) @@ -108,6 +113,12 @@ module type Hooks = sig tzresult Lwt.t + val on_new_validated_block : + block_hash:Block_hash.t -> + block_header:Block_header.t -> + operations:Operation.t list list -> + (Block_hash.t * Block_header.t * Operation.t list list) option Lwt.t + val on_new_head : block_hash:Block_hash.t -> block_header:Block_header.t -> @@ -238,11 +249,32 @@ let make_mocked_services_hooks (state : state) (user_hooks : (module Hooks)) : let module Impl : Faked_services.Mocked_services_hooks = struct type mempool = Mockup.M.Block_services.Mempool.t + let monitor_validated_blocks () = + let next () = + let rec pop_until_ok () = + Lwt_pipe.Unbounded.pop state.validated_blocks_pipe + >>= fun (block_hash, block_header, operations) -> + User_hooks.on_new_validated_block + ~block_hash + ~block_header + ~operations + >>= function + | None -> pop_until_ok () + | Some (hash, head, operations) -> + Lwt.return_some (chain_id, hash, head, operations) + in + pop_until_ok () + in + let shutdown () = () in + Tezos_rpc.Answer.{next; shutdown} + let monitor_heads () = let next () = let rec pop_until_ok () = Lwt_pipe.Unbounded.pop state.heads_pipe >>= fun (block_hash, block_header) -> + (* Sleep a 0.1s to simulate a block application delay *) + Lwt_unix.sleep 0.1 >>= fun () -> User_hooks.on_new_head ~block_hash ~block_header >>= function | None -> pop_until_ok () | Some head -> Lwt.return_some head @@ -295,14 +327,55 @@ let make_mocked_services_hooks (state : state) (user_hooks : (module Hooks)) : else Protocol.hash); } + let may_lie_on_proto_level block x = + (* As for ../protocols, the baker distinguishes activation + blocks from "normal" blocks by comparing the [proto_level] of + the shell header and its predecessor. If the predecessor's + one is different, it must mean that we are considering an + activation block and must not endorse. Here, we do a bit of + hacking in order to return a different proto_level for the + predecessor of the genesis block which is considered as the + current protocol activation block. To perfectly mimic what is + supposed to happen, the first mocked up block created should + be made in the genesis protocol, however, it is not what's + done in the mockup mode. *) + let is_predecessor_of_genesis = + match block with + | `Hash (requested_hash, rel) -> + Int.equal rel 0 + && Block_hash.equal requested_hash genesis_predecessor_block_hash + | _ -> false + in + if is_predecessor_of_genesis then + { + x.rpc_context.block_header with + proto_level = pred x.rpc_context.block_header.proto_level; + } + else x.rpc_context.block_header + + let raw_header (block : Tezos_shell_services.Block_services.block) : + bytes tzresult Lwt.t = + locate_block state block >>=? fun x -> + let shell = may_lie_on_proto_level block x in + let protocol_data = + Data_encoding.Binary.to_bytes_exn + Protocol.block_header_data_encoding + x.protocol_data + in + return + (Data_encoding.Binary.to_bytes_exn + Tezos_base.Block_header.encoding + {shell; protocol_data}) + let header (block : Tezos_shell_services.Block_services.block) : Mockup.M.Block_services.block_header tzresult Lwt.t = locate_block state block >>=? fun x -> + let shell = may_lie_on_proto_level block x in return { Mockup.M.Block_services.hash = x.rpc_context.block_hash; chain_id; - shell = x.rpc_context.block_header; + shell; protocol_data = x.protocol_data; } @@ -716,6 +789,9 @@ let rec listener ~(user_hooks : (module Hooks)) ~state ~broadcast_pipe = process_block state block_hash block_header operations >>=? fun () -> User_hooks.check_chain_after_processing ~level ~round ~chain:state.chain >>=? fun () -> + Lwt_pipe.Unbounded.push + state.validated_blocks_pipe + (block_hash, block_header, operations) ; Lwt_pipe.Unbounded.push state.heads_pipe (block_hash, block_header) ; listener ~user_hooks ~state ~broadcast_pipe @@ -735,6 +811,7 @@ let create_fake_node_state ~i ~live_depth } in let chain0 = [genesis0] in + let validated_blocks_pipe = Lwt_pipe.Unbounded.create () in let heads_pipe = Lwt_pipe.Unbounded.create () in let operations_pipe = Lwt_pipe.Unbounded.create () in let genesis_block_true_hash = @@ -744,6 +821,8 @@ let create_fake_node_state ~i ~live_depth protocol_data = block_header0.protocol_data; } in + (* Only push genesis block as a new head, not a valid block: it is + the shell's semantics to not advertise "transition" blocks. *) Lwt_pipe.Unbounded.push heads_pipe (rpc_context0.block_hash, block_header0) ; return { @@ -768,6 +847,7 @@ let create_fake_node_state ~i ~live_depth .Block_header.context, rpc_context0 ); ]); + validated_blocks_pipe; heads_pipe; operations_pipe; streaming_operations = false; @@ -1026,7 +1106,6 @@ let make_genesis_context ~delegate_selection ~initial_seed ~round0 ~round1 in return (block_header, rpc_context) in - let level0_round0_duration = Protocol.Alpha_context.Round.round_duration round_durations @@ -1052,6 +1131,9 @@ module Default_hooks : Hooks = struct let on_inject_operation ~op_hash ~op = return (op_hash, op, default_propagation_vector) + let on_new_validated_block ~block_hash ~block_header ~operations = + Lwt.return (Some (block_hash, block_header, operations)) + let on_new_head ~block_hash ~block_header = Lwt.return (Some (block_hash, block_header)) diff --git a/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/mockup_simulator.mli b/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/mockup_simulator.mli index c01782653bdf..d9da9a19074f 100644 --- a/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/mockup_simulator.mli +++ b/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/mockup_simulator.mli @@ -74,6 +74,15 @@ module type Hooks = sig tzresult Lwt.t + (** This is called when a new validated block is going to be sent as + the response to a "monitor validated blocks" RPC call. Returning + [None] here terminates the process for the baker. *) + val on_new_validated_block : + block_hash:Block_hash.t -> + block_header:Block_header.t -> + operations:Operation.t list list -> + (Block_hash.t * Block_header.t * Operation.t list list) option Lwt.t + (** This is called when a new head is going to be sent as the response to a "monitor heads" RPC call. Returning [None] here terminates the process for the baker. *) diff --git a/src/proto_016_PtMumbai/lib_delegate/test/test_scenario.ml b/src/proto_016_PtMumbai/lib_delegate/test/test_scenario.ml index cb34cc8d3d63..693b84bdda01 100644 --- a/src/proto_016_PtMumbai/lib_delegate/test/test_scenario.ml +++ b/src/proto_016_PtMumbai/lib_delegate/test/test_scenario.ml @@ -24,7 +24,7 @@ let test_level_5 () = include Default_hooks let stop_on_event = function - | Baking_state.New_proposal {block; _} -> + | Baking_state.New_head_proposal {block; _} -> (* Stop the node as soon as we receive a proposal with a level higher than [level_to_reach]. *) block.shell.level > level_to_reach @@ -54,6 +54,65 @@ let test_level_5 () = in run ~config [(3, (module Hooks)); (2, (module Hooks))] +let test_preendorse_on_valid () = + let level_to_reach = 2l in + let round_to_reach = 1l in + let module Hooks : Hooks = struct + include Default_hooks + + let on_new_head ~block_hash ~block_header = + (* Stop notifying heads on the level to reach, only notify that + it has been validated *) + if block_header.Block_header.shell.level < level_to_reach then + Lwt.return_some (block_hash, block_header) + else Lwt.return_none + + let seen_candidate = ref None + + let pqc_noticed = ref false + + let stop_on_event = function + | Baking_state.Prequorum_reached (candidate, _) -> + (* Register the PQC notice. *) + (match !seen_candidate with + | Some seen_candidate + when Block_hash.(candidate.hash = seen_candidate) -> + pqc_noticed := true + | _ -> ()) ; + false + | Baking_state.Quorum_reached (candidate, _) -> + (* Ensure that we never see a QC on the seen candidate. *) + (match !seen_candidate with + | Some seen_candidate + when Block_hash.(candidate.hash = seen_candidate) -> + Stdlib.failwith "Quorum occured on the seen candidate" + | _ -> ()) ; + false + | New_head_proposal {block; _} -> + (* Ensure that we never notice a new head at the level where + we are not supposed to. *) + if block.shell.level = level_to_reach then + Stdlib.failwith "Unexpected new head event" + else false + | New_valid_proposal {block; _} -> + (* Register the seen valid proposal candidate. *) + if + block.shell.level = level_to_reach + && Protocol.Alpha_context.Round.to_int32 block.round = 0l + then seen_candidate := Some block.hash ; + (* Stop the node when we reach level 2 / round 2. *) + block.shell.level = level_to_reach + && Protocol.Alpha_context.Round.to_int32 block.round >= round_to_reach + | _ -> false + + let check_chain_on_success ~chain:_ = + assert (!seen_candidate <> None) ; + assert !pqc_noticed ; + return_unit + end in + let config = {default_config with timeout = 10} in + run ~config [(1, (module Hooks))] + (* Scenario T1 @@ -649,7 +708,7 @@ let test_scenario_m1 () = return (op_hash, op, propagation_vector) let stop_on_event = function - | Baking_state.New_proposal {block; _} -> block.shell.level > 4l + | Baking_state.New_head_proposal {block; _} -> block.shell.level > 4l | _ -> false end in let config = {default_config with timeout = 60} in @@ -679,7 +738,7 @@ let test_scenario_m2 () = include Default_hooks let stop_on_event = function - | Baking_state.New_proposal {block; _} -> block.shell.level > 5l + | Baking_state.New_head_proposal {block; _} -> block.shell.level > 5l | _ -> false end in let module Missing_node : Hooks = struct @@ -734,7 +793,7 @@ Scenario M3 from other nodes only go to A. 3. The chain should not make progress. Since we have both bootstrap1 and bootstrap2 in delegate selection they have equal voting power. Therefore - it is necessary to have 2 votes for pre-quorums (which is achieved when A + it is necessary to have 2 votes for prequorums (which is achieved when A is proposing) and 2 votes for quorums (impossible because B has no way to obtain PQC and thus cannot send endorsements). @@ -742,7 +801,7 @@ Scenario M3 let test_scenario_m3 () = let stop_on_event0 = function - | Baking_state.New_proposal {block; _} -> + | Baking_state.New_head_proposal {block; _} -> block.shell.level = 1l && Protocol.Alpha_context.Round.to_int32 block.round = 6l | _ -> false @@ -917,7 +976,7 @@ Scenario M5 let test_scenario_m5 () = let stop_on_event0 = function - | Baking_state.New_proposal {block; _} -> block.shell.level >= 2l + | Baking_state.New_head_proposal {block; _} -> block.shell.level >= 2l | _ -> false in let module Node_a_hooks : Hooks = struct @@ -1004,7 +1063,7 @@ Scenario M6 let test_scenario_m6 () = let b_proposal_2_1 = ref None in let stop_on_event0 = function - | Baking_state.New_proposal {block; _} -> block.shell.level > 4l + | Baking_state.New_head_proposal {block; _} -> block.shell.level > 4l | _ -> false in let module Node_a_hooks : Hooks = struct @@ -1134,7 +1193,7 @@ let test_scenario_m7 () = let c_received_2_1 = ref false in let d_received_2_1 = ref false in let stop_on_event0 = function - | Baking_state.New_proposal {block; _} -> block.shell.level > 4l + | Baking_state.New_head_proposal {block; _} -> block.shell.level > 4l | _ -> false in let check_chain_on_success0 node_label ~chain = @@ -1338,7 +1397,7 @@ Scenario M8 let test_scenario_m8 () = let b_proposal_2_0 = ref None in let stop_on_event0 = function - | Baking_state.New_proposal {block; _} -> block.shell.level > 4l + | Baking_state.New_head_proposal {block; _} -> block.shell.level > 4l | _ -> false in let on_inject_operation0 ~op_hash ~op = @@ -1470,6 +1529,7 @@ let tests = let open Tezos_base_test_helpers.Tztest in [ tztest "reaches level 5" `Quick test_level_5; + tztest "cannot progress without new head" `Quick test_preendorse_on_valid; tztest "scenario t1" `Quick test_scenario_t1; tztest "scenario t2" `Quick test_scenario_t2; tztest "scenario t3" `Quick test_scenario_t3; -- GitLab From 48fc1d5ccad55377523534cb1d16bda05728a003 Mon Sep 17 00:00:00 2001 From: vbot Date: Thu, 26 Jan 2023 16:39:11 +0100 Subject: [PATCH 42/46] Changelog: add entry --- CHANGES.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index c398fcc8e77b..fa1af89360c6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -200,6 +200,10 @@ Baker greater than 0. Application-dependent checks are re-enabled for re-proposal and fresh blocks at round greater than 0. +- Reduced the preendorsement injection delay by making the baker + preendorse as soon as the node considers a block as valid instead of + waiting for the node to fully apply it. (MR :gl:`!7516`) + Accuser ------- -- GitLab From a19cea3d1bc07abf6458bf585b42e87dd45eebe4 Mon Sep 17 00:00:00 2001 From: vbot Date: Mon, 20 Feb 2023 10:42:17 +0100 Subject: [PATCH 43/46] Alpha/Baker: add missing mockup RPC --- .../test/mockup_simulator/faked_services.ml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/proto_alpha/lib_delegate/test/mockup_simulator/faked_services.ml b/src/proto_alpha/lib_delegate/test/mockup_simulator/faked_services.ml index e99d009738bf..b2770d9bdd69 100644 --- a/src/proto_alpha/lib_delegate/test/mockup_simulator/faked_services.ml +++ b/src/proto_alpha/lib_delegate/test/mockup_simulator/faked_services.ml @@ -234,6 +234,17 @@ module Make (Hooks : Mocked_services_hooks) = struct | None -> failwith "faked_services.inject_operation: can't deserialize" | Some operation -> Hooks.inject_operation operation) + let inject_private_operation = + Directory.register + Directory.empty + Injection_services.S.private_operation + (fun () _chain bytes -> + match Data_encoding.Binary.of_bytes_opt Operation.encoding bytes with + | None -> + failwith + "faked_services.inject_private_operation: can't deserialize" + | Some operation -> Hooks.inject_operation operation) + let broadcast_block = Directory.register Directory.empty @@ -323,6 +334,7 @@ module Make (Hooks : Mocked_services_hooks) = struct chain chain_id; inject_block; inject_operation; + inject_private_operation; monitor_operations; list_blocks; live_blocks; -- GitLab From 0f464a020bf96442e731733568ca39ed32e430d5 Mon Sep 17 00:00:00 2001 From: vbot Date: Wed, 8 Feb 2023 15:14:10 +0100 Subject: [PATCH 44/46] Alpha/Baker: make new head close the mockup-ed operation stream --- .../test/mockup_simulator/mockup_simulator.ml | 59 ++++++++----------- 1 file changed, 24 insertions(+), 35 deletions(-) diff --git a/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.ml b/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.ml index 82ae1d67c295..cb9ff42c2250 100644 --- a/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.ml +++ b/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.ml @@ -67,8 +67,10 @@ type state = { heads_pipe : (Block_hash.t * Block_header.t) Lwt_pipe.Unbounded.t; (** [heads_pipe] is used to implement the [monitor_heads] RPC. *) - operations_pipe : - (Operation_hash.t * Mockup.M.Protocol.operation) option Lwt_pipe.Unbounded.t; + mutable operations_stream : + (Operation_hash.t * Mockup.M.Protocol.operation) list Lwt_stream.t; + mutable operations_stream_push : + (Operation_hash.t * Mockup.M.Protocol.operation) list option -> unit; (** [operations_pipe] is used to implement the [operations_pipe] RPC. *) mutable streaming_operations : bool; (** A helper flag used to implement the monitor operations RPC. *) @@ -481,23 +483,18 @@ let make_mocked_services_hooks (state : state) (user_hooks : (module Hooks)) : let streamed = ref false in state.streaming_operations <- true ; let next () = - let rec pop_until_ok () = - Lwt_pipe.Unbounded.pop state.operations_pipe >>= function + let rec loop () = + Lwt_stream.get state.operations_stream >>= function | None when !streamed -> Lwt.return None | None -> streamed := true ; Lwt.return (Some []) - | Some op -> ( - User_hooks.on_new_operation op >>= function - | None when !streamed -> pop_until_ok () - | None -> - streamed := true ; - Lwt.return (Some []) - | Some (oph, op) -> - streamed := true ; - Lwt.return (Some [((oph, op), None)])) + | Some ops -> ( + List.filter_map_s User_hooks.on_new_operation ops >>= function + | [] -> loop () + | l -> Lwt.return_some (List.map (fun x -> (x, None)) l)) in - pop_until_ok () + loop () in let shutdown () = () in Tezos_rpc.Answer.{next; shutdown} @@ -745,22 +742,15 @@ let rec process_block state block_hash (block_header : Block_header.t) then ( state.chain <- new_chain ; clear_mempool state >>=? fun () -> - (* The head has changed, the messages in the operations pipe are no - good anymore. *) - ignore (Lwt_pipe.Unbounded.pop_all_now state.operations_pipe) ; - (if state.streaming_operations then ( - state.streaming_operations <- false ; - Lwt_pipe.Unbounded.push state.operations_pipe None ; - Lwt.return ()) - else Lwt.return ()) - >>= fun () -> - (* Put back in the pipe operations that are still alive. *) - List.iter_s - (fun op -> - Lwt_pipe.Unbounded.push state.operations_pipe (Some op) ; - Lwt.return ()) - state.mempool - >>= fun () -> return_unit) + (* The head changed: notify that the stream ended. *) + state.operations_stream_push None ; + state.streaming_operations <- false ; + (* Instanciate a new stream *) + let operations_stream, operations_stream_push = Lwt_stream.create () in + state.operations_stream <- operations_stream ; + state.operations_stream_push <- operations_stream_push ; + state.operations_stream_push (Some state.mempool) ; + return_unit) else return_unit (** This process listens to broadcast block and operations and incorporates @@ -770,9 +760,7 @@ let rec listener ~(user_hooks : (module Hooks)) ~state ~broadcast_pipe = Lwt_pipe.Unbounded.pop broadcast_pipe >>= function | Broadcast_op (operation_hash, packed_operation) -> state.mempool <- (operation_hash, packed_operation) :: state.mempool ; - Lwt_pipe.Unbounded.push - state.operations_pipe - (Some (operation_hash, packed_operation)) ; + state.operations_stream_push (Some [(operation_hash, packed_operation)]) ; User_hooks.check_mempool_after_processing ~mempool:state.mempool >>=? fun () -> listener ~user_hooks ~state ~broadcast_pipe | Broadcast_block (block_hash, block_header, operations) -> @@ -813,7 +801,7 @@ let create_fake_node_state ~i ~live_depth let chain0 = [genesis0] in let validated_blocks_pipe = Lwt_pipe.Unbounded.create () in let heads_pipe = Lwt_pipe.Unbounded.create () in - let operations_pipe = Lwt_pipe.Unbounded.create () in + let operations_stream, operations_stream_push = Lwt_stream.create () in let genesis_block_true_hash = Block_header.hash { @@ -849,7 +837,8 @@ let create_fake_node_state ~i ~live_depth ]); validated_blocks_pipe; heads_pipe; - operations_pipe; + operations_stream; + operations_stream_push; streaming_operations = false; broadcast_pipes; genesis_block_true_hash; -- GitLab From c5f85714ca71f356646efb334be517a3c743d3f0 Mon Sep 17 00:00:00 2001 From: vbot Date: Mon, 20 Feb 2023 11:15:25 +0100 Subject: [PATCH 45/46] Alpha/Baker: do not propagate already handled op in tests --- .../test/mockup_simulator/mockup_simulator.ml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.ml b/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.ml index cb9ff42c2250..01f139d8f7ba 100644 --- a/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.ml +++ b/src/proto_alpha/lib_delegate/test/mockup_simulator/mockup_simulator.ml @@ -759,9 +759,13 @@ let rec listener ~(user_hooks : (module Hooks)) ~state ~broadcast_pipe = let module User_hooks = (val user_hooks : Hooks) in Lwt_pipe.Unbounded.pop broadcast_pipe >>= function | Broadcast_op (operation_hash, packed_operation) -> - state.mempool <- (operation_hash, packed_operation) :: state.mempool ; - state.operations_stream_push (Some [(operation_hash, packed_operation)]) ; - User_hooks.check_mempool_after_processing ~mempool:state.mempool + (if + List.mem_assoc ~equal:Operation_hash.equal operation_hash state.mempool + then return_unit + else ( + state.mempool <- (operation_hash, packed_operation) :: state.mempool ; + state.operations_stream_push (Some [(operation_hash, packed_operation)]) ; + User_hooks.check_mempool_after_processing ~mempool:state.mempool)) >>=? fun () -> listener ~user_hooks ~state ~broadcast_pipe | Broadcast_block (block_hash, block_header, operations) -> get_block_level block_header >>=? fun level -> -- GitLab From 35e66eb0ad878951c70fdaed8d4a2bda6630080e Mon Sep 17 00:00:00 2001 From: Albin Coquereau Date: Mon, 20 Feb 2023 11:28:50 +0100 Subject: [PATCH 46/46] Mumbai/baker: backport mockup fixes --- .../test/mockup_simulator/faked_services.ml | 12 ++++ .../test/mockup_simulator/mockup_simulator.ml | 67 +++++++++---------- 2 files changed, 42 insertions(+), 37 deletions(-) diff --git a/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/faked_services.ml b/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/faked_services.ml index e99d009738bf..b2770d9bdd69 100644 --- a/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/faked_services.ml +++ b/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/faked_services.ml @@ -234,6 +234,17 @@ module Make (Hooks : Mocked_services_hooks) = struct | None -> failwith "faked_services.inject_operation: can't deserialize" | Some operation -> Hooks.inject_operation operation) + let inject_private_operation = + Directory.register + Directory.empty + Injection_services.S.private_operation + (fun () _chain bytes -> + match Data_encoding.Binary.of_bytes_opt Operation.encoding bytes with + | None -> + failwith + "faked_services.inject_private_operation: can't deserialize" + | Some operation -> Hooks.inject_operation operation) + let broadcast_block = Directory.register Directory.empty @@ -323,6 +334,7 @@ module Make (Hooks : Mocked_services_hooks) = struct chain chain_id; inject_block; inject_operation; + inject_private_operation; monitor_operations; list_blocks; live_blocks; diff --git a/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/mockup_simulator.ml b/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/mockup_simulator.ml index 82ae1d67c295..01f139d8f7ba 100644 --- a/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/mockup_simulator.ml +++ b/src/proto_016_PtMumbai/lib_delegate/test/mockup_simulator/mockup_simulator.ml @@ -67,8 +67,10 @@ type state = { heads_pipe : (Block_hash.t * Block_header.t) Lwt_pipe.Unbounded.t; (** [heads_pipe] is used to implement the [monitor_heads] RPC. *) - operations_pipe : - (Operation_hash.t * Mockup.M.Protocol.operation) option Lwt_pipe.Unbounded.t; + mutable operations_stream : + (Operation_hash.t * Mockup.M.Protocol.operation) list Lwt_stream.t; + mutable operations_stream_push : + (Operation_hash.t * Mockup.M.Protocol.operation) list option -> unit; (** [operations_pipe] is used to implement the [operations_pipe] RPC. *) mutable streaming_operations : bool; (** A helper flag used to implement the monitor operations RPC. *) @@ -481,23 +483,18 @@ let make_mocked_services_hooks (state : state) (user_hooks : (module Hooks)) : let streamed = ref false in state.streaming_operations <- true ; let next () = - let rec pop_until_ok () = - Lwt_pipe.Unbounded.pop state.operations_pipe >>= function + let rec loop () = + Lwt_stream.get state.operations_stream >>= function | None when !streamed -> Lwt.return None | None -> streamed := true ; Lwt.return (Some []) - | Some op -> ( - User_hooks.on_new_operation op >>= function - | None when !streamed -> pop_until_ok () - | None -> - streamed := true ; - Lwt.return (Some []) - | Some (oph, op) -> - streamed := true ; - Lwt.return (Some [((oph, op), None)])) + | Some ops -> ( + List.filter_map_s User_hooks.on_new_operation ops >>= function + | [] -> loop () + | l -> Lwt.return_some (List.map (fun x -> (x, None)) l)) in - pop_until_ok () + loop () in let shutdown () = () in Tezos_rpc.Answer.{next; shutdown} @@ -745,22 +742,15 @@ let rec process_block state block_hash (block_header : Block_header.t) then ( state.chain <- new_chain ; clear_mempool state >>=? fun () -> - (* The head has changed, the messages in the operations pipe are no - good anymore. *) - ignore (Lwt_pipe.Unbounded.pop_all_now state.operations_pipe) ; - (if state.streaming_operations then ( - state.streaming_operations <- false ; - Lwt_pipe.Unbounded.push state.operations_pipe None ; - Lwt.return ()) - else Lwt.return ()) - >>= fun () -> - (* Put back in the pipe operations that are still alive. *) - List.iter_s - (fun op -> - Lwt_pipe.Unbounded.push state.operations_pipe (Some op) ; - Lwt.return ()) - state.mempool - >>= fun () -> return_unit) + (* The head changed: notify that the stream ended. *) + state.operations_stream_push None ; + state.streaming_operations <- false ; + (* Instanciate a new stream *) + let operations_stream, operations_stream_push = Lwt_stream.create () in + state.operations_stream <- operations_stream ; + state.operations_stream_push <- operations_stream_push ; + state.operations_stream_push (Some state.mempool) ; + return_unit) else return_unit (** This process listens to broadcast block and operations and incorporates @@ -769,11 +759,13 @@ let rec listener ~(user_hooks : (module Hooks)) ~state ~broadcast_pipe = let module User_hooks = (val user_hooks : Hooks) in Lwt_pipe.Unbounded.pop broadcast_pipe >>= function | Broadcast_op (operation_hash, packed_operation) -> - state.mempool <- (operation_hash, packed_operation) :: state.mempool ; - Lwt_pipe.Unbounded.push - state.operations_pipe - (Some (operation_hash, packed_operation)) ; - User_hooks.check_mempool_after_processing ~mempool:state.mempool + (if + List.mem_assoc ~equal:Operation_hash.equal operation_hash state.mempool + then return_unit + else ( + state.mempool <- (operation_hash, packed_operation) :: state.mempool ; + state.operations_stream_push (Some [(operation_hash, packed_operation)]) ; + User_hooks.check_mempool_after_processing ~mempool:state.mempool)) >>=? fun () -> listener ~user_hooks ~state ~broadcast_pipe | Broadcast_block (block_hash, block_header, operations) -> get_block_level block_header >>=? fun level -> @@ -813,7 +805,7 @@ let create_fake_node_state ~i ~live_depth let chain0 = [genesis0] in let validated_blocks_pipe = Lwt_pipe.Unbounded.create () in let heads_pipe = Lwt_pipe.Unbounded.create () in - let operations_pipe = Lwt_pipe.Unbounded.create () in + let operations_stream, operations_stream_push = Lwt_stream.create () in let genesis_block_true_hash = Block_header.hash { @@ -849,7 +841,8 @@ let create_fake_node_state ~i ~live_depth ]); validated_blocks_pipe; heads_pipe; - operations_pipe; + operations_stream; + operations_stream_push; streaming_operations = false; broadcast_pipes; genesis_block_true_hash; -- GitLab