From 1fce8e2911ce2114c32a67c24fe87a74e282e63b Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Tue, 6 Aug 2024 11:13:55 +0200 Subject: [PATCH 1/6] Store: minor refactoring --- src/lib_store/unix/store.ml | 78 ++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 40 deletions(-) diff --git a/src/lib_store/unix/store.ml b/src/lib_store/unix/store.ml index 4b215436658a..9ebef5406acc 100644 --- a/src/lib_store/unix/store.ml +++ b/src/lib_store/unix/store.ml @@ -1612,6 +1612,42 @@ module Chain = struct else return_unit else return_unit + let custom_delayed_maintenance chain_store new_head delay = + let open Lwt_result_syntax in + let*! scheduled_maintenance = + Stored_data.get chain_store.storage_maintenance.scheduled_maintenance + in + match scheduled_maintenance with + | Some target -> + (* A delayed merge is pending. *) + let level_to_merge_reached = target <= Block.level new_head in + let* () = + if level_to_merge_reached then + Stored_data.write + chain_store.storage_maintenance.scheduled_maintenance + None + else + let*! () = + Store_events.( + emit + delayed_store_merging_countdown + Int32.(sub target (Block.level new_head))) + in + return_unit + in + return level_to_merge_reached + | None -> + (* A merge is ready to be executed, setting the + target for the delayed execution. *) + let new_target = Int32.add (Block.level new_head) delay in + let* () = + Stored_data.write + chain_store.storage_maintenance.scheduled_maintenance + (Some new_target) + in + let*! () = Store_events.(emit delay_store_merging new_target) in + return_false + let set_head chain_store new_head = let open Lwt_result_syntax in Shared.update_with chain_store.chain_state (fun chain_state -> @@ -1750,46 +1786,8 @@ module Chain = struct None in return_true - | Custom delay -> ( - let*! scheduled_maintenance = - Stored_data.get - chain_store.storage_maintenance.scheduled_maintenance - in - match scheduled_maintenance with - | Some target -> - (* A delayed merge is pending. *) - let level_to_merge_reached = - target <= Block.level new_head - in - let* () = - if level_to_merge_reached then - Stored_data.write - chain_store.storage_maintenance - .scheduled_maintenance - None - else - let*! () = - Store_events.( - emit - delayed_store_merging_countdown - Int32.(sub target (Block.level new_head))) - in - return_unit - in - return level_to_merge_reached - | None -> - (* A merge is ready to be executed, setting the - target for the delayed execution. *) - let new_target = Int32.add (Block.level new_head) delay in - let* () = - Stored_data.write - chain_store.storage_maintenance.scheduled_maintenance - (Some new_target) - in - let*! () = - Store_events.(emit delay_store_merging new_target) - in - return_false) + | Custom delay -> + custom_delayed_maintenance chain_store new_head delay in (* We effectively trigger the merge only if the delayed maintenance is disabled or if the targeted delay is -- GitLab From 9943f6d0b6e6687a5620a872fd0438013cfd4465 Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Tue, 6 Aug 2024 16:01:55 +0200 Subject: [PATCH 2/6] Plugin/Alpha: introduce Shell_helpers protocol plugin --- src/lib_validation/protocol_plugin.ml | 19 ++++++++++++++ src/lib_validation/protocol_plugin.mli | 19 +++++++++++++- src/proto_alpha/lib_plugin/plugin.ml | 1 + .../lib_plugin/plugin_registerer.ml | 8 ++++++ src/proto_alpha/lib_plugin/shell_helpers.ml | 26 +++++++++++++++++++ 5 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 src/proto_alpha/lib_plugin/shell_helpers.ml diff --git a/src/lib_validation/protocol_plugin.ml b/src/lib_validation/protocol_plugin.ml index cce8f4cdab9a..9232d02eb1eb 100644 --- a/src/lib_validation/protocol_plugin.ml +++ b/src/lib_validation/protocol_plugin.ml @@ -146,12 +146,22 @@ struct let update_metrics ~protocol_metadata:_ _ _ = Lwt.return_unit end +module type SHELL_HELPERS = sig + val hash : Protocol_hash.t + + val get_blocks_per_cycle : + Tezos_protocol_environment.Context.t -> int32 option Lwt.t +end + let rpc_table : (module RPC) Protocol_hash.Table.t = Protocol_hash.Table.create 5 let metrics_table : (module METRICS) Protocol_hash.Table.t = Protocol_hash.Table.create 5 +let shell_helpers_table : (module SHELL_HELPERS) Protocol_hash.Table.t = + Protocol_hash.Table.create 5 + let register_rpc (module Rpc : RPC) = assert (not (Protocol_hash.Table.mem rpc_table Rpc.Proto.hash)) ; Protocol_hash.Table.add rpc_table Rpc.Proto.hash (module Rpc) @@ -159,10 +169,19 @@ let register_rpc (module Rpc : RPC) = let register_metrics (module Metrics : METRICS) = Protocol_hash.Table.replace metrics_table Metrics.hash (module Metrics) +let register_shell_helpers (module Shell_helpers : SHELL_HELPERS) = + assert (not (Protocol_hash.Table.mem shell_helpers_table Shell_helpers.hash)) ; + Protocol_hash.Table.add + shell_helpers_table + Shell_helpers.hash + (module Shell_helpers) + let find_rpc = Protocol_hash.Table.find rpc_table let find_metrics = Protocol_hash.Table.find metrics_table +let find_shell_helpers = Protocol_hash.Table.find shell_helpers_table + let safe_find_metrics hash = match find_metrics hash with | Some proto_metrics -> Lwt.return proto_metrics diff --git a/src/lib_validation/protocol_plugin.mli b/src/lib_validation/protocol_plugin.mli index f3461dd252e0..cd3b1007c44c 100644 --- a/src/lib_validation/protocol_plugin.mli +++ b/src/lib_validation/protocol_plugin.mli @@ -180,11 +180,22 @@ module type METRICS = sig unit Lwt.t end -(** Emtpy metrics module. All metrics are -1. *) +(** Empty metrics module. All metrics are -1. *) module Undefined_metrics_plugin (P : sig val hash : Protocol_hash.t end) : METRICS +(** Protocol specific plugin to expose some helpers function that are + helpful in scope of the Shell. *) +module type SHELL_HELPERS = sig + val hash : Protocol_hash.t + + (** [get_blocks_per_cycle ctxt] returns the blocks_per_cycle + protocol constant. *) + val get_blocks_per_cycle : + Tezos_protocol_environment.Context.t -> int32 option Lwt.t +end + (** Register a validation plugin for a specific protocol (according to its [Proto.hash]). *) val register_validation_plugin : (module T) -> unit @@ -195,6 +206,9 @@ val register_rpc : (module RPC) -> unit (** Register a metrics plugin module *) val register_metrics : (module METRICS) -> unit +(** Register a Shell_helpers plugin module *) +val register_shell_helpers : (module SHELL_HELPERS) -> unit + (** Retrieves the registered protocol with the provided hash and wraps it together with its validation plugin. @@ -218,3 +232,6 @@ val find_metrics : Protocol_hash.t -> (module METRICS) option (** Same as [find_metrics] but returns [Undefined_metrics_plugin] if not found *) val safe_find_metrics : Protocol_hash.t -> (module METRICS) Lwt.t + +(** Looks for a shell helpers plugin module for a specific protocol *) +val find_shell_helpers : Protocol_hash.t -> (module SHELL_HELPERS) option diff --git a/src/proto_alpha/lib_plugin/plugin.ml b/src/proto_alpha/lib_plugin/plugin.ml index 8fb75ce08aad..2a12f0a3103b 100644 --- a/src/proto_alpha/lib_plugin/plugin.ml +++ b/src/proto_alpha/lib_plugin/plugin.ml @@ -30,3 +30,4 @@ module View_helpers = View_helpers module RPC = RPC module Metrics = Metrics_plugin module Script_interpreter_logging = Script_interpreter_logging +module Shell_helpers = Shell_helpers diff --git a/src/proto_alpha/lib_plugin/plugin_registerer.ml b/src/proto_alpha/lib_plugin/plugin_registerer.ml index a34184865ed4..622478f22935 100644 --- a/src/proto_alpha/lib_plugin/plugin_registerer.ml +++ b/src/proto_alpha/lib_plugin/plugin_registerer.ml @@ -39,8 +39,16 @@ module Metrics = struct let hash = Registerer.Registered.hash end +module Shell_helpers = struct + include Plugin.Shell_helpers + + let hash = Registerer.Registered.hash +end + let () = Protocol_plugin.register_validation_plugin (module Validation) let () = Protocol_plugin.register_rpc (module RPC) let () = Protocol_plugin.register_metrics (module Metrics) + +let () = Protocol_plugin.register_shell_helpers (module Shell_helpers) diff --git a/src/proto_alpha/lib_plugin/shell_helpers.ml b/src/proto_alpha/lib_plugin/shell_helpers.ml new file mode 100644 index 000000000000..961524b18ce2 --- /dev/null +++ b/src/proto_alpha/lib_plugin/shell_helpers.ml @@ -0,0 +1,26 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2024 Nomadic Labs, *) +(* *) +(*****************************************************************************) + +open Protocol + +let constants_key = [Constants_repr.version; "constants"] + +let get_constants ctxt = + let open Lwt_syntax in + let* bytes_opt = Tezos_protocol_environment.Context.find ctxt constants_key in + match bytes_opt with + | Some bytes -> + return + @@ Data_encoding.Binary.of_bytes_opt + Constants_parametric_repr.encoding + bytes + | None -> return_none + +let get_blocks_per_cycle (ctxt : Tezos_protocol_environment.Context.t) = + let open Lwt_option_syntax in + let* {blocks_per_cycle; _} = get_constants ctxt in + Lwt.return_some blocks_per_cycle -- GitLab From 1b0da471c2072da06dce21b2cafd20f4c41b7d80 Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Mon, 30 Sep 2024 17:09:22 +0200 Subject: [PATCH 3/6] Plugin/019_PtParisB: backport Shell_helpers protocol plugin --- src/proto_019_PtParisB/lib_plugin/plugin.ml | 1 + .../lib_plugin/plugin_registerer.ml | 8 ++++++ .../lib_plugin/shell_helpers.ml | 26 +++++++++++++++++++ 3 files changed, 35 insertions(+) create mode 100644 src/proto_019_PtParisB/lib_plugin/shell_helpers.ml diff --git a/src/proto_019_PtParisB/lib_plugin/plugin.ml b/src/proto_019_PtParisB/lib_plugin/plugin.ml index 8fb75ce08aad..2a12f0a3103b 100644 --- a/src/proto_019_PtParisB/lib_plugin/plugin.ml +++ b/src/proto_019_PtParisB/lib_plugin/plugin.ml @@ -30,3 +30,4 @@ module View_helpers = View_helpers module RPC = RPC module Metrics = Metrics_plugin module Script_interpreter_logging = Script_interpreter_logging +module Shell_helpers = Shell_helpers diff --git a/src/proto_019_PtParisB/lib_plugin/plugin_registerer.ml b/src/proto_019_PtParisB/lib_plugin/plugin_registerer.ml index a34184865ed4..622478f22935 100644 --- a/src/proto_019_PtParisB/lib_plugin/plugin_registerer.ml +++ b/src/proto_019_PtParisB/lib_plugin/plugin_registerer.ml @@ -39,8 +39,16 @@ module Metrics = struct let hash = Registerer.Registered.hash end +module Shell_helpers = struct + include Plugin.Shell_helpers + + let hash = Registerer.Registered.hash +end + let () = Protocol_plugin.register_validation_plugin (module Validation) let () = Protocol_plugin.register_rpc (module RPC) let () = Protocol_plugin.register_metrics (module Metrics) + +let () = Protocol_plugin.register_shell_helpers (module Shell_helpers) diff --git a/src/proto_019_PtParisB/lib_plugin/shell_helpers.ml b/src/proto_019_PtParisB/lib_plugin/shell_helpers.ml new file mode 100644 index 000000000000..961524b18ce2 --- /dev/null +++ b/src/proto_019_PtParisB/lib_plugin/shell_helpers.ml @@ -0,0 +1,26 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2024 Nomadic Labs, *) +(* *) +(*****************************************************************************) + +open Protocol + +let constants_key = [Constants_repr.version; "constants"] + +let get_constants ctxt = + let open Lwt_syntax in + let* bytes_opt = Tezos_protocol_environment.Context.find ctxt constants_key in + match bytes_opt with + | Some bytes -> + return + @@ Data_encoding.Binary.of_bytes_opt + Constants_parametric_repr.encoding + bytes + | None -> return_none + +let get_blocks_per_cycle (ctxt : Tezos_protocol_environment.Context.t) = + let open Lwt_option_syntax in + let* {blocks_per_cycle; _} = get_constants ctxt in + Lwt.return_some blocks_per_cycle -- GitLab From 4723a3890b0dc0fcee605d59bfbedefc429c9dbf Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Wed, 7 Aug 2024 10:00:48 +0200 Subject: [PATCH 4/6] Plugin/020_PsParisC: backport Shell_helpers protocol plugin --- src/proto_020_PsParisC/lib_plugin/plugin.ml | 1 + .../lib_plugin/plugin_registerer.ml | 8 ++++++ .../lib_plugin/shell_helpers.ml | 26 +++++++++++++++++++ 3 files changed, 35 insertions(+) create mode 100644 src/proto_020_PsParisC/lib_plugin/shell_helpers.ml diff --git a/src/proto_020_PsParisC/lib_plugin/plugin.ml b/src/proto_020_PsParisC/lib_plugin/plugin.ml index 8fb75ce08aad..2a12f0a3103b 100644 --- a/src/proto_020_PsParisC/lib_plugin/plugin.ml +++ b/src/proto_020_PsParisC/lib_plugin/plugin.ml @@ -30,3 +30,4 @@ module View_helpers = View_helpers module RPC = RPC module Metrics = Metrics_plugin module Script_interpreter_logging = Script_interpreter_logging +module Shell_helpers = Shell_helpers diff --git a/src/proto_020_PsParisC/lib_plugin/plugin_registerer.ml b/src/proto_020_PsParisC/lib_plugin/plugin_registerer.ml index a34184865ed4..622478f22935 100644 --- a/src/proto_020_PsParisC/lib_plugin/plugin_registerer.ml +++ b/src/proto_020_PsParisC/lib_plugin/plugin_registerer.ml @@ -39,8 +39,16 @@ module Metrics = struct let hash = Registerer.Registered.hash end +module Shell_helpers = struct + include Plugin.Shell_helpers + + let hash = Registerer.Registered.hash +end + let () = Protocol_plugin.register_validation_plugin (module Validation) let () = Protocol_plugin.register_rpc (module RPC) let () = Protocol_plugin.register_metrics (module Metrics) + +let () = Protocol_plugin.register_shell_helpers (module Shell_helpers) diff --git a/src/proto_020_PsParisC/lib_plugin/shell_helpers.ml b/src/proto_020_PsParisC/lib_plugin/shell_helpers.ml new file mode 100644 index 000000000000..961524b18ce2 --- /dev/null +++ b/src/proto_020_PsParisC/lib_plugin/shell_helpers.ml @@ -0,0 +1,26 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2024 Nomadic Labs, *) +(* *) +(*****************************************************************************) + +open Protocol + +let constants_key = [Constants_repr.version; "constants"] + +let get_constants ctxt = + let open Lwt_syntax in + let* bytes_opt = Tezos_protocol_environment.Context.find ctxt constants_key in + match bytes_opt with + | Some bytes -> + return + @@ Data_encoding.Binary.of_bytes_opt + Constants_parametric_repr.encoding + bytes + | None -> return_none + +let get_blocks_per_cycle (ctxt : Tezos_protocol_environment.Context.t) = + let open Lwt_option_syntax in + let* {blocks_per_cycle; _} = get_constants ctxt in + Lwt.return_some blocks_per_cycle -- GitLab From 91d5a9987002353c0f9c919728958a86a96596ed Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Tue, 6 Aug 2024 11:17:24 +0200 Subject: [PATCH 5/6] Store: introduce auto maintenance delay --- src/lib_node_config/shared_arg.ml | 1 + src/lib_shell_services/storage_maintenance.ml | 18 ++- .../storage_maintenance.mli | 22 +++- src/lib_store/shared/store_events.ml | 6 +- src/lib_store/unix/store.ml | 111 ++++++++++++++---- 5 files changed, 131 insertions(+), 27 deletions(-) diff --git a/src/lib_node_config/shared_arg.ml b/src/lib_node_config/shared_arg.ml index 6c7f86ae00ca..7bbcc53f3896 100644 --- a/src/lib_node_config/shared_arg.ml +++ b/src/lib_node_config/shared_arg.ml @@ -766,6 +766,7 @@ module Term = struct let parse_storage_maintenance_delay_arg str = match str with | "disabled" -> `Ok Disabled + | "auto" -> `Ok Auto | _ -> ( match Int32.of_string_opt str with | Some delay -> `Ok (Custom delay) diff --git a/src/lib_shell_services/storage_maintenance.ml b/src/lib_shell_services/storage_maintenance.ml index 49a8fb0ed020..ebb63769f93e 100644 --- a/src/lib_shell_services/storage_maintenance.ml +++ b/src/lib_shell_services/storage_maintenance.ml @@ -5,7 +5,7 @@ (* *) (*****************************************************************************) -type delay = Disabled | Custom of Int32.t +type delay = Disabled | Custom of Int32.t | Auto let delay_encoding = let open Data_encoding in @@ -34,8 +34,24 @@ let delay_encoding = (obj1 (req "custom" int32)) (function Custom delay -> Some delay | _ -> None) (fun delay -> Custom delay); + case + ~title:"auto" + ~description: + "When \"auto\" is set, storage maintenance is triggered after a \ + delay that is determined automatically." + (Tag 2) + (constant "auto") + (function Auto -> Some () | _ -> None) + (fun () -> Auto); ]) +let default_auto_delay ~blocks_per_cycle = + let exclusion = Int32.(max 1l (div blocks_per_cycle 20l)) in + let limit = Int32.div blocks_per_cycle 2l in + let delay = Int32.add (Random.int32 (Int32.sub limit exclusion)) exclusion in + delay + let pp_delay fmt = function | Disabled -> Format.fprintf fmt "disabled" | Custom delay -> Format.fprintf fmt "custom %ld" delay + | Auto -> Format.fprintf fmt "auto" diff --git a/src/lib_shell_services/storage_maintenance.mli b/src/lib_shell_services/storage_maintenance.mli index 6cda25617641..14ab2e33aa0a 100644 --- a/src/lib_shell_services/storage_maintenance.mli +++ b/src/lib_shell_services/storage_maintenance.mli @@ -18,9 +18,27 @@ soon as possible, that is, at the very beginning of a new cycle dawn. [Custom n] will trigger the storage maintenance n blocks after a - new cycle dawn. *) -type delay = Disabled | Custom of Int32.t + new cycle dawn. + [Auto] will trigger the storage maintenance after a delay that is + determined automatically. For now, we enforce: + - an exclusion period: the early beginning of the cycle is not + subject to storage maintenance -- it avoids triggering the + maintenance shortly after a protocol activation or when the + payouts are injected into the network + - a trigger limit: the storage maintenance cannot be triggered + after the second half of a cycle -- it avoids overlapping with + the next cycle to come and increase too much the storage size + For a given blocks_per_cycle, we have: + - an exclusion period of 1/20th blocks_per_cycles + - a trigger limit of 1/2 blocks_per_cycles +*) +type delay = Disabled | Custom of Int32.t | Auto val delay_encoding : delay Data_encoding.t val pp_delay : Format.formatter -> delay -> unit + +(** [default_auto_delay ~blocks_per_cycle] returns a random number + between [blocks_per_cycle / 20] and [blocks_per_cycle / 2], in + line with the [delay]'s description. *) +val default_auto_delay : blocks_per_cycle:Int32.t -> Int32.t diff --git a/src/lib_store/shared/store_events.ml b/src/lib_store/shared/store_events.ml index 5105f255d924..4c79f32cc141 100644 --- a/src/lib_store/shared/store_events.ml +++ b/src/lib_store/shared/store_events.ml @@ -277,11 +277,13 @@ let end_merging_stores = ("time", Time.System.Span.encoding) let delay_store_merging = - declare_1 + declare_2 ~section ~level:Info ~name:"start_delayed_maintenance" - ~msg:"delaying storage maintenance (target {level})" + ~msg:"delaying storage ({mode}) maintenance (target {level})" + ~pp1:Storage_maintenance.pp_delay + ("mode", Storage_maintenance.delay_encoding) ("level", Data_encoding.int32) let delayed_store_merging_countdown = diff --git a/src/lib_store/unix/store.ml b/src/lib_store/unix/store.ml index 9ebef5406acc..c1b55d75a5cf 100644 --- a/src/lib_store/unix/store.ml +++ b/src/lib_store/unix/store.ml @@ -1612,6 +1612,46 @@ module Chain = struct else return_unit else return_unit + (* Sets a new target for a delayed merge. *) + let set_delayed_target chain_store ~new_head ~delay = + let open Lwt_result_syntax in + let new_target = Int32.add (Block.level new_head) delay in + let* () = + Stored_data.write + chain_store.storage_maintenance.scheduled_maintenance + (Some new_target) + in + let*! () = + Store_events.(emit delay_store_merging) + (chain_store.storage_maintenance.maintenance_delay, new_target) + in + return_unit + + (* Returns whether or not we should proceed to the merge. True upon + reaching the target. *) + let may_proceed_to_delayed_merge chain_store ~new_head ~target = + let open Lwt_result_syntax in + let level_to_merge_reached = target <= Block.level new_head in + let* () = + if level_to_merge_reached then + Stored_data.write + chain_store.storage_maintenance.scheduled_maintenance + None + else + let*! () = + Store_events.( + emit + delayed_store_merging_countdown + Int32.(sub target (Block.level new_head))) + in + return_unit + in + return level_to_merge_reached + + (* [auto_delayed_maintenance store state head] will: + - trigger a maintenance if a scheduled one is present, + - set a target for the upcoming storage maintenance, depending on + the [delay]. *) let custom_delayed_maintenance chain_store new_head delay = let open Lwt_result_syntax in let*! scheduled_maintenance = @@ -1620,32 +1660,57 @@ module Chain = struct match scheduled_maintenance with | Some target -> (* A delayed merge is pending. *) - let level_to_merge_reached = target <= Block.level new_head in - let* () = - if level_to_merge_reached then - Stored_data.write - chain_store.storage_maintenance.scheduled_maintenance - None - else - let*! () = - Store_events.( - emit - delayed_store_merging_countdown - Int32.(sub target (Block.level new_head))) - in - return_unit - in - return level_to_merge_reached + may_proceed_to_delayed_merge chain_store ~new_head ~target | None -> (* A merge is ready to be executed, setting the target for the delayed execution. *) - let new_target = Int32.add (Block.level new_head) delay in - let* () = - Stored_data.write - chain_store.storage_maintenance.scheduled_maintenance - (Some new_target) + let* () = set_delayed_target chain_store ~new_head ~delay in + return_false + + (* [auto_delayed_maintenance store state head] will: + - trigger a maintenance if a scheduled one is present, + - generate a new random delay for the upcoming storage maintenance. + Unlike [custom_delayed_maintenance], this delay is randomly + computed each time and depends on the [blocks_per_cycle] constant + of the current protocol. See [Storage_maintenance.delay]. *) + let auto_delayed_maintenance chain_store chain_state new_head = + let open Lwt_result_syntax in + let*! scheduled_maintenance = + Stored_data.get chain_store.storage_maintenance.scheduled_maintenance + in + match scheduled_maintenance with + | Some target -> + (* A delayed merge is pending. *) + may_proceed_to_delayed_merge chain_store ~new_head ~target + | None -> + (* A merge is ready to be executed, setting the + target for the delayed execution. *) + let proto_level = Block_repr.proto_level new_head in + let*! protocol_levels = + Stored_data.get chain_state.protocol_levels_data + in + let* protocol_hash = + match Protocol_levels.find proto_level protocol_levels with + | Some {protocol; _} -> return protocol + | None -> tzfail (Cannot_find_protocol proto_level) + in + let* auto_delay = + (* Looking for the blocks_per_cycle protocol constant. If + not available, the delay is set to 0. *) + match Protocol_plugin.find_shell_helpers protocol_hash with + | Some (module Shell_helpers) -> ( + let*! ctxt = Block.context_exn chain_store new_head in + let*! bpc_opt = Shell_helpers.get_blocks_per_cycle ctxt in + match bpc_opt with + | Some blocks_per_cycle -> + let v = + Storage_maintenance.default_auto_delay ~blocks_per_cycle + in + return v + | None -> return 0l) + | None -> return 0l in - let*! () = Store_events.(emit delay_store_merging new_target) in + let* () = set_delayed_target chain_store ~new_head ~delay:auto_delay in return_false let set_head chain_store new_head = @@ -1788,6 +1853,8 @@ module Chain = struct return_true | Custom delay -> custom_delayed_maintenance chain_store new_head delay + | Auto -> + auto_delayed_maintenance chain_store chain_state new_head in (* We effectively trigger the merge only if the delayed maintenance is disabled or if the targeted delay is -- GitLab From e21c611ded872946db0cb0830a8711394941aaed Mon Sep 17 00:00:00 2001 From: Victor Allombert Date: Wed, 7 Aug 2024 09:48:24 +0200 Subject: [PATCH 6/6] Tezt: test auto storage maintenance delay --- tezt/tests/storage_maintenance.ml | 133 +++++++++++++++++++++++++++++- 1 file changed, 129 insertions(+), 4 deletions(-) diff --git a/tezt/tests/storage_maintenance.ml b/tezt/tests/storage_maintenance.ml index 5ec888830010..3d30fe5b85d7 100644 --- a/tezt/tests/storage_maintenance.ml +++ b/tezt/tests/storage_maintenance.ml @@ -39,7 +39,7 @@ let wait_for_context_split node ~expected = let test_context_pruning_call = Protocol.register_test ~__FILE__ - ~title:(Format.asprintf "storage context pruning call") + ~title:"storage context pruning call" ~tags:[team; "storage"; "maintenance"; "context"; "pruning"] @@ fun protocol -> let* node1, client = @@ -86,7 +86,7 @@ let wait_for_complete_storage_maintenance node target = let test_disabled_maintenance_delay = Protocol.register_test ~__FILE__ - ~title:(Format.asprintf "storage maintenance disabled delay") + ~title:"storage maintenance disabled delay" ~tags:[team; "storage"; "maintenance"; "delay"; "disabled"] @@ fun protocol -> let* node = @@ -139,7 +139,7 @@ let test_disabled_maintenance_delay = let test_custom_maintenance_delay = Protocol.register_test ~__FILE__ - ~title:(Format.asprintf "storage maintenance custom delay") + ~title:"storage maintenance custom delay" ~tags:[team; "storage"; "maintenance"; "delay"; "custom"] @@ fun protocol -> let custom_delay = 2 in @@ -264,7 +264,132 @@ let test_custom_maintenance_delay = let* () = wait_delayed_storage_maintenance_C in unit +(* The exclusion and limit are computed according to the + `Lib_shell_services.Storage_maintenance.default_auto_delay` + function. *) +let check_auto_delay_value ~blocks_per_cycle ~target ~level = + let exclusion = max 1 (blocks_per_cycle / 20) in + let limit = blocks_per_cycle / 2 in + let generated_delay = target - level in + Check.( + (generated_delay >= exclusion) + int + ~error_msg:"delay is %L but is expected to be above %R") ; + Check.( + (generated_delay <= limit) + int + ~error_msg:"delay is %L but is expected to be below %R") ; + unit + +let wait_for_auto_delayed_maintenance node = + let filter json = + let mode = JSON.(json |-> "mode" |> as_string) in + let target = JSON.(json |-> "level" |> as_int) in + if mode = "auto" then Some target else None + in + Node.wait_for node "start_delayed_maintenance.v0" filter + +let get_blocks_per_cycle client = + let* constants = + Client.RPC.call client @@ RPC.get_chain_block_context_constants () + in + let blocks_per_cycle = JSON.(constants |-> "blocks_per_cycle" |> as_int) in + return blocks_per_cycle + +(* This test aims to check the behaviour of the auto mode of the + storage maintenance. As this "auto" parameter introduces + randomness, the test will run several steps to, hopefully, have + various delayed values. + A step consist in: + - bake several blocks to trigger a delayed maintenance, + - check that the delay respects the constraints, + - bake several blocks to actually merge. +*) +let test_auto_maintenance_delay = + Protocol.register_test + ~__FILE__ + ~title:"storage maintenance auto delay" + ~tags:[team; "storage"; "maintenance"; "delay"; "auto"] + @@ fun protocol -> + let* delayed_node = + Node.init + ~name:"delayed_node" + ~event_sections_levels:[("node.store", `Info)] + Node.[Synchronisation_threshold 0; Storage_maintenance_delay "auto"] + in + let* client = Client.init ~endpoint:(Node delayed_node) () in + let* () = + Client.activate_protocol_and_wait ~protocol ~node:delayed_node client + in + let* blocks_per_cycle = get_blocks_per_cycle client in + let step cpt ~next_cycle_dist = + Log.info "Starting step %d" cpt ; + let wait_context_split = + wait_for_context_split delayed_node ~expected:true + in + let wait_delayed_maintenance = + wait_for_auto_delayed_maintenance delayed_node + in + Log.info "Bake %d blocks to trigger a store merge" next_cycle_dist ; + let* () = bake_blocks delayed_node client ~blocks_to_bake:next_cycle_dist in + Log.info "Wait for the maintenance delay event" ; + let* next_target = wait_delayed_maintenance in + let merge_target_level = (cpt * blocks_per_cycle) + 1 in + let merge_trigger_level = ((cpt + 1) * blocks_per_cycle) + 1 in + (* We must ensure that the context split is not delayed. Indeed, for + the sake of performance, the context split aims to be called on the + block candidate to a future GC. See + [Lib_context.Sigs.Context.split] for more details. *) + Log.info "Waiting for the context split event" ; + let* split_level = wait_context_split in + Check.( + (merge_target_level = split_level) + int + ~error_msg:"split level was expected on level %L but found on %R") ; + let* () = + check_auto_delay_value + ~blocks_per_cycle + ~target:next_target + ~level:merge_trigger_level + in + let* current_level = Node.get_level delayed_node in + let just_before_delayed_trigger_level = next_target - current_level - 1 in + Log.info + "Bake enough blocks (%d) to be 1 block before the trigger of the delayed \ + maintenance" + just_before_delayed_trigger_level ; + let* () = + bake_blocks + delayed_node + client + ~blocks_to_bake:just_before_delayed_trigger_level + in + let wait_merge = + wait_for_complete_storage_maintenance delayed_node merge_target_level + in + Log.info "Bake 1 block to trigger the actual delayed maintenance" ; + let* () = bake_blocks delayed_node client ~blocks_to_bake:1 in + let* () = wait_merge in + let to_next_cycle = + blocks_per_cycle - just_before_delayed_trigger_level - 1 + in + Log.info "Step %d finished" cpt ; + return to_next_cycle + in + let run_steps nb = + let rec loop cpt acc = + if cpt >= nb then unit + else + let* acc = step cpt ~next_cycle_dist:acc in + loop (cpt + 1) acc + in + loop 0 8 + in + (* Run several steps to get various "auto" (random) delays. *) + run_steps 3 + let register ~protocols = test_context_pruning_call protocols ; test_disabled_maintenance_delay protocols ; - test_custom_maintenance_delay protocols + test_custom_maintenance_delay protocols ; + test_auto_maintenance_delay protocols -- GitLab