From f5d09606cbf22d7c13e068311a3ed7e7ede45043 Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Tue, 16 Jul 2024 21:12:12 +0200 Subject: [PATCH 1/7] Shell/RPC: export history_mode RPC --- src/lib_shell/config_directory.ml | 4 ++- src/lib_shell_services/config_services.ml | 35 ++++++++++++---------- src/lib_shell_services/config_services.mli | 23 ++++++++------ 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/src/lib_shell/config_directory.ml b/src/lib_shell/config_directory.ml index a1710f958e7d..4d19a12b7a55 100644 --- a/src/lib_shell/config_directory.ml +++ b/src/lib_shell/config_directory.ml @@ -49,7 +49,9 @@ let build_rpc_directory ~user_activated_upgrades ~user_activated_protocol_overrides ~dal_config Tezos_rpc.Directory.empty - |> register Config_services.history_mode (fun () () () -> + |> register + Config_services.History_mode_services.history_mode + (fun () () () -> let chain_store = Store.main_chain_store store in let history_mode = match Store.Chain.history_mode chain_store with diff --git a/src/lib_shell_services/config_services.ml b/src/lib_shell_services/config_services.ml index 70a0c4d8d002..93d7e8a3dab3 100644 --- a/src/lib_shell_services/config_services.ml +++ b/src/lib_shell_services/config_services.ml @@ -48,22 +48,24 @@ module Network = struct Tezos_rpc.Path.(path / "network" / "dal") end -let history_mode_encoding = - Data_encoding.( - obj2 - (req "history_mode" History_mode.encoding) - (opt "blocks_preservation_cycles" (ranged_int 0 255))) +module History_mode_services = struct + let history_mode_encoding = + Data_encoding.( + obj2 + (req "history_mode" History_mode.encoding) + (opt "blocks_preservation_cycles" (ranged_int 0 255))) -let history_mode = - Tezos_rpc.Service.get_service - ~description: - "Returns the history mode of the node's underlying storage. In full or \ - rolling mode, it provides the values of `additional_cycles` and \ - `blocks_preservation_cycles`. The sum of these values is the total \ - number of stored cycles." - ~query:Tezos_rpc.Query.empty - ~output:history_mode_encoding - Tezos_rpc.Path.(path / "history_mode") + let history_mode = + Tezos_rpc.Service.get_service + ~description: + "Returns the history mode of the node's underlying storage. In full or \ + rolling mode, it provides the values of `additional_cycles` and \ + `blocks_preservation_cycles`. The sum of these values is the total \ + number of stored cycles." + ~query:Tezos_rpc.Query.empty + ~output:history_mode_encoding + Tezos_rpc.Path.(path / "history_mode") +end module Logging = struct let configure = @@ -80,3 +82,6 @@ let user_activated_upgrades cctxt = let dal_config cctxt = Tezos_rpc.Context.make_call Network.dal_config cctxt () () () + +let history_mode cctxt = + Tezos_rpc.Context.make_call History_mode_services.history_mode cctxt () () () diff --git a/src/lib_shell_services/config_services.mli b/src/lib_shell_services/config_services.mli index 999f44d694af..1190a01afaa6 100644 --- a/src/lib_shell_services/config_services.mli +++ b/src/lib_shell_services/config_services.mli @@ -55,15 +55,17 @@ module Network : sig ([`GET], unit, unit, unit, unit, Dal_config.t) Tezos_rpc.Service.t end -val history_mode : - ( [`GET], - unit, - unit, - unit, - unit, - History_mode.t * int option, - error trace ) - Tezos_rpc.Service.raw +module History_mode_services : sig + val history_mode : + ( [`GET], + unit, + unit, + unit, + unit, + History_mode.t * int option, + error trace ) + Tezos_rpc.Service.raw +end module Logging : sig val configure : @@ -80,3 +82,6 @@ val user_activated_upgrades : #Tezos_rpc.Context.simple -> User_activated.upgrades tzresult Lwt.t val dal_config : #Tezos_rpc.Context.simple -> Dal_config.t tzresult Lwt.t + +val history_mode : + #Tezos_rpc.Context.simple -> (History_mode.t * int option) tzresult Lwt.t -- GitLab From 7c56abe5fc0407ae051e85e07c9ac5f330e5b1b3 Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Tue, 16 Jul 2024 21:10:36 +0200 Subject: [PATCH 2/7] DAL/Plugin: export blocks_per_cycle --- src/lib_dal_node/dal_plugin.ml | 14 ++++++++++---- src/lib_dal_node/dal_plugin.mli | 1 + .../lib_dal/dal_plugin_registration.ml | 1 + src/proto_alpha/lib_dal/dal_plugin_registration.ml | 2 +- src/proto_beta/lib_dal/dal_plugin_registration.ml | 2 +- 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/lib_dal_node/dal_plugin.ml b/src/lib_dal_node/dal_plugin.ml index 6ff8b1db75b1..b787d87db359 100644 --- a/src/lib_dal_node/dal_plugin.ml +++ b/src/lib_dal_node/dal_plugin.ml @@ -43,6 +43,7 @@ type proto_parameters = { sc_rollup_challenge_window_in_blocks : int; commitment_period_in_blocks : int; dal_attested_slots_validity_lag : int; + blocks_per_cycle : int32; } let proto_parameters_encoding : proto_parameters Data_encoding.t = @@ -58,6 +59,7 @@ let proto_parameters_encoding : proto_parameters Data_encoding.t = sc_rollup_challenge_window_in_blocks; commitment_period_in_blocks; dal_attested_slots_validity_lag; + blocks_per_cycle; } -> ( feature_enable, incentives_enable, @@ -67,7 +69,8 @@ let proto_parameters_encoding : proto_parameters Data_encoding.t = cryptobox_parameters, sc_rollup_challenge_window_in_blocks, commitment_period_in_blocks, - dal_attested_slots_validity_lag )) + dal_attested_slots_validity_lag, + blocks_per_cycle )) (fun ( feature_enable, incentives_enable, number_of_slots, @@ -76,7 +79,8 @@ let proto_parameters_encoding : proto_parameters Data_encoding.t = cryptobox_parameters, sc_rollup_challenge_window_in_blocks, commitment_period_in_blocks, - dal_attested_slots_validity_lag ) -> + dal_attested_slots_validity_lag, + blocks_per_cycle ) -> { feature_enable; incentives_enable; @@ -87,8 +91,9 @@ let proto_parameters_encoding : proto_parameters Data_encoding.t = sc_rollup_challenge_window_in_blocks; commitment_period_in_blocks; dal_attested_slots_validity_lag; + blocks_per_cycle; }) - (obj9 + (obj10 (req "feature_enable" bool) (req "incentives_enable" bool) (req "number_of_slots" int31) @@ -99,7 +104,8 @@ let proto_parameters_encoding : proto_parameters Data_encoding.t = Tezos_crypto_dal.Cryptobox.Verifier.parameters_encoding) (req "sc_rollup_challenge_window_in_blocks" int31) (req "commitment_period_in_blocks" int31) - (req "dal_attested_slots_validity_lag" int31)) + (req "dal_attested_slots_validity_lag" int31) + (req "blocks_per_cycle" int32)) module type T = sig module Proto : Registered_protocol.T diff --git a/src/lib_dal_node/dal_plugin.mli b/src/lib_dal_node/dal_plugin.mli index 10fd7b177276..df0422c7f255 100644 --- a/src/lib_dal_node/dal_plugin.mli +++ b/src/lib_dal_node/dal_plugin.mli @@ -52,6 +52,7 @@ type proto_parameters = { sc_rollup_challenge_window_in_blocks : int; commitment_period_in_blocks : int; dal_attested_slots_validity_lag : int; + blocks_per_cycle : int32; } val proto_parameters_encoding : proto_parameters Data_encoding.t diff --git a/src/proto_020_PsParisC/lib_dal/dal_plugin_registration.ml b/src/proto_020_PsParisC/lib_dal/dal_plugin_registration.ml index ceb1bed3a0cb..5ecadaaae6b6 100644 --- a/src/proto_020_PsParisC/lib_dal/dal_plugin_registration.ml +++ b/src/proto_020_PsParisC/lib_dal/dal_plugin_registration.ml @@ -67,6 +67,7 @@ module Plugin = struct dal_attested_slots_validity_lag = parametric.sc_rollup.reveal_activation_level .dal_attested_slots_validity_lag; + blocks_per_cycle = parametric.blocks_per_cycle; } let block_info ?chain ?block ~metadata ctxt = diff --git a/src/proto_alpha/lib_dal/dal_plugin_registration.ml b/src/proto_alpha/lib_dal/dal_plugin_registration.ml index 7c8e5bf51a6e..5ecadaaae6b6 100644 --- a/src/proto_alpha/lib_dal/dal_plugin_registration.ml +++ b/src/proto_alpha/lib_dal/dal_plugin_registration.ml @@ -52,7 +52,6 @@ module Plugin = struct } = parametric.dal in - return { Dal_plugin.feature_enable; @@ -68,6 +67,7 @@ module Plugin = struct dal_attested_slots_validity_lag = parametric.sc_rollup.reveal_activation_level .dal_attested_slots_validity_lag; + blocks_per_cycle = parametric.blocks_per_cycle; } let block_info ?chain ?block ~metadata ctxt = diff --git a/src/proto_beta/lib_dal/dal_plugin_registration.ml b/src/proto_beta/lib_dal/dal_plugin_registration.ml index 7c8e5bf51a6e..5ecadaaae6b6 100644 --- a/src/proto_beta/lib_dal/dal_plugin_registration.ml +++ b/src/proto_beta/lib_dal/dal_plugin_registration.ml @@ -52,7 +52,6 @@ module Plugin = struct } = parametric.dal in - return { Dal_plugin.feature_enable; @@ -68,6 +67,7 @@ module Plugin = struct dal_attested_slots_validity_lag = parametric.sc_rollup.reveal_activation_level .dal_attested_slots_validity_lag; + blocks_per_cycle = parametric.blocks_per_cycle; } let block_info ?chain ?block ~metadata ctxt = -- GitLab From 4f9971cb753d6589a1bf58e8ad085db448f895a8 Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Tue, 16 Jul 2024 21:16:21 +0200 Subject: [PATCH 3/7] DAL/Node: check history mode of DAL node --- src/bin_dal_node/daemon.ml | 39 +++++++++++++++++++++++++++++++++++++ src/bin_dal_node/errors.ml | 23 +++++++++++++++++++++- src/bin_dal_node/errors.mli | 1 + src/bin_dal_node/event.ml | 12 ++++++++++++ 4 files changed, 74 insertions(+), 1 deletion(-) diff --git a/src/bin_dal_node/daemon.ml b/src/bin_dal_node/daemon.ml index 5bc1787f7b4d..378e2a0c3a15 100644 --- a/src/bin_dal_node/daemon.ml +++ b/src/bin_dal_node/daemon.ml @@ -781,6 +781,41 @@ let wait_for_l1_bootstrapped (cctxt : Rpc_context.t) = let*! () = Event.(emit l1_node_bootstrapped) () in return_unit +let get_head_plugin cctxt = + let open Lwt_result_syntax in + let* header = Shell_services.Blocks.Header.shell_header cctxt () in + let head_level = header.Block_header.level in + let* (module Plugin : Dal_plugin.T) = + Proto_plugins.resolve_plugin_for_level cctxt ~level:head_level + in + let* proto_parameters = + Plugin.get_constants `Main (`Level head_level) cctxt + in + return (head_level, (module Plugin : Dal_plugin.T), proto_parameters) + +(* This function checks that in case the history mode is Rolling with a custom + number of blocks, these number of blocks are sufficient. *) +let check_history_mode config profile_ctxt proto_parameters = + let open Lwt_result_syntax in + let supports_refutations = + Profile_manager.supports_refutations profile_ctxt + in + let storage_period = + Profile_manager.get_attested_data_default_store_period + profile_ctxt + proto_parameters + in + match config.Configuration_file.history_mode with + | Rolling {blocks = `Some b} -> + let minimal_levels = + if supports_refutations then storage_period + else proto_parameters.attestation_lag + in + if b < minimal_levels then + tzfail @@ Errors.Not_enough_history {stored_levels = b; minimal_levels} + else return_unit + | Rolling {blocks = `Auto} | Full -> return_unit + (* FIXME: https://gitlab.com/tezos/tezos/-/issues/3605 Improve general architecture, handle L1 disconnection etc *) @@ -907,6 +942,10 @@ let run ~data_dir ~configuration_override = in (* First wait for the L1 node to be bootstrapped. *) let* () = wait_for_l1_bootstrapped cctxt in + (* Check the DAL node's history mode. *) + let* ((_, _, proto_parameters) as _plugin_info) = get_head_plugin cctxt in + let* profile_ctxt = Handler.build_profile_context config in + let* () = check_history_mode config profile_ctxt proto_parameters in let*! crawler = let open Constants in Crawler.start diff --git a/src/bin_dal_node/errors.ml b/src/bin_dal_node/errors.ml index 6ebc44264f2e..80c3906f222a 100644 --- a/src/bin_dal_node/errors.ml +++ b/src/bin_dal_node/errors.ml @@ -30,6 +30,7 @@ type error += | Profile_incompatibility | Invalid_slot_index of {slot_index : int; number_of_slots : int} | Cryptobox_initialisation_failed of string + | Not_enough_history of {stored_levels : int; minimal_levels : int} (* TODO: https://gitlab.com/tezos/tezos/-/issues/4622 @@ -91,7 +92,27 @@ let () = msg) Data_encoding.(obj1 (req "error" string)) (function Cryptobox_initialisation_failed str -> Some str | _ -> None) - (fun str -> Cryptobox_initialisation_failed str) + (fun str -> Cryptobox_initialisation_failed str) ; + register_error_kind + `Permanent + ~id:"dal.node.not_enough_history" + ~title:"Not enough history" + ~description:"The node does not store sufficiently many levels" + ~pp:(fun ppf (stored_levels, minimal_levels) -> + Format.fprintf + ppf + "The node's history mode specifies that data for %d levels should be \ + stored, but the minimum required is %d levels." + stored_levels + minimal_levels) + Data_encoding.( + obj2 (req "stored_levels" int31) (req "minimal_levels" int31)) + (function + | Not_enough_history {stored_levels; minimal_levels} -> + Some (stored_levels, minimal_levels) + | _ -> None) + (fun (stored_levels, minimal_levels) -> + Not_enough_history {stored_levels; minimal_levels}) ; (** This part defines and handles more elaborate errors for the DAL node. *) diff --git a/src/bin_dal_node/errors.mli b/src/bin_dal_node/errors.mli index 04fae3f0ef43..22ccf7e0772e 100644 --- a/src/bin_dal_node/errors.mli +++ b/src/bin_dal_node/errors.mli @@ -30,6 +30,7 @@ type error += | Profile_incompatibility | Invalid_slot_index of {slot_index : int; number_of_slots : int} | Cryptobox_initialisation_failed of string + | Not_enough_history of {stored_levels : int; minimal_levels : int} (** The errors below are used to extend tzresult/tztrace monad/errors with Some specific errors on which we'd like to match in the DAL node's code. *) diff --git a/src/bin_dal_node/event.ml b/src/bin_dal_node/event.ml index d8582c7d9c44..ea57adb71fd4 100644 --- a/src/bin_dal_node/event.ml +++ b/src/bin_dal_node/event.ml @@ -193,6 +193,18 @@ let failed_to_fetch_block = ("last_notified", Data_encoding.int32) ("error", Error_monad.trace_encoding) +let history_mode_warning = + declare_2 + ~section + ~name:"dal_node_history_mode_warning" + ~msg: + "The node will only store data related to the last {stored_levels} \ + levels, but it should store data for {storage_period} levels in order \ + to be able to participate in refutation games" + ~level:Warning + ("stored_levels", Data_encoding.int31) + ("storage_period", Data_encoding.int31) + let configuration_loaded = declare_0 ~section -- GitLab From f92d708ff21fe29d7c48bc75a016385b68095c6c Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Tue, 16 Jul 2024 21:16:36 +0200 Subject: [PATCH 4/7] DAL/Node: check history mode of L1 node --- src/bin_dal_node/daemon.ml | 51 ++++++++++++++++++++++++++++++++++++- src/bin_dal_node/errors.ml | 24 +++++++++++++++++ src/bin_dal_node/errors.mli | 1 + 3 files changed, 75 insertions(+), 1 deletion(-) diff --git a/src/bin_dal_node/daemon.ml b/src/bin_dal_node/daemon.ml index 378e2a0c3a15..a0d52bf4586f 100644 --- a/src/bin_dal_node/daemon.ml +++ b/src/bin_dal_node/daemon.ml @@ -816,6 +816,54 @@ let check_history_mode config profile_ctxt proto_parameters = else return_unit | Rolling {blocks = `Auto} | Full -> return_unit +(* This function checks the L1 node stores enough block data for the DAL node to + function correctly. *) +let check_l1_history_mode profile_ctxt cctxt proto_parameters = + let open Lwt_result_syntax in + let* l1_history_mode = + let* l1_mode, bpc_opt = Config_services.history_mode cctxt in + (* Note: For the DAL node it does not matter if the L1 node is in Full or + Rolling mode, because the DAL node is not interested in blocks outside of + a certain time window. *) + return + @@ + match (l1_mode, bpc_opt) with + | Archive, _ -> `L1_archive + | Full None, None + | Rolling None, None + | Full (Some _), None + | Rolling (Some _), None + | Full None, Some _ + | Rolling None, Some _ -> + (* unreachable cases, at least for now *) assert false + | Full (Some additional_cycles), Some blocks_preservation_cycles + | Rolling (Some additional_cycles), Some blocks_preservation_cycles -> + `L1_rolling (additional_cycles.offset + blocks_preservation_cycles) + in + let handle ~dal_blocks ~l1_cycles = + let blocks_per_cycle = + Int32.to_int proto_parameters.Dal_plugin.blocks_per_cycle + in + let dal_cycles = dal_blocks / blocks_per_cycle in + let minimal_cycles = + if dal_blocks mod blocks_per_cycle = 0 then dal_cycles else 1 + dal_cycles + in + if l1_cycles < minimal_cycles then + tzfail + @@ Errors.Not_enough_l1_history + {stored_cycles = l1_cycles; minimal_cycles} + else return_unit + in + match l1_history_mode with + | `L1_archive -> return_unit + | `L1_rolling c -> + let b = + Profile_manager.get_attested_data_default_store_period + profile_ctxt + proto_parameters + in + handle ~dal_blocks:b ~l1_cycles:c + (* FIXME: https://gitlab.com/tezos/tezos/-/issues/3605 Improve general architecture, handle L1 disconnection etc *) @@ -942,10 +990,11 @@ let run ~data_dir ~configuration_override = in (* First wait for the L1 node to be bootstrapped. *) let* () = wait_for_l1_bootstrapped cctxt in - (* Check the DAL node's history mode. *) + (* Check the DAL node's and L1 node's history mode. *) let* ((_, _, proto_parameters) as _plugin_info) = get_head_plugin cctxt in let* profile_ctxt = Handler.build_profile_context config in let* () = check_history_mode config profile_ctxt proto_parameters in + let* () = check_l1_history_mode profile_ctxt cctxt proto_parameters in let*! crawler = let open Constants in Crawler.start diff --git a/src/bin_dal_node/errors.ml b/src/bin_dal_node/errors.ml index 80c3906f222a..5c03fb05f3b7 100644 --- a/src/bin_dal_node/errors.ml +++ b/src/bin_dal_node/errors.ml @@ -31,6 +31,7 @@ type error += | Invalid_slot_index of {slot_index : int; number_of_slots : int} | Cryptobox_initialisation_failed of string | Not_enough_history of {stored_levels : int; minimal_levels : int} + | Not_enough_l1_history of {stored_cycles : int; minimal_cycles : int} (* TODO: https://gitlab.com/tezos/tezos/-/issues/4622 @@ -113,6 +114,29 @@ let () = | _ -> None) (fun (stored_levels, minimal_levels) -> Not_enough_history {stored_levels; minimal_levels}) ; + register_error_kind + `Permanent + ~id:"dal.node.not_enough_l1_history" + ~title:"Not enough L1 history" + ~description:"The L1 node does not store sufficiently many cycles" + ~pp:(fun ppf (stored_cycles, minimal_cycles) -> + Format.fprintf + ppf + "The L1 node's history mode stores block data for %d cycles, but the \ + minimum required by the DAL node is %d cycles. Increase the number of \ + cycles the L1 node stores using the CLI argument `--history-mode \ + rolling:%d`." + stored_cycles + minimal_cycles + minimal_cycles) + Data_encoding.( + obj2 (req "stored_cycles" int31) (req "minimal_cycles" int31)) + (function + | Not_enough_l1_history {stored_cycles; minimal_cycles} -> + Some (stored_cycles, minimal_cycles) + | _ -> None) + (fun (stored_cycles, minimal_cycles) -> + Not_enough_l1_history {stored_cycles; minimal_cycles}) (** This part defines and handles more elaborate errors for the DAL node. *) diff --git a/src/bin_dal_node/errors.mli b/src/bin_dal_node/errors.mli index 22ccf7e0772e..fc5e70cfb55f 100644 --- a/src/bin_dal_node/errors.mli +++ b/src/bin_dal_node/errors.mli @@ -31,6 +31,7 @@ type error += | Invalid_slot_index of {slot_index : int; number_of_slots : int} | Cryptobox_initialisation_failed of string | Not_enough_history of {stored_levels : int; minimal_levels : int} + | Not_enough_l1_history of {stored_cycles : int; minimal_cycles : int} (** The errors below are used to extend tzresult/tztrace monad/errors with Some specific errors on which we'd like to match in the DAL node's code. *) -- GitLab From 4427cba0561249b931a72e0838790987391c8010 Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Thu, 18 Jul 2024 11:43:22 +0200 Subject: [PATCH 5/7] Tezt/DAL: refactor setup_node --- tezt/tests/dal.ml | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/tezt/tests/dal.ml b/tezt/tests/dal.ml index e0cbb38a4775..17086c63da48 100644 --- a/tezt/tests/dal.ml +++ b/tezt/tests/dal.ml @@ -181,14 +181,21 @@ let slot_size_param slot_size = make_int_parameter ["dal_parametric"; "slot_size"] slot_size (* Some initialization functions to start needed nodes. *) +let generate_protocol_parameters base protocol parameter_overrides = + let* parameter_file = + Protocol.write_parameter_file ~base parameter_overrides + in + let* client = Client.init_mockup ~parameter_file ~protocol () in + Client.RPC.call client @@ RPC.get_chain_block_context_constants () let setup_node ?(custom_constants = None) ?(additional_bootstrap_accounts = 0) - ~parameters ~protocol ?activation_timestamp ?(event_sections_levels = []) - ?(node_arguments = []) ?(dal_bootstrap_peers = []) () = - (* Temporary setup to initialise the node. *) + ~parameter_overrides ~protocol ?activation_timestamp + ?(event_sections_levels = []) ?(node_arguments = []) + ?(dal_bootstrap_peers = []) () = let base = Either.right (protocol, custom_constants) in - let* parameter_file = Protocol.write_parameter_file ~base parameters in - let* client = Client.init_mockup ~parameter_file ~protocol () in + let* proto_parameters = + generate_protocol_parameters base protocol parameter_overrides + in let nodes_args = Node. [ @@ -197,7 +204,9 @@ let setup_node ?(custom_constants = None) ?(additional_bootstrap_accounts = 0) in let node = Node.create nodes_args in let* () = Node.config_init node [] in - let* dal_parameters = Dal.Parameters.from_client client in + let dal_parameters = + Dal.Parameters.from_protocol_parameters proto_parameters + in let config : Cryptobox.Config.t = {activated = true; bootstrap_peers = dal_bootstrap_peers} in @@ -221,7 +230,7 @@ let setup_node ?(custom_constants = None) ?(additional_bootstrap_accounts = 0) Protocol.write_parameter_file ~additional_bootstrap_accounts ~base - parameters + parameter_overrides in let* () = Client.activate_protocol_and_wait @@ -240,7 +249,7 @@ let with_layer1 ?custom_constants ?additional_bootstrap_accounts ?node_arguments ?activation_timestamp ?dal_bootstrap_peers ?(parameters = []) ?(prover = true) ?smart_rollup_timeout_period_in_blocks f ~protocol = - let parameters = + let parameter_overrides = make_int_parameter ["dal_parametric"; "attestation_lag"] attestation_lag @ make_int_parameter ["dal_parametric"; "number_of_shards"] number_of_shards @ make_int_parameter @@ -285,7 +294,7 @@ let with_layer1 ?custom_constants ?additional_bootstrap_accounts ?node_arguments ?activation_timestamp ?dal_bootstrap_peers - ~parameters + ~parameter_overrides ~protocol () in @@ -3750,12 +3759,12 @@ let make_invalid_dal_node protocol parameters = (* Create another L1 node with different DAL parameters. *) let* node2, _client2, _xdal_parameters2 = let crypto_params = parameters.Dal.Parameters.cryptobox in - let parameters = + let parameter_overrides = dal_enable_param (Some true) @ redundancy_factor_param (Some (crypto_params.redundancy_factor / 2)) @ slot_size_param (Some (crypto_params.slot_size / 2)) in - setup_node ~protocol ~parameters () + setup_node ~protocol ~parameter_overrides () in (* Create a second DAL node with node2 and client2 as argument (so different DAL parameters compared to dal_node1. *) @@ -4117,7 +4126,7 @@ let test_l1_migration_scenario ?(tags = []) ~migrate_from ~migrate_to (Protocol.name migrate_to) description) @@ fun () -> - let parameters = + let parameter_overrides = make_int_parameter ["consensus_committee_size"] consensus_committee_size @ make_int_parameter ["dal_parametric"; "attestation_lag"] attestation_lag @ make_int_parameter ["dal_parametric"; "number_of_slots"] number_of_slots @@ -4129,7 +4138,7 @@ let test_l1_migration_scenario ?(tags = []) ~migrate_from ~migrate_to @ make_int_parameter ["dal_parametric"; "page_size"] page_size in let* node, client, dal_parameters = - setup_node ~parameters ~protocol:migrate_from () + setup_node ~parameter_overrides ~protocol:migrate_from () in Log.info "Set user-activated-upgrade at level %d" migration_level ; @@ -4768,7 +4777,7 @@ let test_start_dal_node_around_migration ~migrate_from ~migrate_to ~offset let slot_size = Some 126944 in let redundancy_factor = Some 8 in let page_size = Some 3967 in - let parameters = + let parameter_overrides = make_int_parameter ["consensus_committee_size"] consensus_committee_size @ make_int_parameter ["dal_parametric"; "attestation_lag"] attestation_lag @ make_int_parameter ["dal_parametric"; "number_of_slots"] number_of_slots @@ -4780,7 +4789,7 @@ let test_start_dal_node_around_migration ~migrate_from ~migrate_to ~offset @ make_int_parameter ["dal_parametric"; "page_size"] page_size in let* node, client, dal_parameters = - setup_node ~parameters ~protocol:migrate_from () + setup_node ~parameter_overrides ~protocol:migrate_from () in (* The [+2] here is a bit arbitrary. We just want to avoid possible corner @@ -6073,7 +6082,7 @@ module Garbage_collection = struct (* The period in blocks for which cells are kept. *) let non_gc_period = 2 * (a + b + c) in Log.info "The \"non-GC period\" is %d" non_gc_period ; - let parameters = + let parameter_overrides = make_int_parameter ["smart_rollup_commitment_period_in_blocks"] (Some a) @@ -6088,7 +6097,7 @@ module Garbage_collection = struct (Some c) in let* node, client, dal_parameters = - setup_node ~parameters ~protocol () + setup_node ~parameter_overrides ~protocol () in let number_of_slots = dal_parameters.Dal.Parameters.number_of_slots in let lag = dal_parameters.attestation_lag in -- GitLab From 18d7e36c428dcf3be536faa249c8b134832fa5a5 Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Wed, 17 Jul 2024 16:56:57 +0200 Subject: [PATCH 6/7] Tezt/Dal: adapt L1 history mode --- tezt/lib_tezos/dal_common.ml | 34 ++++++++ tezt/lib_tezos/dal_common.mli | 5 ++ tezt/tests/dal.ml | 151 +++++++++++++++++++++++++++++++--- 3 files changed, 180 insertions(+), 10 deletions(-) diff --git a/tezt/lib_tezos/dal_common.ml b/tezt/lib_tezos/dal_common.ml index 9d494158d27c..bbd49dc1196d 100644 --- a/tezt/lib_tezos/dal_common.ml +++ b/tezt/lib_tezos/dal_common.ml @@ -80,6 +80,40 @@ module Parameters = struct Client.RPC.call client @@ RPC.get_chain_block_context_constants () in from_protocol_parameters json |> return + + let storage_period_with_refutation_in_cycles ~proto_parameters = + let blocks_per_cycle = + JSON.(proto_parameters |-> "blocks_per_cycle" |> as_int) + in + let challenge_window = + JSON.( + proto_parameters |-> "smart_rollup_challenge_window_in_blocks" |> as_int) + in + let commitment_period = + JSON.( + proto_parameters |-> "smart_rollup_commitment_period_in_blocks" + |> as_int) + in + let validity_lag = + JSON.( + proto_parameters |-> "smart_rollup_reveal_activation_level" + |-> "dal_attested_slots_validity_lag" |> as_int) + in + let blocks = 2 * (challenge_window + commitment_period + validity_lag) in + if blocks mod blocks_per_cycle = 0 then blocks / blocks_per_cycle + else 1 + (blocks / blocks_per_cycle) + + let storage_period_without_refutation_in_cycles ~proto_parameters = + let blocks_per_cycle = + JSON.(proto_parameters |-> "blocks_per_cycle" |> as_int) + in + let attestation_lag = + JSON.( + proto_parameters |-> "dal_parametric" |-> "attestation_lag" |> as_int) + in + let blocks = 2 * attestation_lag in + if blocks mod blocks_per_cycle = 0 then blocks / blocks_per_cycle + else 1 + (blocks / blocks_per_cycle) end module Committee = struct diff --git a/tezt/lib_tezos/dal_common.mli b/tezt/lib_tezos/dal_common.mli index 74362e459ca7..b3d42ed6bd24 100644 --- a/tezt/lib_tezos/dal_common.mli +++ b/tezt/lib_tezos/dal_common.mli @@ -42,6 +42,11 @@ module Parameters : sig val from_protocol_parameters : JSON.t -> t val from_client : Client.t -> t Lwt.t + + val storage_period_with_refutation_in_cycles : proto_parameters:JSON.t -> int + + val storage_period_without_refutation_in_cycles : + proto_parameters:JSON.t -> int end module Helpers : sig diff --git a/tezt/tests/dal.ml b/tezt/tests/dal.ml index 17086c63da48..7dfa66665939 100644 --- a/tezt/tests/dal.ml +++ b/tezt/tests/dal.ml @@ -181,6 +181,11 @@ let slot_size_param slot_size = make_int_parameter ["dal_parametric"; "slot_size"] slot_size (* Some initialization functions to start needed nodes. *) +type l1_history_mode = + | Default_with_refutation + | Default_without_refutation + | Custom of Node.history_mode + let generate_protocol_parameters base protocol parameter_overrides = let* parameter_file = Protocol.write_parameter_file ~base parameter_overrides @@ -188,18 +193,113 @@ let generate_protocol_parameters base protocol parameter_overrides = let* client = Client.init_mockup ~parameter_file ~protocol () in Client.RPC.call client @@ RPC.get_chain_block_context_constants () +(* Compute the L1 history mode. This function may update the protocol parameters + and this is why it needs additional, a priori unrelated parameters. *) +let history_mode base protocol parameter_overrides proto_parameters + l1_history_mode = + let update_some_rollup_params factor = + let challenge_window = + JSON.( + proto_parameters |-> "smart_rollup_challenge_window_in_blocks" |> as_int) + / factor + in + let commitment_period = + max + 1 + (JSON.( + proto_parameters |-> "smart_rollup_commitment_period_in_blocks" + |> as_int) + / factor) + in + let validity_lag = + JSON.( + proto_parameters |-> "smart_rollup_reveal_activation_level" + |-> "dal_attested_slots_validity_lag" |> as_int) + / factor + in + ( ["smart_rollup_reveal_activation_level"; "dal_attested_slots_validity_lag"], + `Int validity_lag ) + :: (["smart_rollup_challenge_window_in_blocks"], `Int challenge_window) + :: (["smart_rollup_commitment_period_in_blocks"], `Int commitment_period) + :: parameter_overrides + in + match l1_history_mode with + | Custom history_mode -> return (parameter_overrides, history_mode) + | Default_without_refutation -> + let cycles = + Dal.Parameters.storage_period_without_refutation_in_cycles + ~proto_parameters + in + let blocks_preservation_cycles = + JSON.(proto_parameters |-> "blocks_preservation_cycles" |> as_int) + in + let additional_cycles = cycles - blocks_preservation_cycles in + return (parameter_overrides, Node.Rolling (Some additional_cycles)) + | Default_with_refutation -> + let cycles = + Dal.Parameters.storage_period_with_refutation_in_cycles + ~proto_parameters + in + let blocks_preservation_cycles = + JSON.(proto_parameters |-> "blocks_preservation_cycles" |> as_int) + in + let additional_cycles = cycles - blocks_preservation_cycles in + (* The shell has an upper bound of 1000 stored cycles in Full and Rolling + mode. In case this limit is crossed, we update some of the relevant + parameters. *) + if additional_cycles > 1000 then ( + let factor = 1 + (cycles / 1000) in + let new_parameter_overrides = update_some_rollup_params factor in + (* This may not work correctly if the updated parameters were already + present in [parameters]. *) + let* proto_parameters = + generate_protocol_parameters base protocol new_parameter_overrides + in + Log.info + "The DAL node needs the L1 node to store %d cycles of block data." + cycles ; + Log.info + "Reducing 'smart_rollup_challenge_window_in_blocks', \ + 'smart_rollup_commitment_period_in_blocks' and \ + 'dal_attested_slots_validity_lag' by a factor of %d." + factor ; + let cycles = + Dal.Parameters.storage_period_with_refutation_in_cycles + ~proto_parameters + in + Log.info + "Now the DAL node needs the L1 node to store %d cycles of block data" + cycles ; + let additional_cycles = cycles - blocks_preservation_cycles in + if additional_cycles > 1000 then + Test.fail "Could not adjust sc_rollup parameters automatically!" + else + return (new_parameter_overrides, Node.Rolling (Some additional_cycles))) + else return (parameter_overrides, Node.Rolling (Some additional_cycles)) + let setup_node ?(custom_constants = None) ?(additional_bootstrap_accounts = 0) ~parameter_overrides ~protocol ?activation_timestamp ?(event_sections_levels = []) ?(node_arguments = []) - ?(dal_bootstrap_peers = []) () = + ?(dal_bootstrap_peers = []) ?(l1_history_mode = Default_without_refutation) + () = let base = Either.right (protocol, custom_constants) in let* proto_parameters = generate_protocol_parameters base protocol parameter_overrides in + let* parameter_overrides, history_mode = + history_mode + base + protocol + parameter_overrides + proto_parameters + l1_history_mode + in let nodes_args = Node. [ - Synchronisation_threshold 0; History_mode (Full None); No_bootstrap_peers; + Synchronisation_threshold 0; + No_bootstrap_peers; + History_mode history_mode; ] in let node = Node.create nodes_args in @@ -247,8 +347,8 @@ let with_layer1 ?custom_constants ?additional_bootstrap_accounts ?attestation_threshold ?number_of_shards ?redundancy_factor ?commitment_period ?challenge_window ?dal_enable ?event_sections_levels ?node_arguments ?activation_timestamp ?dal_bootstrap_peers - ?(parameters = []) ?(prover = true) ?smart_rollup_timeout_period_in_blocks f - ~protocol = + ?(parameters = []) ?(prover = true) ?smart_rollup_timeout_period_in_blocks + ?l1_history_mode f ~protocol = let parameter_overrides = make_int_parameter ["dal_parametric"; "attestation_lag"] attestation_lag @ make_int_parameter ["dal_parametric"; "number_of_shards"] number_of_shards @@ -294,6 +394,7 @@ let with_layer1 ?custom_constants ?additional_bootstrap_accounts ?node_arguments ?activation_timestamp ?dal_bootstrap_peers + ?l1_history_mode ~parameter_overrides ~protocol () @@ -414,8 +515,8 @@ let scenario_with_layer1_and_dal_nodes ?regression ?(tags = []) ?delay_increment_per_round ?redundancy_factor ?slot_size ?number_of_shards ?number_of_slots ?attestation_lag ?attestation_threshold ?commitment_period ?challenge_window ?(dal_enable = true) ?activation_timestamp - ?bootstrap_profile ?producer_profiles ?history_mode ?prover variant scenario - = + ?bootstrap_profile ?producer_profiles ?history_mode ?prover ?l1_history_mode + variant scenario = let description = "Testing DAL node" in let tags = if List.mem team tags then tags else team :: tags in let tags = @@ -429,6 +530,12 @@ let scenario_with_layer1_and_dal_nodes ?regression ?(tags = []) ~uses:(fun protocol -> Constant.octez_dal_node :: uses protocol) (Printf.sprintf "%s (%s)" description variant) (fun protocol -> + let l1_history_mode = + match (l1_history_mode, producer_profiles) with + | Some mode, _ -> mode + | None, Some (_ :: _) -> Default_with_refutation + | _ -> Default_without_refutation + in with_layer1 ~custom_constants ?minimal_block_delay @@ -443,6 +550,7 @@ let scenario_with_layer1_and_dal_nodes ?regression ?(tags = []) ?challenge_window ?activation_timestamp ?prover + ~l1_history_mode ~protocol ~dal_enable @@ fun parameters cryptobox node client -> @@ -457,7 +565,7 @@ let scenario_with_all_nodes ?custom_constants ?node_arguments ?challenge_window ?minimal_block_delay ?delay_increment_per_round ?activation_timestamp ?bootstrap_profile ?producer_profiles ?smart_rollup_timeout_period_in_blocks ?(regression = true) ?prover - ?attestation_threshold variant scenario = + ?attestation_threshold ?l1_history_mode variant scenario = let description = "Testing DAL rollup and node with L1" in let tags = if List.mem team tags then tags else team :: tags in let tags = @@ -473,6 +581,12 @@ let scenario_with_all_nodes ?custom_constants ?node_arguments :: uses protocol) (Printf.sprintf "%s (%s)" description variant) (fun protocol -> + let l1_history_mode = + match (l1_history_mode, producer_profiles) with + | Some mode, _ -> mode + | None, Some (_ :: _) -> Default_with_refutation + | _ -> Default_without_refutation + in with_layer1 ~custom_constants ?node_arguments @@ -490,6 +604,7 @@ let scenario_with_all_nodes ?custom_constants ?node_arguments ?smart_rollup_timeout_period_in_blocks ?prover ?attestation_threshold + ~l1_history_mode ~protocol ~dal_enable @@ fun parameters _cryptobox node client -> @@ -1862,7 +1977,9 @@ let test_dal_node_import_snapshot _protocol parameters _cryptobox node client let* () = bake_for client in let* export_level = Node.wait_for_level node (level + 1) in let file = Temp.file "snapshot" in - let* () = Node.snapshot_export ~export_level node file in + let* () = + Node.snapshot_export ~history_mode:Rolling_history ~export_level node file + in let node2 = Node.create [] in let* () = Node.config_init node2 [] in (* We update the configuration because by default on sandbox mode, @@ -4138,7 +4255,11 @@ let test_l1_migration_scenario ?(tags = []) ~migrate_from ~migrate_to @ make_int_parameter ["dal_parametric"; "page_size"] page_size in let* node, client, dal_parameters = - setup_node ~parameter_overrides ~protocol:migrate_from () + setup_node + ~parameter_overrides + ~protocol:migrate_from + ~l1_history_mode:Default_with_refutation + () in Log.info "Set user-activated-upgrade at level %d" migration_level ; @@ -7624,12 +7745,14 @@ let register ~protocols = ~tags:["bootstrap"; Tag.memory_3k] ~bootstrap_profile:true ~prover:false + ~l1_history_mode:Default_with_refutation "peer discovery via bootstrap node" test_peer_discovery_via_bootstrap_node protocols ; scenario_with_layer1_and_dal_nodes ~tags:["gossipsub"; "rpc"] ~bootstrap_profile:true + ~l1_history_mode:Default_with_refutation "GS/RPC get_connections" test_rpc_get_connections protocols ; @@ -7639,6 +7762,7 @@ let register ~protocols = ~bootstrap_profile:true "trusted peers reconnection" ~prover:false + ~l1_history_mode:Default_with_refutation test_peers_reconnection protocols ; scenario_with_layer1_and_dal_nodes @@ -7651,6 +7775,7 @@ let register ~protocols = ~tags:["attestation"; "p2p"; Tag.memory_3k] ~attestation_threshold:100 ~bootstrap_profile:true + ~l1_history_mode:Default_with_refutation "attestation through p2p" test_attestation_through_p2p protocols ; @@ -7663,6 +7788,7 @@ let register ~protocols = scenario_with_layer1_and_dal_nodes ~tags:["amplification"; Tag.memory_4k] ~bootstrap_profile:true + ~l1_history_mode:Default_with_refutation ~redundancy_factor:2 (* With a redundancy factor of 4 or more, not much luck is needed for a bootstrap account (with 1/5 of the stake) to be @@ -7677,13 +7803,13 @@ let register ~protocols = scenario_with_layer1_and_dal_nodes ~tags:["amplification"; "simple"; Tag.memory_4k] ~bootstrap_profile:true + ~l1_history_mode:Default_with_refutation "observer triggers amplification (without lost shards)" Amplification.test_amplification_without_lost_shards protocols ; scenario_with_layer1_and_dal_nodes ~tags:["gc"; "simple"; Tag.memory_3k] ~producer_profiles:[0] - ~history_mode:(Dal_node.Custom 150) ~number_of_slots:1 "garbage collection of shards for producer" Garbage_collection.test_gc_simple_producer @@ -7691,6 +7817,7 @@ let register ~protocols = scenario_with_layer1_and_dal_nodes ~tags:["gc"; "attester"; Tag.memory_4k] ~bootstrap_profile:true + ~l1_history_mode:Default_with_refutation ~number_of_slots:1 "garbage collection of shards for producer and attester" Garbage_collection.test_gc_producer_and_attester @@ -7698,6 +7825,7 @@ let register ~protocols = scenario_with_layer1_and_dal_nodes ~tags:["gc"; "multi"; Tag.memory_4k] ~bootstrap_profile:true + ~l1_history_mode:Default_with_refutation ~number_of_slots:1 "garbage collection of shards for all profiles" Garbage_collection.test_gc_with_all_profiles @@ -7711,6 +7839,7 @@ let register ~protocols = protocols ; scenario_with_layer1_and_dal_nodes ~bootstrap_profile:true + ~l1_history_mode:Default_with_refutation ~number_of_slots:1 "new attester attests" test_new_attester_attests @@ -7778,6 +7907,7 @@ let register ~protocols = (Refutations.scenario_with_two_rollups_a_faulty_dal_node_and_a_correct_one ~refute_operations_priority:`Faulty_first) ~smart_rollup_timeout_period_in_blocks:20 + ~l1_history_mode:Default_with_refutation ~tags:[Tag.slow] protocols ; @@ -7791,6 +7921,7 @@ let register ~protocols = (Refutations.scenario_with_two_rollups_a_faulty_dal_node_and_a_correct_one ~refute_operations_priority:`Honest_first) ~smart_rollup_timeout_period_in_blocks:20 + ~l1_history_mode:Default_with_refutation ~tags:[Tag.slow] protocols ; -- GitLab From d98168584c37b169f6e4f77aefe4ccb5ce8fae49 Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Mon, 22 Jul 2024 19:31:41 +0200 Subject: [PATCH 7/7] DAL/Tezt: remove source of flakiness --- tezt/tests/dal.ml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tezt/tests/dal.ml b/tezt/tests/dal.ml index 7dfa66665939..f50a4b58417d 100644 --- a/tezt/tests/dal.ml +++ b/tezt/tests/dal.ml @@ -1397,7 +1397,11 @@ let test_dal_node_slot_management _protocol parameters _cryptobox _node client let* () = bake_for client in let* published_level = Client.level client in (* Finalize the publication. *) + let wait_for_dal_node = + wait_for_layer1_final_block dal_node published_level + in let* () = bake_for ~count:2 client in + let* () = wait_for_dal_node in let* received_slot = Dal_RPC.( call dal_node @@ -5635,6 +5639,10 @@ module Garbage_collection = struct in Log.info "Producer DAL node is running" ; + let* current_level = Client.level client in + let wait_for_producer = + wait_for_layer1_final_block slot_producer (current_level + 1) + in let* published_level, _commitment, () = publish_store_and_wait_slot node @@ -5646,6 +5654,7 @@ module Garbage_collection = struct ~number_of_extra_blocks_to_bake:2 @@ Helpers.make_slot ~slot_size "content" in + let* () = wait_for_producer in Log.info "RPC to producer for first shard" ; (* Check the producer stored the first shard *) @@ -5803,6 +5812,13 @@ module Garbage_collection = struct wait_for_first_shard ~published_level ~slot_index attester in + let* current_level = Client.level client in + let wait_for_producer = + wait_for_layer1_final_block slot_producer (current_level + 1) + in + let wait_for_attester = + wait_for_layer1_final_block attester (current_level + 1) + in let* published_level, _commitment, shard_index_attester = publish_store_and_wait_slot node @@ -5814,6 +5830,8 @@ module Garbage_collection = struct ~number_of_extra_blocks_to_bake:2 @@ Helpers.make_slot ~slot_size "content" in + let* () = wait_for_producer in + let* () = wait_for_attester in Log.info "RPC first shard producer" ; let* _shard_producer = @@ -6042,6 +6060,16 @@ module Garbage_collection = struct return (shard_index_observer, shard_index_attester) in + let* current_level = Client.level client in + let wait_for_producer = + wait_for_layer1_final_block slot_producer (current_level + 1) + in + let wait_for_observer = + wait_for_layer1_final_block observer (current_level + 1) + in + let wait_for_attester = + wait_for_layer1_final_block attester (current_level + 1) + in let* ( published_level, _commitment, (shard_index_observer, shard_index_attester) ) = @@ -6055,6 +6083,9 @@ module Garbage_collection = struct ~number_of_extra_blocks_to_bake:2 @@ Helpers.make_slot ~slot_size "content" in + let* () = wait_for_producer in + let* () = wait_for_observer in + let* () = wait_for_attester in Log.info "RPC first shard observer" ; let* _shard_observer = -- GitLab