From da00a8f55fa5e408b5cd821982e73cddd865811e Mon Sep 17 00:00:00 2001 From: Diane Gallois-Wong Date: Wed, 25 Sep 2024 15:28:28 +0200 Subject: [PATCH 1/4] Shell: small validation refactorisation --- src/lib_shell/block_validator_process.ml | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/lib_shell/block_validator_process.ml b/src/lib_shell/block_validator_process.ml index ffd8c45b910f..1e3fc91678a2 100644 --- a/src/lib_shell/block_validator_process.ml +++ b/src/lib_shell/block_validator_process.ml @@ -654,6 +654,19 @@ let apply_block ?(simulate = false) ?(should_validate = true) in let* metadata = Store.Block.get_block_metadata chain_store predecessor in let max_operations_ttl = Store.Block.max_operations_ttl metadata in + VP.apply_block + ~simulate + ~should_validate + validator + chain_store + ~predecessor + ~max_operations_ttl + header + operations + +let validate_block (E {validator_process = (module VP); validator}) chain_store + ~predecessor header block_hash operations = + let open Lwt_result_syntax in let* live_blocks, live_operations = Store.Chain.compute_live_blocks chain_store ~block:predecessor in @@ -664,20 +677,14 @@ let apply_block ?(simulate = false) ?(should_validate = true) block_hash operations in - VP.apply_block - ~simulate - ~should_validate + VP.validate_block validator chain_store ~predecessor - ~max_operations_ttl header + block_hash operations -let validate_block (E {validator_process = (module VP); validator}) chain_store - ~predecessor header operations = - VP.validate_block validator chain_store ~predecessor header operations - let context_garbage_collection (E {validator_process = (module VP); validator}) context_index context_hash ~gc_lockfile_path = VP.context_garbage_collection -- GitLab From 1ab516e9a3c546cc60510361e1b3f6e0d927eac6 Mon Sep 17 00:00:00 2001 From: Albin Coquereau Date: Wed, 2 Oct 2024 14:48:59 +0200 Subject: [PATCH 2/4] shell: introduce a profiler module in block_validator_process --- src/lib_shell/block_validator_process.ml | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/lib_shell/block_validator_process.ml b/src/lib_shell/block_validator_process.ml index 1e3fc91678a2..2c9bfb9f5562 100644 --- a/src/lib_shell/block_validator_process.ml +++ b/src/lib_shell/block_validator_process.ml @@ -25,6 +25,10 @@ (* *) (*****************************************************************************) +module Profiler = struct + include (val Profiler.wrap Shell_profiling.block_validator_profiler) +end + type validator_environment = { user_activated_upgrades : User_activated.upgrades; user_activated_protocol_overrides : User_activated.protocol_overrides; @@ -330,9 +334,7 @@ module Internal_validator_process = struct let report = Tezos_base.Profiler.report validator.headless in (match report with | None -> () - | Some report -> ( - try Profiler.inc Shell_profiling.block_validator_profiler report - with _ -> ())) ; + | Some report -> ( try Profiler.inc report with _ -> ())) ; return result let preapply_block validator ~chain_id ~timestamp ~protocol_data ~live_blocks @@ -434,9 +436,7 @@ module Internal_validator_process = struct let report = Tezos_base.Profiler.report validator.headless in (match report with | None -> () - | Some report -> ( - try Profiler.inc Shell_profiling.block_validator_profiler report - with _ -> ())) ; + | Some report -> ( try Profiler.inc report with _ -> ())) ; return res let context_garbage_collection _validator context_index context_hash @@ -506,9 +506,7 @@ module External_validator_process = struct let* res, report = send_request validator request in (match report with | None -> () - | Some report -> ( - try Profiler.inc Shell_profiling.block_validator_profiler report - with _ -> ())) ; + | Some report -> ( try Profiler.inc report with _ -> ())) ; return res let preapply_block validator ~chain_id ~timestamp ~protocol_data ~live_blocks @@ -560,9 +558,7 @@ module External_validator_process = struct let* res, report = send_request validator request in (match report with | None -> () - | Some report -> ( - try Profiler.inc Shell_profiling.block_validator_profiler report - with _ -> ())) ; + | Some report -> ( try Profiler.inc report with _ -> ())) ; return res let context_garbage_collection validator _index context_hash ~gc_lockfile_path -- GitLab From 441f6c1306f34cf2d8546d8f702560e8d9231c51 Mon Sep 17 00:00:00 2001 From: Albin Coquereau Date: Fri, 27 Sep 2024 10:57:56 +0200 Subject: [PATCH 3/4] shell: profile check_liveness --- src/lib_shell/block_validator_process.ml | 30 ++++++++++++++---------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/lib_shell/block_validator_process.ml b/src/lib_shell/block_validator_process.ml index 2c9bfb9f5562..aa8c15a69f50 100644 --- a/src/lib_shell/block_validator_process.ml +++ b/src/lib_shell/block_validator_process.ml @@ -648,7 +648,11 @@ let apply_block ?(simulate = false) ?(should_validate = true) is_validated (Block_validator_errors.Applying_non_validated_block block_hash)) in - let* metadata = Store.Block.get_block_metadata chain_store predecessor in + let* metadata = + (Store.Block.get_block_metadata + chain_store + predecessor [@profiler.record_s "get_predecessor_metadata"]) + in let max_operations_ttl = Store.Block.max_operations_ttl metadata in VP.apply_block ~simulate @@ -664,14 +668,16 @@ let validate_block (E {validator_process = (module VP); validator}) chain_store ~predecessor header block_hash operations = let open Lwt_result_syntax in let* live_blocks, live_operations = - Store.Chain.compute_live_blocks chain_store ~block:predecessor + (Store.Chain.compute_live_blocks + chain_store + ~block:predecessor [@profiler.record_s "compute_live_blocks"]) in let*? () = - Block_validation.check_liveness - ~live_operations - ~live_blocks - block_hash - operations + (Block_validation.check_liveness + ~live_operations + ~live_blocks + block_hash + operations [@profiler.record_f "check_liveness"]) in VP.validate_block validator @@ -683,11 +689,11 @@ let validate_block (E {validator_process = (module VP); validator}) chain_store let context_garbage_collection (E {validator_process = (module VP); validator}) context_index context_hash ~gc_lockfile_path = - VP.context_garbage_collection - validator - context_index - context_hash - ~gc_lockfile_path + (VP.context_garbage_collection + validator + context_index + context_hash + ~gc_lockfile_path [@profiler.record_s "context_garbage_collection"]) let context_split (E {validator_process = (module VP); validator}) context_index = -- GitLab From ca19426a146680238519bd8b362017d8605c40c6 Mon Sep 17 00:00:00 2001 From: Albin Coquereau Date: Wed, 2 Oct 2024 14:52:54 +0200 Subject: [PATCH 4/4] shell: profile preapply --- src/lib_shell/block_validator_process.ml | 23 ++++++++-- src/lib_validation/block_validation.ml | 55 +++++++++++++----------- src/lib_validation/external_validator.ml | 5 ++- 3 files changed, 52 insertions(+), 31 deletions(-) diff --git a/src/lib_shell/block_validator_process.ml b/src/lib_shell/block_validator_process.ml index aa8c15a69f50..0db075672eb4 100644 --- a/src/lib_shell/block_validator_process.ml +++ b/src/lib_shell/block_validator_process.ml @@ -391,6 +391,10 @@ module Internal_validator_process = struct operations in validator.preapply_result <- Some apply_result ; + let report = Tezos_base.Profiler.report validator.headless in + (match report with + | None -> () + | Some report -> ( try Profiler.inc report with _ -> ())) ; return result let validate_block validator chain_store ~predecessor header _hash operations @@ -532,7 +536,10 @@ module External_validator_process = struct operations; } in - let* res, _report = send_request validator request in + let* res, report = send_request validator request in + (match report with + | None -> () + | Some report -> ( try Profiler.inc report with _ -> ())) ; return res let validate_block validator chain_store ~predecessor header hash operations = @@ -711,7 +718,9 @@ let preapply_block (E {validator_process = (module VP); validator} : t) let open Lwt_result_syntax in let chain_id = Store.Chain.chain_id chain_store in let* live_blocks, live_operations = - Store.Chain.compute_live_blocks chain_store ~block:predecessor + (Store.Chain.compute_live_blocks + chain_store + ~block:predecessor [@profiler.record_s "compute_live_blocks"]) in let predecessor_shell_header = Store.Block.shell_header predecessor in let predecessor_hash = Store.Block.hash predecessor in @@ -721,12 +730,18 @@ let preapply_block (E {validator_process = (module VP); validator} : t) let predecessor_ops_metadata_hash = Store.Block.all_operations_metadata_hash predecessor in - let* metadata = Store.Block.get_block_metadata chain_store predecessor in + let* metadata = + (Store.Block.get_block_metadata + chain_store + predecessor [@profiler.record_s "get_block_metadata"]) + in let predecessor_max_operations_ttl = Store.Block.max_operations_ttl metadata in let* predecessor_resulting_context_hash = - Store.Block.resulting_context_hash chain_store predecessor + (Store.Block.resulting_context_hash + chain_store + predecessor [@profiler.record_s "resulting_context_hash"]) in VP.preapply_block validator diff --git a/src/lib_validation/block_validation.ml b/src/lib_validation/block_validation.ml index 1292870dfc81..822584cb93b2 100644 --- a/src/lib_validation/block_validation.ml +++ b/src/lib_validation/block_validation.ml @@ -945,14 +945,17 @@ module Make (Proto : Protocol_plugin.T) = struct in let open Lwt_result_syntax in let* validation_state = - Proto.validate_operation - ~check_signature:op.check_signature - pv.validation_state - op.hash - operation + (Proto.validate_operation + ~check_signature:op.check_signature + pv.validation_state + op.hash + operation [@profiler.record_s "validate_operation"]) in let* application_state, receipt = - Proto.apply_operation pv.application_state op.hash operation + (Proto.apply_operation + pv.application_state + op.hash + operation [@profiler.record_s "apply_operations"]) in return (validation_state, application_state, receipt)) in @@ -1055,20 +1058,20 @@ module Make (Proto : Protocol_plugin.T) = struct {predecessor_hash; timestamp; block_header_data = protocol_data} in let* validation_state = - Proto.begin_validation - context - chain_id - mode - ~predecessor:predecessor_shell_header - ~cache + (Proto.begin_validation + context + chain_id + mode + ~predecessor:predecessor_shell_header + ~cache [@profiler.record_s "begin_validation"]) in let* application_state = - Proto.begin_application - context - chain_id - mode - ~predecessor:predecessor_shell_header - ~cache + (Proto.begin_application + context + chain_id + mode + ~predecessor:predecessor_shell_header + ~cache [@profiler.record_s "begin_application"]) in let preapply_state = { @@ -1178,9 +1181,9 @@ module Make (Proto : Protocol_plugin.T) = struct in let* () = Proto.finalize_validation preapply_state.validation_state in let* validation_result, block_header_metadata = - Proto.finalize_application - preapply_state.application_state - (Some shell_header) + (Proto.finalize_application + preapply_state.application_state + (Some shell_header) [@profiler.record_s "finalize_application"]) in let*! validation_result = may_patch_protocol @@ -1257,11 +1260,11 @@ module Make (Proto : Protocol_plugin.T) = struct ({shell_header with context = header_context_hash}, validation_result_list) in let* block_metadata, ops_metadata = - compute_metadata - ~operation_metadata_size_limit - new_protocol_env_version - block_header_metadata - applied_ops_metadata + (compute_metadata + ~operation_metadata_size_limit + new_protocol_env_version + block_header_metadata + applied_ops_metadata [@profiler.record_s "compute_metadata"]) in let max_operations_ttl = max diff --git a/src/lib_validation/external_validator.ml b/src/lib_validation/external_validator.ml index 3a928f0ba4e1..e84169c8817d 100644 --- a/src/lib_validation/external_validator.ml +++ b/src/lib_validation/external_validator.ml @@ -242,6 +242,7 @@ module Processing = struct predecessor_resulting_context_hash; operations; } -> + () [@profiler.record "preapply_block"] ; let*! block_preapplication_result = let* predecessor_context = Error_monad.catch_es (fun () -> @@ -291,7 +292,9 @@ module Processing = struct Lwt.return (Ok res, Some last_preapplied_context) | Error _ as err -> Lwt.return (err, None) in - continue res cache cachable_result None + Tezos_protocol_environment.Environment_profiler.stop () ; + let report = Tezos_base.Profiler.report headless in + continue res cache cachable_result report | External_validation.Validate { chain_id; -- GitLab