From 61cd8ddf725c64276a6790e29854709310f07f1e Mon Sep 17 00:00:00 2001 From: Guillaume Genestier Date: Wed, 17 Sep 2025 19:39:54 +0200 Subject: [PATCH 1/5] DAL: allow to disable shards validation in batch mode --- src/lib_dal_node/message_validation.ml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/lib_dal_node/message_validation.ml b/src/lib_dal_node/message_validation.ml index facb572fb982..8cc59660a4c0 100644 --- a/src/lib_dal_node/message_validation.ml +++ b/src/lib_dal_node/message_validation.ml @@ -326,12 +326,18 @@ let gossipsub_batch_validation ctxt cryptobox ~head_level proto_parameters batch (List.hd batch_list) in let res = - Dal_metrics.sample_time - ~sampling_frequency: - Constants.shards_verification_sampling_frequency - ~metric_updater:Dal_metrics.update_shards_verification_time - ~to_sample:(fun () -> - Cryptobox.verify_shard_multi cryptobox commitment shards proofs) + if Node_context.get_disable_shard_validation ctxt then Ok () + else + Dal_metrics.sample_time + ~sampling_frequency: + Constants.shards_verification_sampling_frequency + ~metric_updater:Dal_metrics.update_shards_verification_time + ~to_sample:(fun () -> + Cryptobox.verify_shard_multi + cryptobox + commitment + shards + proofs) in match res with | Ok () -> -- GitLab From 90019f99d9c3f1687c105672322643ccb38844ad Mon Sep 17 00:00:00 2001 From: Guillaume Genestier Date: Wed, 17 Sep 2025 19:39:54 +0200 Subject: [PATCH 2/5] DAL: add open telemetry profiling for cryptography in batch mode --- src/lib_dal_node/message_validation.ml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lib_dal_node/message_validation.ml b/src/lib_dal_node/message_validation.ml index 8cc59660a4c0..13af6944508e 100644 --- a/src/lib_dal_node/message_validation.ml +++ b/src/lib_dal_node/message_validation.ml @@ -338,6 +338,11 @@ let gossipsub_batch_validation ctxt cryptobox ~head_level proto_parameters batch commitment shards proofs) + [@profiler.wrap_f + {driver_ids = [Opentelemetry]} + (Opentelemetry_helpers.trace_slot + ~name:"verify_shards" + Types.Slot_id.{slot_level = level; slot_index})] in match res with | Ok () -> -- GitLab From 59638af6176826bcbb9ebbca01f470dac72e7d0f Mon Sep 17 00:00:00 2001 From: Guillaume Genestier Date: Wed, 17 Sep 2025 19:39:54 +0200 Subject: [PATCH 3/5] Stdlib: Add lexicographic comparison of arrays --- src/lib_stdlib/compare.ml | 16 ++++++++++++++++ src/lib_stdlib/compare.mli | 2 ++ 2 files changed, 18 insertions(+) diff --git a/src/lib_stdlib/compare.ml b/src/lib_stdlib/compare.ml index e041882af1ff..741bd9a3bba1 100644 --- a/src/lib_stdlib/compare.ml +++ b/src/lib_stdlib/compare.ml @@ -91,6 +91,22 @@ module List (P : COMPARABLE) = Make (struct if hd <> 0 then hd else compare xs ys end) +module Array (P : COMPARABLE) = Make (struct + type t = P.t array + + let compare a b = + let len_a = Array.length a in + let len_b = Array.length b in + let rec loop i = + if i = len_a then if i = len_b then 0 else -1 + else if i = len_b then 1 + else + let c = P.compare a.(i) b.(i) in + if c <> 0 then c else loop (i + 1) + in + loop 0 +end) + module Option (P : COMPARABLE) = Make (struct type t = P.t option diff --git a/src/lib_stdlib/compare.mli b/src/lib_stdlib/compare.mli index 424b2709a472..4d851ffa7b04 100644 --- a/src/lib_stdlib/compare.mli +++ b/src/lib_stdlib/compare.mli @@ -151,6 +151,8 @@ module Q : S with type t = Q.t module List (P : COMPARABLE) : S with type t = P.t list +module Array (P : COMPARABLE) : S with type t = P.t array + module Option (P : COMPARABLE) : S with type t = P.t option module Result (Ok : COMPARABLE) (Error : COMPARABLE) : -- GitLab From eb16ca0d75d3a9ae0971cbe2457f9e42adff12ff Mon Sep 17 00:00:00 2001 From: Guillaume Genestier Date: Wed, 17 Sep 2025 19:39:55 +0200 Subject: [PATCH 4/5] DAL/Gossipsub: Make Message module COMPARABLE --- src/lib_crypto_dal/cryptobox.ml | 4 +++- src/lib_crypto_dal/cryptobox.mli | 8 ++++++-- src/lib_dal_node_services/types.ml | 10 ++++++++++ src/lib_dal_node_services/types.mli | 2 ++ src/lib_gossipsub/gossipsub_intf.ml | 14 ++++++++++++-- 5 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/lib_crypto_dal/cryptobox.ml b/src/lib_crypto_dal/cryptobox.ml index 3517949b0f05..0397dfab4a94 100644 --- a/src/lib_crypto_dal/cryptobox.ml +++ b/src/lib_crypto_dal/cryptobox.ml @@ -148,7 +148,9 @@ module Inner = struct type page = bytes - type share = Scalar.t array + module Share = Compare.Array (Scalar) + + type share = Share.t type shard = {index : int; share : share} diff --git a/src/lib_crypto_dal/cryptobox.mli b/src/lib_crypto_dal/cryptobox.mli index df89a5429615..471f975ae2ed 100644 --- a/src/lib_crypto_dal/cryptobox.mli +++ b/src/lib_crypto_dal/cryptobox.mli @@ -199,14 +199,18 @@ val string_of_commit_error : string (** A portion of the data represented by a polynomial. *) -type share +module Share : Compare.COMPARABLE + +type share = Share.t (** A shard is share with its index (see {!val:shards_from_polynomial}). *) type shard = {index : int; share : share} (** A proof that a shard belongs to some commitment. *) -type shard_proof +module Proof : Compare.COMPARABLE + +type shard_proof = Proof.t module Verifier : VERIFIER diff --git a/src/lib_dal_node_services/types.ml b/src/lib_dal_node_services/types.ml index 9970810cdc07..f76114b59f52 100644 --- a/src/lib_dal_node_services/types.ml +++ b/src/lib_dal_node_services/types.ml @@ -178,6 +178,16 @@ module Message = struct (obj2 (req "share" Cryptobox.share_encoding) (req "shard_proof" Cryptobox.shard_proof_encoding)) + + module Cmp = struct + type nonrec t = t + + let compare t1 t2 = + Compare.or_else (Cryptobox.Share.compare t1.share t2.share) (fun () -> + Cryptobox.Proof.compare t1.shard_proof t2.shard_proof) + end + + include Compare.Make (Cmp) end module Peer = struct diff --git a/src/lib_dal_node_services/types.mli b/src/lib_dal_node_services/types.mli index e2f3249afb29..4c668b155f7f 100644 --- a/src/lib_dal_node_services/types.mli +++ b/src/lib_dal_node_services/types.mli @@ -116,6 +116,8 @@ module Message : sig include PRINTABLE with type t := t include ENCODABLE with type t := t + + include COMPARABLE with type t := t end (** A peer from the point of view of gossipsub. *) diff --git a/src/lib_gossipsub/gossipsub_intf.ml b/src/lib_gossipsub/gossipsub_intf.ml index 3eebe69ca4ce..6045c6ccf61e 100644 --- a/src/lib_gossipsub/gossipsub_intf.ml +++ b/src/lib_gossipsub/gossipsub_intf.ml @@ -53,7 +53,11 @@ module type AUTOMATON_SUBCONFIG = sig end module Message : sig - include PRINTABLE + type t + + include PRINTABLE with type t := t + + include COMPARABLE with type t := t (** [valid] performs an application layer-level validity check on a message id and a message if given. @@ -564,7 +568,13 @@ module type AUTOMATON = sig end (** Module for message *) - module Message : PRINTABLE + module Message : sig + type t + + include PRINTABLE with type t := t + + include COMPARABLE with type t := t + end (** Module for time *) module Time : PRINTABLE -- GitLab From 6391f3345c6d3fee905dd5a974fbf5194199578e Mon Sep 17 00:00:00 2001 From: Guillaume Genestier Date: Wed, 17 Sep 2025 19:41:13 +0200 Subject: [PATCH 5/5] Gossipsub: Use map instead of list for batch to avoid treating twice the same shard --- src/lib_gossipsub/gossipsub_worker.ml | 31 +++++++++++++++++++++------ 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/src/lib_gossipsub/gossipsub_worker.ml b/src/lib_gossipsub/gossipsub_worker.ml index 363da1901f91..6692f0838856 100644 --- a/src/lib_gossipsub/gossipsub_worker.ml +++ b/src/lib_gossipsub/gossipsub_worker.ml @@ -270,9 +270,17 @@ module Make (C : Gossipsub_intf.WORKER_CONFIGURATION) : Some (value, t) end - type batch_state = - | Pending - | Accumulating of (GS.receive_message * Peer.Set.t) list + module ReceiveMsgMap = Map.Make (struct + type t = GS.receive_message + + let compare (m1 : t) (m2 : t) = + Compare.or_else + (Compare.or_else (Topic.compare m1.topic m2.topic) (fun () -> + Message_id.compare m1.message_id m2.message_id)) + (fun () -> Message.compare m1.message m2.message) + end) + + type batch_state = Pending | Accumulating of Peer.Set.t ReceiveMsgMap.t (** The worker's state is made of the gossipsub automaton's state, and a stream of events to process. It also has two output streams to @@ -779,7 +787,7 @@ module Make (C : Gossipsub_intf.WORKER_CONFIGURATION) : let current_batch = ref Pending in fun output time_interval events_stream -> match output with - | GS.To_include_in_batch content -> ( + | GS.To_include_in_batch (msg, peers) -> ( match !current_batch with | Pending -> let open Lwt_syntax in @@ -787,15 +795,24 @@ module Make (C : Gossipsub_intf.WORKER_CONFIGURATION) : let* () = Lwt_unix.sleep (GS.Span.to_float_s time_interval) in let batch = match !current_batch with - | Accumulating batch -> batch + | Accumulating batch -> ReceiveMsgMap.bindings batch | Pending -> [] in current_batch := Pending ; Stream.push (Process_batch batch) events_stream ; return_unit) ; - current_batch := Accumulating [content] + let content_map = ReceiveMsgMap.singleton msg peers in + current_batch := Accumulating content_map | Accumulating prev_contents -> - current_batch := Accumulating (content :: prev_contents)) + let new_contents = + ReceiveMsgMap.update + msg + (function + | None -> Some peers + | Some prev_peers -> Some (Peer.Set.inter peers prev_peers)) + prev_contents + in + current_batch := Accumulating new_contents) | _ -> () (** Handling messages received from the P2P network. *) -- GitLab