From 6603145e1b8dc3efa6b9e79eb565221575fb7be8 Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Wed, 26 Jun 2024 18:33:20 +0200 Subject: [PATCH 1/2] [POC] DAL: remember the managers who submitted DAL commitments --- .../lib_benchmarks_proto/dal_benchmarks.ml | 7 ++++++- .../lib_dal/dal_plugin_registration.ml | 4 +++- src/proto_alpha/lib_plugin/RPC.ml | 2 +- src/proto_alpha/lib_protocol/alpha_context.mli | 12 ++++++++---- src/proto_alpha/lib_protocol/apply.ml | 2 +- src/proto_alpha/lib_protocol/dal_apply.ml | 5 +++-- src/proto_alpha/lib_protocol/dal_apply.mli | 11 +++++++---- src/proto_alpha/lib_protocol/dal_slot_repr.ml | 18 +++++++++++++++--- src/proto_alpha/lib_protocol/dal_slot_repr.mli | 9 +++++++-- .../lib_protocol/dal_slot_storage.ml | 9 +++++++-- .../lib_protocol/dal_slot_storage.mli | 6 +++--- src/proto_alpha/lib_protocol/init_storage.ml | 2 ++ src/proto_alpha/lib_protocol/raw_context.ml | 5 +++-- src/proto_alpha/lib_protocol/raw_context.mli | 4 ++-- src/proto_alpha/lib_protocol/storage.ml | 5 +++-- src/proto_alpha/lib_protocol/storage.mli | 2 +- 16 files changed, 72 insertions(+), 31 deletions(-) diff --git a/src/proto_alpha/lib_benchmarks_proto/dal_benchmarks.ml b/src/proto_alpha/lib_benchmarks_proto/dal_benchmarks.ml index 82a1f5aa6a99..8e2f3b535e2f 100644 --- a/src/proto_alpha/lib_benchmarks_proto/dal_benchmarks.ml +++ b/src/proto_alpha/lib_benchmarks_proto/dal_benchmarks.ml @@ -107,7 +107,12 @@ module Publish_commitment : Benchmark.S = struct in let workload = () in let closure () = - match Dal_apply.apply_publish_commitment ctxt op with + match + Dal_apply.apply_publish_commitment + ctxt + op + ~source:Environment.Signature.Public_key_hash.zero + with | Error errs -> Format.eprintf "%a@." Environment.Error_monad.pp_trace errs ; Stdlib.failwith diff --git a/src/proto_alpha/lib_dal/dal_plugin_registration.ml b/src/proto_alpha/lib_dal/dal_plugin_registration.ml index 7c8e5bf51a6e..d80fffd99475 100644 --- a/src/proto_alpha/lib_dal/dal_plugin_registration.ml +++ b/src/proto_alpha/lib_dal/dal_plugin_registration.ml @@ -275,7 +275,9 @@ module Plugin = struct Not resilient to DAL parameters change. *) ~number_of_slots:dal_constants.number_of_slots - attested_slot_headers + (List.map + (fun ext -> ext.Dal.Slot.Header.header) + attested_slot_headers) |> Environment.wrap_tzresult in (* 7. We finally export and return the cells alongside their hashes as a diff --git a/src/proto_alpha/lib_plugin/RPC.ml b/src/proto_alpha/lib_plugin/RPC.ml index e8c0d72df73a..793e0d1d2683 100644 --- a/src/proto_alpha/lib_plugin/RPC.ml +++ b/src/proto_alpha/lib_plugin/RPC.ml @@ -3210,7 +3210,7 @@ module Dal = struct RPC_path.(path / "shards") let published_slot_headers = - let output = Data_encoding.(list Dal.Slot.Header.encoding) in + let output = Data_encoding.(list Dal.Slot.Header.extended_encoding) in RPC_service.get_service ~description:"Get the published slots headers for the given level" ~query:level_query diff --git a/src/proto_alpha/lib_protocol/alpha_context.mli b/src/proto_alpha/lib_protocol/alpha_context.mli index 7b68e75c6818..d268580a0774 100644 --- a/src/proto_alpha/lib_protocol/alpha_context.mli +++ b/src/proto_alpha/lib_protocol/alpha_context.mli @@ -2859,10 +2859,14 @@ module Dal : sig type t = {id : id; commitment : Commitment.t} + type extended = {header : t; source : Signature.public_key_hash} + val id_encoding : id Data_encoding.t val encoding : t Data_encoding.t + val extended_encoding : extended Data_encoding.t + val pp_id : Format.formatter -> id -> unit val pp : Format.formatter -> t -> unit @@ -2870,10 +2874,10 @@ module Dal : sig val equal : t -> t -> bool end - val register_slot_header : context -> Header.t -> context tzresult + val register_slot_header : context -> Header.extended -> context tzresult val find_slot_headers : - context -> Raw_level.t -> Header.t list option tzresult Lwt.t + context -> Raw_level.t -> Header.extended list option tzresult Lwt.t val finalize_current_slot_headers : context -> context Lwt.t @@ -2882,8 +2886,8 @@ module Dal : sig val compute_attested_slot_headers : is_slot_attested:(Header.t -> bool) -> - Header.t list -> - Header.t list * Attestation.t + Header.extended list -> + Header.extended list * Attestation.t end module Operations : sig diff --git a/src/proto_alpha/lib_protocol/apply.ml b/src/proto_alpha/lib_protocol/apply.ml index add25bdc4430..febfd4c26aad 100644 --- a/src/proto_alpha/lib_protocol/apply.ml +++ b/src/proto_alpha/lib_protocol/apply.ml @@ -1461,7 +1461,7 @@ let apply_manager_operation : [] ) | Dal_publish_commitment slot_header -> let*? ctxt, slot_header = - Dal_apply.apply_publish_commitment ctxt slot_header + Dal_apply.apply_publish_commitment ctxt slot_header ~source in let consumed_gas = Gas.consumed ~since:ctxt_before_op ~until:ctxt in let result = diff --git a/src/proto_alpha/lib_protocol/dal_apply.ml b/src/proto_alpha/lib_protocol/dal_apply.ml index c3c5b1aa16fc..1ad04b49a2a2 100644 --- a/src/proto_alpha/lib_protocol/dal_apply.ml +++ b/src/proto_alpha/lib_protocol/dal_apply.ml @@ -87,7 +87,7 @@ let apply_attestation ctxt attestation ~power = let validate_publish_commitment ctxt _operation = assert_dal_feature_enabled ctxt -let apply_publish_commitment ctxt operation = +let apply_publish_commitment ctxt operation ~source = let open Result_syntax in let* ctxt = Gas.consume ctxt Dal_costs.cost_Dal_publish_commitment in let number_of_slots = Dal.number_of_slots ctxt in @@ -100,7 +100,8 @@ let apply_publish_commitment ctxt operation = ~current_level operation in - let* ctxt = Dal.Slot.register_slot_header ctxt slot_header in + let extended_header = Dal.Slot.Header.{header = slot_header; source} in + let* ctxt = Dal.Slot.register_slot_header ctxt extended_header in return (ctxt, slot_header) let finalisation ctxt = diff --git a/src/proto_alpha/lib_protocol/dal_apply.mli b/src/proto_alpha/lib_protocol/dal_apply.mli index 55bd78051f1a..e97aa13dd356 100644 --- a/src/proto_alpha/lib_protocol/dal_apply.mli +++ b/src/proto_alpha/lib_protocol/dal_apply.mli @@ -57,11 +57,14 @@ val apply_attestation : t -> Dal.Attestation.t -> power:int -> t tzresult val validate_publish_commitment : t -> Dal.Operations.Publish_commitment.t -> unit tzresult -(** [apply_publish_commitment ctxt slot_header] applies the publication of - slot header [slot_header] on top of [ctxt]. Fails if the slot contains - already a slot header. *) +(** [apply_publish_commitment ctxt slot_header ~source] applies the publication + of slot header [slot_header] on top of [ctxt]. Fails if the slot contains + already a slot header. *) val apply_publish_commitment : - t -> Dal.Operations.Publish_commitment.t -> (t * Dal.Slot.Header.t) tzresult + t -> + Dal.Operations.Publish_commitment.t -> + source:public_key_hash -> + (t * Dal.Slot.Header.t) tzresult (** [finalisation ctxt] should be executed at block finalisation time. A set of slots attested at level [ctxt.current_level - lag] diff --git a/src/proto_alpha/lib_protocol/dal_slot_repr.ml b/src/proto_alpha/lib_protocol/dal_slot_repr.ml index c1d216aee885..5021943b41a9 100644 --- a/src/proto_alpha/lib_protocol/dal_slot_repr.ml +++ b/src/proto_alpha/lib_protocol/dal_slot_repr.ml @@ -63,6 +63,8 @@ module Header = struct type t = {id : id; commitment : Commitment.t} + type extended = {header : t; source : Signature.public_key_hash} + let slot_id_equal {published_level; index} s2 = Raw_level_repr.equal published_level s2.published_level && Dal_slot_index_repr.equal index s2.index @@ -100,6 +102,15 @@ module Header = struct (fun ((), x) -> x); ]) + let extended_encoding = + let open Data_encoding in + conv + (fun {header; source} -> (header, source)) + (fun (header, source) -> {header; source}) + (obj2 + (req "header" encoding) + (req "source" Signature.Public_key_hash.encoding)) + let pp_id fmt {published_level; index} = Format.fprintf fmt @@ -218,7 +229,7 @@ module Slot_market = struct module Slot_index_map = Map.Make (Dal_slot_index_repr) - type t = {length : int; slot_headers : Header.t Slot_index_map.t} + type t = {length : int; slot_headers : Header.extended Slot_index_map.t} let init ~length = if Compare.Int.(length < 0) then @@ -228,8 +239,9 @@ module Slot_market = struct let length {length; _} = length - let register t new_slot_header = + let register t new_extended_slot_header = let open Header in + let new_slot_header = new_extended_slot_header.header in if not Compare.Int.( @@ -241,7 +253,7 @@ module Slot_market = struct let update = function | None -> has_changed := true ; - Some new_slot_header + Some new_extended_slot_header | Some x -> Some x in let slot_headers = diff --git a/src/proto_alpha/lib_protocol/dal_slot_repr.mli b/src/proto_alpha/lib_protocol/dal_slot_repr.mli index 40c8f94c91f0..80cd75266753 100644 --- a/src/proto_alpha/lib_protocol/dal_slot_repr.mli +++ b/src/proto_alpha/lib_protocol/dal_slot_repr.mli @@ -93,12 +93,17 @@ module Header : sig slot's commitment. *) type t = {id : id; commitment : Commitment.t} + type extended = {header : t; source : Signature.public_key_hash} + (** encoding for values of type {!type-id}. *) val id_encoding : id Data_encoding.t (** encoding for values of type {!t}. *) val encoding : t Data_encoding.t + (** encoding for values of type {!extended}. *) + val extended_encoding : extended Data_encoding.t + (** pretty-printer for values of type {!type-id}. *) val pp_id : Format.formatter -> id -> unit @@ -192,10 +197,10 @@ module Slot_market : sig registered. Returns [Some (_, false)] otherwise. Returns [None] if the [index] is not in the interval [0;length] where [length] is the value provided to the [init] function. *) - val register : t -> Header.t -> (t * bool) option + val register : t -> Header.extended -> (t * bool) option (** [candidates t] returns a list of slot header candidates. *) - val candidates : t -> Header.t list + val candidates : t -> Header.extended list end (** This module provides an abstract data structure (type {!History.t}) that diff --git a/src/proto_alpha/lib_protocol/dal_slot_storage.ml b/src/proto_alpha/lib_protocol/dal_slot_storage.ml index 944056b4f2b3..23a4ff60ab86 100644 --- a/src/proto_alpha/lib_protocol/dal_slot_storage.ml +++ b/src/proto_alpha/lib_protocol/dal_slot_storage.ml @@ -33,9 +33,11 @@ let finalize_current_slot_headers ctxt = let compute_attested_slot_headers ~is_slot_attested seen_slot_headers = let open Dal_slot_repr in - let fold_attested_slots (rev_attested_slot_headers, attestation) slot = + let fold_attested_slots (rev_attested_slot_headers, attestation) extended_slot + = + let slot = extended_slot.Header.header in if is_slot_attested slot then - ( slot :: rev_attested_slot_headers, + ( extended_slot :: rev_attested_slot_headers, Dal_attestation_repr.commit attestation slot.Header.id.index ) else (rev_attested_slot_headers, attestation) in @@ -58,6 +60,9 @@ let update_skip_list ctxt ~confirmed_slot_headers ~level_attested ~number_of_slots = let open Lwt_result_syntax in let* slots_history = get_slot_headers_history ctxt in + let confirmed_slot_headers = + List.map (fun ext -> ext.Dal_slot_repr.Header.header) confirmed_slot_headers + in let*? slots_history = Dal_slot_repr.History.add_confirmed_slot_headers_no_cache ~number_of_slots diff --git a/src/proto_alpha/lib_protocol/dal_slot_storage.mli b/src/proto_alpha/lib_protocol/dal_slot_storage.mli index 4dc2d270fca3..8e79265d68bd 100644 --- a/src/proto_alpha/lib_protocol/dal_slot_storage.mli +++ b/src/proto_alpha/lib_protocol/dal_slot_storage.mli @@ -51,7 +51,7 @@ val find_slot_headers : Raw_context.t -> Raw_level_repr.t -> - Dal_slot_repr.Header.t list option tzresult Lwt.t + Dal_slot_repr.Header.extended list option tzresult Lwt.t (** [finalize_current_slot_headers ctxt] finalizes the current slot headers posted on this block and marks them as pending into the @@ -78,5 +78,5 @@ val get_slot_headers_history : slot headers and the corresponding bitset. *) val compute_attested_slot_headers : is_slot_attested:(Dal_slot_repr.Header.t -> bool) -> - Dal_slot_repr.Header.t list -> - Dal_slot_repr.Header.t list * Dal_attestation_repr.t + Dal_slot_repr.Header.extended list -> + Dal_slot_repr.Header.extended list * Dal_attestation_repr.t diff --git a/src/proto_alpha/lib_protocol/init_storage.ml b/src/proto_alpha/lib_protocol/init_storage.ml index eff9f1abc538..b31801241409 100644 --- a/src/proto_alpha/lib_protocol/init_storage.ml +++ b/src/proto_alpha/lib_protocol/init_storage.ml @@ -207,6 +207,8 @@ let prepare_first_block chain_id ctxt ~typecheck_smart_contract that are not slashed whereas unstake_requests are slashed. *) let*! ctxt = Pending_denunciations_storage.clear ctxt in let*! ctxt = Raw_context.remove ctxt ["last_snapshot"] in + (* FIXME/DAL: reset "slot_headers" when migrating from P to Q, as the + type changed. *) return (ctxt, []) in let* ctxt = diff --git a/src/proto_alpha/lib_protocol/raw_context.ml b/src/proto_alpha/lib_protocol/raw_context.ml index b4d6028256ba..87af8e549c97 100644 --- a/src/proto_alpha/lib_protocol/raw_context.ml +++ b/src/proto_alpha/lib_protocol/raw_context.ml @@ -1782,12 +1782,13 @@ module Dal = struct in {ctxt with back = {ctxt.back with dal_attestation_slot_accountability}} - let register_slot_header ctxt slot_header = + let register_slot_header ctxt extended_slot_header = let open Result_syntax in + let slot_header = extended_slot_header.Dal_slot_repr.Header.header in match Dal_slot_repr.Slot_market.register ctxt.back.dal_slot_fee_market - slot_header + extended_slot_header with | None -> let length = diff --git a/src/proto_alpha/lib_protocol/raw_context.mli b/src/proto_alpha/lib_protocol/raw_context.mli index 1f34b498ba21..5b8187c43f0c 100644 --- a/src/proto_alpha/lib_protocol/raw_context.mli +++ b/src/proto_alpha/lib_protocol/raw_context.mli @@ -444,11 +444,11 @@ module Dal : sig the candidate is registered. [Some (ctxt,false)] if another candidate was already registered previously. Returns an error if the slot is invalid. *) - val register_slot_header : t -> Dal_slot_repr.Header.t -> t tzresult + val register_slot_header : t -> Dal_slot_repr.Header.extended -> t tzresult (** [candidates ctxt] returns the current list of slot for which there is at least one candidate. *) - val candidates : t -> Dal_slot_repr.Header.t list + val candidates : t -> Dal_slot_repr.Header.extended list (** [is_slot_index_attested ctxt slot_index] returns [true] if the [slot_index] is declared available by the protocol. [false] diff --git a/src/proto_alpha/lib_protocol/storage.ml b/src/proto_alpha/lib_protocol/storage.ml index b1d0075db47f..354dc8560b82 100644 --- a/src/proto_alpha/lib_protocol/storage.ml +++ b/src/proto_alpha/lib_protocol/storage.ml @@ -2165,9 +2165,10 @@ module Dal = struct let name = ["slot_headers"] end) (struct - type t = Dal_slot_repr.Header.t list + type t = Dal_slot_repr.Header.extended list - let encoding = Data_encoding.(list Dal_slot_repr.Header.encoding) + let encoding = + Data_encoding.(list Dal_slot_repr.Header.extended_encoding) end) module History = diff --git a/src/proto_alpha/lib_protocol/storage.mli b/src/proto_alpha/lib_protocol/storage.mli index 12fbd3bccadd..e395dac7b827 100644 --- a/src/proto_alpha/lib_protocol/storage.mli +++ b/src/proto_alpha/lib_protocol/storage.mli @@ -1010,7 +1010,7 @@ module Dal : sig Non_iterable_indexed_data_storage with type t = Raw_context.t and type key = Raw_level_repr.t - and type value = Dal_slot_repr.Header.t list + and type value = Dal_slot_repr.Header.extended list (** This is a permanent storage for slot headers confirmed by the L1. *) module History : -- GitLab From a5021809f9a92624936f69d54bebff57f2834f45 Mon Sep 17 00:00:00 2001 From: "iguerNL@Functori" Date: Wed, 26 Jun 2024 20:16:45 +0200 Subject: [PATCH 2/2] [POC] Dal: propagate (extended) attested Dal commitments to apply to push them into SORU inbox --- src/proto_alpha/lib_protocol/alpha_context.mli | 4 +++- src/proto_alpha/lib_protocol/apply.ml | 5 ++++- src/proto_alpha/lib_protocol/dal_apply.ml | 6 +++--- src/proto_alpha/lib_protocol/dal_apply.mli | 3 ++- src/proto_alpha/lib_protocol/dal_slot_storage.ml | 8 ++++---- src/proto_alpha/lib_protocol/dal_slot_storage.mli | 4 +++- 6 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/proto_alpha/lib_protocol/alpha_context.mli b/src/proto_alpha/lib_protocol/alpha_context.mli index d268580a0774..6f2a371cc7df 100644 --- a/src/proto_alpha/lib_protocol/alpha_context.mli +++ b/src/proto_alpha/lib_protocol/alpha_context.mli @@ -2882,7 +2882,9 @@ module Dal : sig val finalize_current_slot_headers : context -> context Lwt.t val finalize_pending_slot_headers : - context -> number_of_slots:int -> (context * Attestation.t) tzresult Lwt.t + context -> + number_of_slots:int -> + (context * Attestation.t * Header.extended list) tzresult Lwt.t val compute_attested_slot_headers : is_slot_attested:(Header.t -> bool) -> diff --git a/src/proto_alpha/lib_protocol/apply.ml b/src/proto_alpha/lib_protocol/apply.ml index febfd4c26aad..f6ca4b4cd882 100644 --- a/src/proto_alpha/lib_protocol/apply.ml +++ b/src/proto_alpha/lib_protocol/apply.ml @@ -3049,7 +3049,10 @@ let finalize_application ctxt block_data_contents ~round ~predecessor_hash may_start_new_cycle ctxt in let* ctxt = Amendment.may_start_new_voting_period ctxt in - let* ctxt, dal_attestation = Dal_apply.finalisation ctxt in + (* TODO: push _attested_slot_headers into rollups' inbox *) + let* ctxt, dal_attestation, _attested_slot_headers = + Dal_apply.finalisation ctxt + in let* ctxt = Sc_rollup.Inbox.finalize_inbox_level ctxt in let balance_updates = migration_balance_updates @ baking_receipts @ cycle_end_balance_updates diff --git a/src/proto_alpha/lib_protocol/dal_apply.ml b/src/proto_alpha/lib_protocol/dal_apply.ml index 1ad04b49a2a2..43657f0a503b 100644 --- a/src/proto_alpha/lib_protocol/dal_apply.ml +++ b/src/proto_alpha/lib_protocol/dal_apply.ml @@ -108,7 +108,7 @@ let finalisation ctxt = let open Lwt_result_syntax in only_if_dal_feature_enabled ctxt - ~default:(fun ctxt -> return (ctxt, Dal.Attestation.empty)) + ~default:(fun ctxt -> return (ctxt, Dal.Attestation.empty, [])) (fun ctxt -> let*! ctxt = Dal.Slot.finalize_current_slot_headers ctxt in (* The fact that slots confirmation is done at finalization is very @@ -127,7 +127,7 @@ let finalisation ctxt = level where the game started. *) let number_of_slots = (Constants.parametric ctxt).dal.number_of_slots in - let+ ctxt, attestation = + let+ ctxt, attestation, attested_slot_headers = Dal.Slot.finalize_pending_slot_headers ctxt ~number_of_slots in - (ctxt, attestation)) + (ctxt, attestation, attested_slot_headers)) diff --git a/src/proto_alpha/lib_protocol/dal_apply.mli b/src/proto_alpha/lib_protocol/dal_apply.mli index e97aa13dd356..98fe9e40a8b1 100644 --- a/src/proto_alpha/lib_protocol/dal_apply.mli +++ b/src/proto_alpha/lib_protocol/dal_apply.mli @@ -72,4 +72,5 @@ val apply_publish_commitment : [lag] is a parametric constant specific to the data-availability layer. *) -val finalisation : t -> (t * Dal.Attestation.t) tzresult Lwt.t +val finalisation : + t -> (t * Dal.Attestation.t * Dal.Slot.Header.extended list) tzresult Lwt.t diff --git a/src/proto_alpha/lib_protocol/dal_slot_storage.ml b/src/proto_alpha/lib_protocol/dal_slot_storage.ml index 23a4ff60ab86..88d985713133 100644 --- a/src/proto_alpha/lib_protocol/dal_slot_storage.ml +++ b/src/proto_alpha/lib_protocol/dal_slot_storage.ml @@ -78,11 +78,11 @@ let finalize_pending_slot_headers ctxt ~number_of_slots = let {Level_repr.level = raw_level; _} = Raw_context.current_level ctxt in let Constants_parametric_repr.{dal; _} = Raw_context.constants ctxt in match Raw_level_repr.(sub raw_level dal.attestation_lag) with - | None -> return (ctxt, Dal_attestation_repr.empty) + | None -> return (ctxt, Dal_attestation_repr.empty, []) | Some level_attested -> let* seen_slots = find_slot_headers ctxt level_attested in let*! ctxt = Storage.Dal.Slot.Headers.remove ctxt level_attested in - let* ctxt, attestation, confirmed_slot_headers = + let* ctxt, attestation, attested_slot_headers = match seen_slots with | None -> return (ctxt, Dal_attestation_repr.empty, []) | Some seen_slots -> @@ -99,8 +99,8 @@ let finalize_pending_slot_headers ctxt ~number_of_slots = let* ctxt = update_skip_list ctxt - ~confirmed_slot_headers + ~confirmed_slot_headers:attested_slot_headers ~level_attested ~number_of_slots in - return (ctxt, attestation) + return (ctxt, attestation, attested_slot_headers) diff --git a/src/proto_alpha/lib_protocol/dal_slot_storage.mli b/src/proto_alpha/lib_protocol/dal_slot_storage.mli index 8e79265d68bd..d1e9bf29a36c 100644 --- a/src/proto_alpha/lib_protocol/dal_slot_storage.mli +++ b/src/proto_alpha/lib_protocol/dal_slot_storage.mli @@ -66,7 +66,9 @@ val finalize_current_slot_headers : Raw_context.t -> Raw_context.t Lwt.t val finalize_pending_slot_headers : Raw_context.t -> number_of_slots:int -> - (Raw_context.t * Dal_attestation_repr.t) tzresult Lwt.t + (Raw_context.t * Dal_attestation_repr.t * Dal_slot_repr.Header.extended list) + tzresult + Lwt.t (** [get_slot_headers_history ctxt] returns the current value of slots_history stored in [ctxt], or Slots_history.genesis if no value is stored yet. *) -- GitLab