From 36f39debb570b9c137d3af5f1ddc59bcb2607375 Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Thu, 9 Oct 2025 06:50:15 +0200 Subject: [PATCH 1/5] DAL/Node: add published_level to skip-list-slots table and migrate db --- src/lib_dal_node/block_handler.ml | 18 +++++-- src/lib_dal_node/dal_store_sqlite3.ml | 53 ++++++++++--------- src/lib_dal_node/dal_store_sqlite3.mli | 29 +++++----- ...add_attestation_lag_to_skip_list_slots.sql | 15 ++++++ .../script-inputs/dal_store_migrations | 1 + src/lib_dal_node/slot_manager.ml | 16 +++--- src/lib_dal_node/store.ml | 17 +++--- src/lib_dal_node/store.mli | 21 ++++---- .../DAL Node- debug print store schemas.out | 15 +++--- 9 files changed, 109 insertions(+), 76 deletions(-) create mode 100644 src/lib_dal_node/migrations/001_add_attestation_lag_to_skip_list_slots.sql diff --git a/src/lib_dal_node/block_handler.ml b/src/lib_dal_node/block_handler.ml index 3f9b0b5a84cf..11533f2db141 100644 --- a/src/lib_dal_node/block_handler.ml +++ b/src/lib_dal_node/block_handler.ml @@ -74,9 +74,13 @@ let remove_old_level_stored_data proto_parameters ctxt current_level = (* TODO: https://gitlab.com/tezos/tezos/-/issues/7258 We may want to remove this check. *) if Node_context.supports_refutations ctxt then - let* res = - Store.Skip_list_cells.remove store ~attested_level:oldest_level + let published_level = + Int32.( + sub + oldest_level + (of_int proto_parameters.Types.attestation_lag)) in + let* res = Store.Skip_list_cells.remove store ~published_level in match res with | Ok () -> Event.emit_removed_skip_list_cells ~level:oldest_level | Error error -> @@ -194,7 +198,15 @@ let store_skip_list_cells ctxt cctxt dal_constants ~attested_level cells_of_level in let store = Node_context.get_store ctxt in - Store.Skip_list_cells.insert store ~attested_level cells_of_level + (* DAL/FIXME: use the lag and published_level returned by the cells_of_level + in https://gitlab.com/tezos/tezos/-/merge_requests/19512 *) + let attestation_lag = dal_constants.Types.attestation_lag in + let published_level = Int32.(sub attested_level (of_int attestation_lag)) in + Store.Skip_list_cells.insert + store + ~published_level + ~attestation_lag + cells_of_level (* This functions counts, for each slot, the number of shards attested by the bakers. *) let attested_shards_per_slot attestations slot_to_committee ~number_of_slots diff --git a/src/lib_dal_node/dal_store_sqlite3.ml b/src/lib_dal_node/dal_store_sqlite3.ml index b9960b20ea6a..279e36223e3d 100644 --- a/src/lib_dal_node/dal_store_sqlite3.ml +++ b/src/lib_dal_node/dal_store_sqlite3.ml @@ -67,7 +67,7 @@ module Q = struct You can review the result at [tezt/tests/expected/dal.ml/DAL Node- debug print store schemas.out]. *) - let version = 2 + let version = 1 let all : Dal_node_migrations.migration list = Dal_node_migrations.migrations version @@ -138,7 +138,9 @@ module Types = struct open Tezos_dal_node_services.Types - let attested_level : level Caqti_type.t = int32 + let published_level : level Caqti_type.t = int32 + + let attestation_lag : int Caqti_type.t = int16 let dal_slot_index : slot_index Caqti_type.t = int16 @@ -176,14 +178,14 @@ module Skip_list_cells = struct let find_by_slot_id_opt : (level * slot_index, Skip_list_cell.t, [`One | `Zero]) t = let open Caqti_type.Std in - (t2 attested_level dal_slot_index ->? skip_list_cell) + (t2 published_level dal_slot_index ->? skip_list_cell) @@ {sql| SELECT cell FROM skip_list_cells WHERE hash = ( SELECT skip_list_cell_hash FROM skip_list_slots - WHERE attested_level = $1 AND slot_index = $2 + WHERE published_level = $1 AND slot_index = $2 )|sql} let find_by_level : @@ -192,24 +194,27 @@ module Skip_list_cells = struct [`Zero | `One | `Many] ) t = let open Caqti_type.Std in - (attested_level ->* t3 skip_list_cell skip_list_hash int) + (published_level ->* t3 skip_list_cell skip_list_hash int) @@ {sql| SELECT c.cell, c.hash, c.slot_index FROM skip_list_cells AS c JOIN skip_list_slots AS s ON s.skip_list_cell_hash = c.hash - WHERE s.attested_level = $1 + WHERE s.published_level = $1 ORDER BY s.slot_index DESC |sql} let insert_skip_list_slot : - (level * slot_index * Skip_list_hash.t, unit, [`Zero]) t = - (t3 attested_level dal_slot_index skip_list_hash ->. unit) + (level * slot_index * int * Skip_list_hash.t, unit, [`Zero]) t = + (t4 published_level dal_slot_index attestation_lag skip_list_hash ->. unit) @@ {sql| INSERT INTO skip_list_slots - (attested_level, slot_index, skip_list_cell_hash) - VALUES ($1, $2, $3) - ON CONFLICT(attested_level, slot_index) DO UPDATE SET skip_list_cell_hash = $3 + (published_level, slot_index, attestation_lag, skip_list_cell_hash) + VALUES ($1, $2, $3, $4) + ON CONFLICT(published_level, slot_index) + DO UPDATE SET + attestation_lag = excluded.attestation_lag, + skip_list_cell_hash = excluded.skip_list_cell_hash |sql} let insert_skip_list_cell : @@ -223,20 +228,20 @@ module Skip_list_cells = struct |sql} let delete_skip_list_cell : (level, unit, [`Zero]) t = - (attested_level ->. unit) + (published_level ->. unit) @@ {sql| DELETE FROM skip_list_cells WHERE hash IN ( SELECT skip_list_cell_hash FROM skip_list_slots - WHERE attested_level = $1) + WHERE published_level = $1) |sql} let delete_skip_list_slot : (level, unit, [`Zero]) t = - (attested_level ->. unit) + (published_level ->. unit) @@ {sql| DELETE FROM skip_list_slots - WHERE attested_level = $1 + WHERE published_level = $1 |sql} end @@ -244,22 +249,22 @@ module Skip_list_cells = struct with_connection store conn @@ fun conn -> Sqlite.Db.find_opt conn Q.find_opt skip_list_hash - let find_by_slot_id_opt ?conn store ~attested_level ~slot_index = + let find_by_slot_id_opt ?conn store ~published_level ~slot_index = with_connection store conn @@ fun conn -> - Sqlite.Db.find_opt conn Q.find_by_slot_id_opt (attested_level, slot_index) + Sqlite.Db.find_opt conn Q.find_by_slot_id_opt (published_level, slot_index) - let find_by_level ?conn store ~attested_level = + let find_by_level ?conn store ~published_level = with_connection store conn @@ fun conn -> - Sqlite.Db.collect_list conn Q.find_by_level attested_level + Sqlite.Db.collect_list conn Q.find_by_level published_level - let remove ?conn store ~attested_level = + let remove ?conn store ~published_level = let open Lwt_result_syntax in with_connection store conn @@ fun conn -> - let* () = Sqlite.Db.exec conn Q.delete_skip_list_cell attested_level in - let* () = Sqlite.Db.exec conn Q.delete_skip_list_slot attested_level in + let* () = Sqlite.Db.exec conn Q.delete_skip_list_cell published_level in + let* () = Sqlite.Db.exec conn Q.delete_skip_list_slot published_level in return_unit - let insert ?conn store ~attested_level items = + let insert ?conn store ~published_level ~attestation_lag items = let open Lwt_result_syntax in with_connection store conn @@ fun conn -> List.iter_es @@ -268,7 +273,7 @@ module Skip_list_cells = struct Sqlite.Db.exec conn Q.insert_skip_list_slot - (attested_level, slot_index, cell_hash) + (published_level, slot_index, attestation_lag, cell_hash) in Sqlite.Db.exec conn Q.insert_skip_list_cell (cell_hash, cell)) items diff --git a/src/lib_dal_node/dal_store_sqlite3.mli b/src/lib_dal_node/dal_store_sqlite3.mli index 544f4c25754d..8fc444e72d5e 100644 --- a/src/lib_dal_node/dal_store_sqlite3.mli +++ b/src/lib_dal_node/dal_store_sqlite3.mli @@ -48,23 +48,23 @@ module Skip_list_cells : sig Skip_list_hash.t -> Skip_list_cell.t option tzresult Lwt.t - (** [find_by_slot_id_opt ?conn store ~attested_level ~slot_index] returns the cell - associated to ([attested_level], [slot_index]) in the [store], if + (** [find_by_slot_id_opt ?conn store ~published_level ~slot_index] returns the cell + associated to ([published_level], [slot_index]) in the [store], if any. Uses the [conn] if provided (defaults to [None]). *) val find_by_slot_id_opt : ?conn:conn -> t -> - attested_level:int32 -> + published_level:int32 -> slot_index:Types.slot_index -> Skip_list_cell.t option tzresult Lwt.t - (** [find_by_level ?conn store ~attested_level] retrieves the tuples (cell * - hash * slot_index) for the given attested level, if any. The results are + (** [find_by_level ?conn store ~published_level] retrieves the tuples (cell * + hash * slot_index) for the given published level, if any. The results are sorted in decreasing order w.r.t. slot indices. *) val find_by_level : ?conn:conn -> t -> - attested_level:int32 -> + published_level:int32 -> (Dal_proto_types.Skip_list_cell.t * Dal_proto_types.Skip_list_hash.t * Types.slot_index) @@ -72,21 +72,22 @@ module Skip_list_cells : sig tzresult Lwt.t - (** [insert ?conn store ~attested_level values] inserts the given - list of [values] associated to the given [attested_level] in the - [store]. Any existing value is overridden. Uses the [conn] if - provided (defaults to [None]). *) + (** [insert ?conn store ~published_level ~attestation_lag values] inserts the + given list of [values] associated to the given [published_level] in the + [store]. Any existing value is overridden. Uses the [conn] if provided + (defaults to [None]). *) val insert : ?conn:conn -> t -> - attested_level:int32 -> + published_level:int32 -> + attestation_lag:int -> (Skip_list_hash.t * Skip_list_cell.t * Types.slot_index) list -> unit tzresult Lwt.t - (** [remove ?conn store ~attested_level] removes any data related to - [attested_level] from the [store]. Uses the [conn] if provided + (** [remove ?conn store ~published_level] removes any data related to + [published_level] from the [store]. Uses the [conn] if provided (defaults to [None]). *) - val remove : ?conn:conn -> t -> attested_level:int32 -> unit tzresult Lwt.t + val remove : ?conn:conn -> t -> published_level:int32 -> unit tzresult Lwt.t (** [schemas t] returns the list of SQL statements allowing to recreate the tables of the store [t]. *) diff --git a/src/lib_dal_node/migrations/001_add_attestation_lag_to_skip_list_slots.sql b/src/lib_dal_node/migrations/001_add_attestation_lag_to_skip_list_slots.sql new file mode 100644 index 000000000000..2ec8b1109538 --- /dev/null +++ b/src/lib_dal_node/migrations/001_add_attestation_lag_to_skip_list_slots.sql @@ -0,0 +1,15 @@ +CREATE TABLE skip_list_slots_new ( + published_level INTEGER NOT NULL, + slot_index INTEGER NOT NULL, + attestation_lag INTEGER NOT NULL, + skip_list_cell_hash VARCHAR NOT NULL, + PRIMARY KEY(published_level, slot_index) +); + +INSERT INTO skip_list_slots_new (published_level, slot_index, attestation_lag, skip_list_cell_hash) +SELECT attested_level - 8, slot_index, 8, skip_list_cell_hash +FROM skip_list_slots; + +DROP TABLE skip_list_slots; + +ALTER TABLE skip_list_slots_new RENAME TO skip_list_slots; diff --git a/src/lib_dal_node/script-inputs/dal_store_migrations b/src/lib_dal_node/script-inputs/dal_store_migrations index 86aedc60d406..bfb7ad14c0b9 100644 --- a/src/lib_dal_node/script-inputs/dal_store_migrations +++ b/src/lib_dal_node/script-inputs/dal_store_migrations @@ -1 +1,2 @@ 7655cb4123007e77d2593aef3119382acb0f979665f2f25d00723f3591dcecde src/lib_dal_node/migrations/000_initial.sql +f056656620e911f4f550059ac92ef1d97b9e62136156e9e04bb73fe1c448e68a src/lib_dal_node/migrations/001_add_attestation_lag_to_skip_list_slots.sql diff --git a/src/lib_dal_node/slot_manager.ml b/src/lib_dal_node/slot_manager.ml index d9c382ff319c..7daa5582ffb7 100644 --- a/src/lib_dal_node/slot_manager.ml +++ b/src/lib_dal_node/slot_manager.ml @@ -356,20 +356,19 @@ let try_get_commitment_of_slot_id_from_memory ctxt slot_id = skip list cell stored in the SQLite store. Steps: - - Fetch the skip list cell for the given [attested_level] and slot index from - the SQLite store. + - Fetch the skip list cell for the given [slot_id] from the SQLite store. - Decode the cell using the DAL plugin. - Return the extracted slot header, if available. Returns [None] if the cell is not found in the store. *) let try_get_slot_header_from_indexed_skip_list (module Plugin : Dal_plugin.T) - ctxt ~attested_level slot_id = + ctxt slot_id = let open Lwt_result_syntax in let* cell_bytes_opt = Store.Skip_list_cells.find_by_slot_id_opt (Node_context.get_store ctxt) - ~attested_level - ~slot_index:slot_id.Types.Slot_id.slot_index + ~published_level:slot_id.Types.Slot_id.slot_level + ~slot_index:slot_id.slot_index in match cell_bytes_opt with | None -> return_none @@ -435,15 +434,12 @@ let try_get_commitment_of_slot_id_from_skip_list dal_plugin ctxt dal_constants let open Lwt_result_syntax in let*! published_slot_header_opt = let*! from_sqlite = - try_get_slot_header_from_indexed_skip_list - dal_plugin - ctxt - ~attested_level - slot_id + try_get_slot_header_from_indexed_skip_list dal_plugin ctxt slot_id in match from_sqlite with | Ok (Some _header as res) -> return res | _ -> + (* TODO: the attested level is wrong! *) try_get_slot_header_from_L1_skip_list dal_plugin ctxt diff --git a/src/lib_dal_node/store.ml b/src/lib_dal_node/store.ml index a82322363fb2..981a292b41a1 100644 --- a/src/lib_dal_node/store.ml +++ b/src/lib_dal_node/store.ml @@ -907,31 +907,32 @@ module Skip_list_cells = struct t.skip_list_cells_store skip_list_hash - let find_by_slot_id_opt ?conn t ~attested_level ~slot_index = + let find_by_slot_id_opt ?conn t ~published_level ~slot_index = Dal_store_sqlite3.Skip_list_cells.find_by_slot_id_opt ?conn t.skip_list_cells_store - ~attested_level + ~published_level ~slot_index - let find_by_level ?conn t ~attested_level = + let find_by_level ?conn t ~published_level = Dal_store_sqlite3.Skip_list_cells.find_by_level ?conn t.skip_list_cells_store - ~attested_level + ~published_level - let insert ?conn t ~attested_level items = + let insert ?conn t ~published_level ~attestation_lag items = Dal_store_sqlite3.Skip_list_cells.insert ?conn t.skip_list_cells_store - ~attested_level + ~published_level + ~attestation_lag items - let remove ?conn t ~attested_level = + let remove ?conn t ~published_level = Dal_store_sqlite3.Skip_list_cells.remove ?conn t.skip_list_cells_store - ~attested_level + ~published_level let schemas data_dir = let open Lwt_result_syntax in diff --git a/src/lib_dal_node/store.mli b/src/lib_dal_node/store.mli index a9c4f9b4ff83..03792d4845fd 100644 --- a/src/lib_dal_node/store.mli +++ b/src/lib_dal_node/store.mli @@ -255,13 +255,13 @@ module Skip_list_cells : sig Skip_list_hash.t -> Skip_list_cell.t option tzresult Lwt.t - (** [find_by_slot_id_opt ?conn store ~attested_level ~slot_index] returns the - cell associated to ([attested_level], [slot_index]) in the [store], if + (** [find_by_slot_id_opt ?conn store ~published_level ~slot_index] returns the + cell associated to ([published_level], [slot_index]) in the [store], if any. *) val find_by_slot_id_opt : ?conn:Sqlite.conn -> t -> - attested_level:int32 -> + published_level:int32 -> slot_index:Types.slot_index -> Dal_proto_types.Skip_list_cell.t option tzresult Lwt.t @@ -269,7 +269,7 @@ module Skip_list_cells : sig val find_by_level : ?conn:Sqlite.conn -> t -> - attested_level:int32 -> + published_level:int32 -> (Dal_proto_types.Skip_list_cell.t * Dal_proto_types.Skip_list_hash.t * Types.slot_index) @@ -277,22 +277,23 @@ module Skip_list_cells : sig tzresult Lwt.t - (** [insert ?conn store ~attested_level values] inserts the given list of [values] - associated to the given [attested_level] in the [store]. Any existing value - is overridden. *) + (** [insert ?conn store ~published_level ~attestation_lag values] inserts the + given list of [values] associated to the given [published_level] in the + [store]. Any existing value is overridden. *) val insert : ?conn:Dal_store_sqlite3.conn -> t -> - attested_level:int32 -> + published_level:int32 -> + attestation_lag:int -> (Skip_list_hash.t * Skip_list_cell.t * Types.slot_index) list -> unit tzresult Lwt.t - (** [remove ?conn store ~attested_level] removes any data related to [attested_level] + (** [remove ?conn store ~published_level] removes any data related to [published_level] from the [store]. *) val remove : ?conn:Dal_store_sqlite3.conn -> t -> - attested_level:int32 -> + published_level:int32 -> unit tzresult Lwt.t (** [schemas data_dir] returns the list of SQL statements allowing diff --git a/tezt/tests/expected/dal.ml/DAL Node- debug print store schemas.out b/tezt/tests/expected/dal.ml/DAL Node- debug print store schemas.out index 8b8347eba8f5..f1478c55a357 100644 --- a/tezt/tests/expected/dal.ml/DAL Node- debug print store schemas.out +++ b/tezt/tests/expected/dal.ml/DAL Node- debug print store schemas.out @@ -1,13 +1,14 @@ ./octez-dal-node debug print store schemas -CREATE TABLE skip_list_slots ( - attested_level INTEGER NOT NULL, - slot_index INTEGER NOT NULL, - skip_list_cell_hash VARCHAR NOT NULL, - PRIMARY KEY(attested_level, slot_index) -); - CREATE TABLE skip_list_cells ( hash VARCHAR NOT NULL PRIMARY KEY, cell VARCHAR NOT NULL +); + +CREATE TABLE "skip_list_slots" ( + published_level INTEGER NOT NULL, + slot_index INTEGER NOT NULL, + attestation_lag INTEGER NOT NULL, + skip_list_cell_hash VARCHAR NOT NULL, + PRIMARY KEY(published_level, slot_index) ) -- GitLab From 45dbc7e33eff452d88d0437a04609c424c98c8c6 Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Tue, 14 Oct 2025 19:36:56 +0200 Subject: [PATCH 2/5] DAL/Node: use the slot_id directly in `Skip_list_cells.find_by_slot_id_opt` --- src/lib_dal_node/dal_store_sqlite3.ml | 8 ++++++-- src/lib_dal_node/dal_store_sqlite3.mli | 12 ++++-------- src/lib_dal_node/slot_manager.ml | 3 +-- src/lib_dal_node/store.ml | 5 ++--- src/lib_dal_node/store.mli | 8 +++----- 5 files changed, 16 insertions(+), 20 deletions(-) diff --git a/src/lib_dal_node/dal_store_sqlite3.ml b/src/lib_dal_node/dal_store_sqlite3.ml index 279e36223e3d..420a9bd4b515 100644 --- a/src/lib_dal_node/dal_store_sqlite3.ml +++ b/src/lib_dal_node/dal_store_sqlite3.ml @@ -249,9 +249,13 @@ module Skip_list_cells = struct with_connection store conn @@ fun conn -> Sqlite.Db.find_opt conn Q.find_opt skip_list_hash - let find_by_slot_id_opt ?conn store ~published_level ~slot_index = + let find_by_slot_id_opt ?conn store slot_id = with_connection store conn @@ fun conn -> - Sqlite.Db.find_opt conn Q.find_by_slot_id_opt (published_level, slot_index) + Sqlite.Db.find_opt + conn + Q.find_by_slot_id_opt + ( slot_id.Tezos_dal_node_services.Types.Slot_id.slot_level, + slot_id.slot_index ) let find_by_level ?conn store ~published_level = with_connection store conn @@ fun conn -> diff --git a/src/lib_dal_node/dal_store_sqlite3.mli b/src/lib_dal_node/dal_store_sqlite3.mli index 8fc444e72d5e..91c7c722e9fa 100644 --- a/src/lib_dal_node/dal_store_sqlite3.mli +++ b/src/lib_dal_node/dal_store_sqlite3.mli @@ -48,15 +48,11 @@ module Skip_list_cells : sig Skip_list_hash.t -> Skip_list_cell.t option tzresult Lwt.t - (** [find_by_slot_id_opt ?conn store ~published_level ~slot_index] returns the cell - associated to ([published_level], [slot_index]) in the [store], if - any. Uses the [conn] if provided (defaults to [None]). *) + (** [find_by_slot_id_opt ?conn store ~slot_id] returns the cell associated to + ([slot_id]) in the [store], if any. Uses the [conn] if provided (defaults + to [None]). *) val find_by_slot_id_opt : - ?conn:conn -> - t -> - published_level:int32 -> - slot_index:Types.slot_index -> - Skip_list_cell.t option tzresult Lwt.t + ?conn:conn -> t -> Types.slot_id -> Skip_list_cell.t option tzresult Lwt.t (** [find_by_level ?conn store ~published_level] retrieves the tuples (cell * hash * slot_index) for the given published level, if any. The results are diff --git a/src/lib_dal_node/slot_manager.ml b/src/lib_dal_node/slot_manager.ml index 7daa5582ffb7..2178e1b73afb 100644 --- a/src/lib_dal_node/slot_manager.ml +++ b/src/lib_dal_node/slot_manager.ml @@ -367,8 +367,7 @@ let try_get_slot_header_from_indexed_skip_list (module Plugin : Dal_plugin.T) let* cell_bytes_opt = Store.Skip_list_cells.find_by_slot_id_opt (Node_context.get_store ctxt) - ~published_level:slot_id.Types.Slot_id.slot_level - ~slot_index:slot_id.slot_index + slot_id in match cell_bytes_opt with | None -> return_none diff --git a/src/lib_dal_node/store.ml b/src/lib_dal_node/store.ml index 981a292b41a1..8d669694f741 100644 --- a/src/lib_dal_node/store.ml +++ b/src/lib_dal_node/store.ml @@ -907,12 +907,11 @@ module Skip_list_cells = struct t.skip_list_cells_store skip_list_hash - let find_by_slot_id_opt ?conn t ~published_level ~slot_index = + let find_by_slot_id_opt ?conn t slot_id = Dal_store_sqlite3.Skip_list_cells.find_by_slot_id_opt ?conn t.skip_list_cells_store - ~published_level - ~slot_index + slot_id let find_by_level ?conn t ~published_level = Dal_store_sqlite3.Skip_list_cells.find_by_level diff --git a/src/lib_dal_node/store.mli b/src/lib_dal_node/store.mli index 03792d4845fd..9033be40339d 100644 --- a/src/lib_dal_node/store.mli +++ b/src/lib_dal_node/store.mli @@ -255,14 +255,12 @@ module Skip_list_cells : sig Skip_list_hash.t -> Skip_list_cell.t option tzresult Lwt.t - (** [find_by_slot_id_opt ?conn store ~published_level ~slot_index] returns the - cell associated to ([published_level], [slot_index]) in the [store], if - any. *) + (** [find_by_slot_id_opt ?conn store slot_id] returns the cell associated to + ([slot_id.slot_level], [slot_id.slot_index]) in the [store], if any. *) val find_by_slot_id_opt : ?conn:Sqlite.conn -> t -> - published_level:int32 -> - slot_index:Types.slot_index -> + Types.slot_id -> Dal_proto_types.Skip_list_cell.t option tzresult Lwt.t (** See {!Dal_store_sqlite3.Skip_list_cells.find_by_level}. *) -- GitLab From 815e43485edc0a53520ec25dd66a839af511ed8f Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Wed, 15 Oct 2025 06:43:47 +0200 Subject: [PATCH 3/5] DAL/Tests: adapt tests after skip-list db migration --- tezt/tests/dal.ml | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/tezt/tests/dal.ml b/tezt/tests/dal.ml index 38b188552bf2..2eb6674c9bc6 100644 --- a/tezt/tests/dal.ml +++ b/tezt/tests/dal.ml @@ -94,13 +94,13 @@ let check_skip_list_store dal_node ~number_of_slots ~expected_levels = sf "%s/store/skip_list_store/store.sqlite" (Dal_node.data_dir dal_node) in let db = Sqlite3.db_open path in - let attested_levels = ref [] in - let query = "SELECT attested_level FROM skip_list_slots;" in + let published_levels = ref [] in + let query = "SELECT published_level FROM skip_list_slots;" in let res = Sqlite3.exec_not_null_no_headers ~cb:(fun row -> - let attested_level = row.(0) in - attested_levels := attested_level :: !attested_levels) + let published_level = row.(0) in + published_levels := published_level :: !published_levels) db query in @@ -113,7 +113,7 @@ let check_skip_list_store dal_node ~number_of_slots ~expected_levels = (Sqlite3.Rc.to_string err) in Check.( - List.sort_uniq String.compare !attested_levels + List.sort_uniq String.compare !published_levels = List.sort String.compare expected_levels) ~__LOC__ Check.(list string) @@ -4861,10 +4861,9 @@ let test_restart_dal_node _protocol dal_parameters _cryptobox node client let* () = wait_for_dal_node in if profile <> Dal_RPC.Bootstrap then let expected_levels = - let offset = dal_parameters.attestation_lag + 1 in List.init - (last_finalized_level - offset + 1) - (fun i -> string_of_int (offset + i)) + (last_finalized_level - dal_parameters.attestation_lag) + (fun i -> string_of_int (i + 1)) in check_skip_list_store dal_node @@ -7191,7 +7190,7 @@ module Garbage_collection = struct check_skip_list_store dal_node ~number_of_slots - ~expected_levels:[string_of_int (lag + 1)] + ~expected_levels:[string_of_int 1] in (* We just want to observe the GC taking effect, so we need to bake at least [non_gc_period] blocks. *) @@ -7209,10 +7208,9 @@ module Garbage_collection = struct = 9, and it injects at level 19. So we have cells for level 10 to 19, that is, 10 levels. *) let expected_levels = - (* The first level with stored cells is [last_final_level - - non_gc_period + 1 = lag + 2]. *) - let offset = lag + 2 in - List.init non_gc_period (fun i -> string_of_int (offset + i)) + (* The first published level with stored cells is [last_final_level - + non_gc_period + 1 - lag = 2]. *) + List.init non_gc_period (fun i -> string_of_int (i + 2)) in check_skip_list_store dal_node ~number_of_slots ~expected_levels) protocols -- GitLab From 2e835b206f8dcaaf09969971934ab670ca70164e Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Tue, 14 Oct 2025 20:48:01 +0200 Subject: [PATCH 4/5] DAL/Node: disable fetching slot id from L1 context for backup slot retrieval That's because this is broken around migration and needs further investigation/changes. Filed #8075 concerning this. --- src/lib_dal_node/slot_manager.ml | 52 ++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/src/lib_dal_node/slot_manager.ml b/src/lib_dal_node/slot_manager.ml index 2178e1b73afb..8cafb68d97e8 100644 --- a/src/lib_dal_node/slot_manager.ml +++ b/src/lib_dal_node/slot_manager.ml @@ -384,7 +384,7 @@ let try_get_slot_header_from_indexed_skip_list (module Plugin : Dal_plugin.T) - Retrieve the skip list cells for [attested_level] using the plugin from L1. - Locate the one matching the [slot_index] of [slot_id]. - Extract and return the slot header via the plugin. *) -let try_get_slot_header_from_L1_skip_list (module Plugin : Dal_plugin.T) ctxt +let _try_get_slot_header_from_L1_skip_list (module Plugin : Dal_plugin.T) ctxt ~dal_constants ~attested_level slot_id = let open Lwt_result_syntax in let Types.Slot_id.{slot_level; slot_index} = slot_id in @@ -423,13 +423,12 @@ let try_get_slot_header_from_L1_skip_list (module Plugin : Dal_plugin.T) ctxt Retrieval order: 1. Attempt to get the header from the local SQLite skip list. - 2. If not found, fall back to fetching it from the L1 skip list context. + 2. ~~If not found, fall back to fetching it from the L1 skip list context.~~ Returns [Some commitment] if found and matches the [slot_id], [None] otherwise. Performs assertions to ensure the returned slot header matches the requested [slot_id]. *) -let try_get_commitment_of_slot_id_from_skip_list dal_plugin ctxt dal_constants - slot_id ~attested_level = +let try_get_commitment_of_slot_id_from_skip_list dal_plugin ctxt slot_id = let open Lwt_result_syntax in let*! published_slot_header_opt = let*! from_sqlite = @@ -438,13 +437,19 @@ let try_get_commitment_of_slot_id_from_skip_list dal_plugin ctxt dal_constants match from_sqlite with | Ok (Some _header as res) -> return res | _ -> - (* TODO: the attested level is wrong! *) - try_get_slot_header_from_L1_skip_list - dal_plugin - ctxt - ~dal_constants - ~attested_level - slot_id + (* FIXME: https://gitlab.com/tezos/tezos/-/issues/8075 + The attested level is wrong around migration! + Example: If [M] is the migration level, for [published_level = M-2], + [attested_lag = 8] in P1, and [attested_lag = 5] in P2, we get + [attested_level = M+6], but at this level skip-list cells for + [published_level = M+1] are found in the L1 context. *) + (* try_get_slot_header_from_L1_skip_list *) + (* dal_plugin *) + (* ctxt *) + (* ~dal_constants *) + (* ~attested_level *) + (* slot_id *) + failwith "Failed to retrieve slot id from skip-list store" in match published_slot_header_opt with | Ok (Some Dal_plugin.{published_level; slot_index; commitment}) -> @@ -466,23 +471,26 @@ let get_commitment_from_slot_id ctxt slot_id = | Some res -> return res | None -> ( let published_level = slot_id.Types.Slot_id.slot_level in - let*? dal_plugin, dal_constants = - Node_context.get_plugin_and_parameters_for_level - ctxt - ~level:published_level + let*? parameters = + Node_context.get_proto_parameters ctxt ~level:(`Level published_level) in let attested_level = Int32.add published_level - (Int32.of_int dal_constants.Types.attestation_lag) + (Int32.of_int parameters.Types.attestation_lag) + in + (* Note: the precise [attested_level] is not important; however, it is + important that we use the right plugin to decode cells in + [try_get_slot_header_from_indexed_skip_list]. Even if the + [attestation_lag] value decreases in protocol P2 wrt P1, we are sure to + get the right plugin, because [attested_level] is computed with the + (bigger) value from P1, so we are sure [attested_level] is in P2 when + needed. *) + let*? dal_plugin = + Node_context.get_plugin_for_level ctxt ~level:attested_level in let*! res = - try_get_commitment_of_slot_id_from_skip_list - dal_plugin - ctxt - dal_constants - slot_id - ~attested_level + try_get_commitment_of_slot_id_from_skip_list dal_plugin ctxt slot_id in match res with | Ok (Some res) -> return res -- GitLab From 563489e3c1a09b78c7ab95657de9ac48a6f0c9ca Mon Sep 17 00:00:00 2001 From: Eugen Zalinescu Date: Wed, 15 Oct 2025 08:54:34 +0200 Subject: [PATCH 5/5] DAL/Node: use previous level's protocol parameters when storing skip-list cells This is because if `attested_level` is the migration level, then the protocol parameters at `attested_level` are those from the new protocol, but we'd like to use those of the old protocol, because skip-list cells were build using the previous protocol's encoding. --- src/lib_dal_node/block_handler.ml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/lib_dal_node/block_handler.ml b/src/lib_dal_node/block_handler.ml index 11533f2db141..1a1ecd50d80d 100644 --- a/src/lib_dal_node/block_handler.ml +++ b/src/lib_dal_node/block_handler.ml @@ -165,6 +165,11 @@ let may_update_topics ctxt proto_parameters ~block_level = (Node_context.get_gs_worker ctxt) committee +(* Here [Plugin] and [dal_constants] should be the ones for the predecessor of + the [attested_level]. This is because if [attested_level] is the migration + level, then skip-list cells were build using the previous protocol's encoding + and parameters; while the plugin and parameters at [attested_level] are the + ones for the new protocol (given that the migration already occurred). *) let store_skip_list_cells ctxt cctxt dal_constants ~attested_level (module Plugin : Dal_plugin.T) = let open Lwt_result_syntax in @@ -420,8 +425,8 @@ let process_commitments ctxt cctxt store proto_parameters block_level [@profiler.aggregate_s {verbosity = Notice} "publish_slot_data"])) slot_headers -let process_finalized_block_data ctxt cctxt store proto_parameters block_level - (module Plugin : Dal_plugin.T) = +let process_finalized_block_data ctxt cctxt store ~prev_proto_parameters + ~proto_parameters block_level (module Plugin : Dal_plugin.T) = let open Lwt_result_syntax in let* block_info = (Plugin.block_info @@ -435,7 +440,7 @@ let process_finalized_block_data ctxt cctxt store proto_parameters block_level store_skip_list_cells ctxt cctxt - proto_parameters + prev_proto_parameters ~attested_level:block_level (module Plugin : Dal_plugin.T) [@profiler.record_s {verbosity = Notice} "store_skip_list_cells"] @@ -448,7 +453,7 @@ let process_finalized_block_data ctxt cctxt store proto_parameters block_level let* () = (Slot_manager.update_selected_slot_headers_statuses ~block_level - ~attestation_lag:proto_parameters.attestation_lag + ~attestation_lag:proto_parameters.Types.attestation_lag ~number_of_slots:proto_parameters.number_of_slots (Plugin.is_attested dal_attestation) store @@ -514,8 +519,8 @@ let process_block ctxt cctxt l1_crawler proto_parameters finalized_shell_header let store = Node_context.get_store ctxt in let block_level = finalized_shell_header.Block_header.level in let pred_level = Int32.pred block_level in - let*? (module Plugin) = - Node_context.get_plugin_for_level ctxt ~level:pred_level + let*? (module Plugin), prev_proto_parameters = + Node_context.get_plugin_and_parameters_for_level ctxt ~level:pred_level in let* () = if proto_parameters.Types.feature_enable then @@ -525,7 +530,8 @@ let process_block ctxt cctxt l1_crawler proto_parameters finalized_shell_header ctxt cctxt store - proto_parameters + ~prev_proto_parameters + ~proto_parameters block_level (module Plugin) [@profiler.record_s {verbosity = Notice} "process_finalized_block_data"] -- GitLab