diff --git a/teztale/bin_teztale_archiver/Proxford_machine.real.ml b/teztale/bin_teztale_archiver/Proxford_machine.real.ml index b248ccfdc0449b1cb67385835112c3bb1dd436f0..f4bd381cc7b9e154903b74b89145ba4f74b8f09c 100644 --- a/teztale/bin_teztale_archiver/Proxford_machine.real.ml +++ b/teztale/bin_teztale_archiver/Proxford_machine.real.ml @@ -183,6 +183,8 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int round = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of _cctxt _level = return_nil + let raw_block_round shell_header = let wrap = Environment.wrap_tzresult in let open Result_syntax in diff --git a/teztale/bin_teztale_archiver/PsBabyM1_machine.real.ml b/teztale/bin_teztale_archiver/PsBabyM1_machine.real.ml index 3d853064c72e3823662a632263168b1fe1c28cfb..1bae8307acad82926502422c723d42ea001a02c1 100644 --- a/teztale/bin_teztale_archiver/PsBabyM1_machine.real.ml +++ b/teztale/bin_teztale_archiver/PsBabyM1_machine.real.ml @@ -158,6 +158,8 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int priority = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of _cctxt _level = return_nil + let block_round header = match Data_encoding.Binary.of_bytes diff --git a/teztale/bin_teztale_archiver/PsCARTHA_machine.real.ml b/teztale/bin_teztale_archiver/PsCARTHA_machine.real.ml index 70bed6b30514013bda5f8efa7c0435bc8d83a9f8..7a6faa08148d7d84778bf050456747f6aedbb381 100644 --- a/teztale/bin_teztale_archiver/PsCARTHA_machine.real.ml +++ b/teztale/bin_teztale_archiver/PsCARTHA_machine.real.ml @@ -158,6 +158,8 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int priority = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of _cctxt _level = return_nil + let block_round header = match Data_encoding.Binary.of_bytes diff --git a/teztale/bin_teztale_archiver/PsDELPH1_machine.real.ml b/teztale/bin_teztale_archiver/PsDELPH1_machine.real.ml index 71ce75bebe0c5ae6e377a03fd73524bd82cfc47d..1a33aa52d137effcf800ca0b5cd61df7f9b5f833 100644 --- a/teztale/bin_teztale_archiver/PsDELPH1_machine.real.ml +++ b/teztale/bin_teztale_archiver/PsDELPH1_machine.real.ml @@ -158,6 +158,8 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int priority = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of _cctxt _level = return_nil + let block_round header = match Data_encoding.Binary.of_bytes diff --git a/teztale/bin_teztale_archiver/PsFLoren_machine.real.ml b/teztale/bin_teztale_archiver/PsFLoren_machine.real.ml index 0d8862faefc83b33b8053a2ab9bf443b7715c5f9..9155f9d92168bc1f5015ce17d8b2c517965c5b6d 100644 --- a/teztale/bin_teztale_archiver/PsFLoren_machine.real.ml +++ b/teztale/bin_teztale_archiver/PsFLoren_machine.real.ml @@ -172,6 +172,8 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int priority = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of _cctxt _level = return_nil + let block_round header = match Data_encoding.Binary.of_bytes diff --git a/teztale/bin_teztale_archiver/PsParisC_machine.real.ml b/teztale/bin_teztale_archiver/PsParisC_machine.real.ml index 8ee4ebc9c5afd9bcab7ab933f1868b74de0c98f9..f19e7600df8a1d8a66937c6fb7c46cc7a2fc9841 100644 --- a/teztale/bin_teztale_archiver/PsParisC_machine.real.ml +++ b/teztale/bin_teztale_archiver/PsParisC_machine.real.ml @@ -188,6 +188,25 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int round = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of cctxt level = + let raw_level = Protocol.Alpha_context.Raw_level.of_int32_exn level in + let* shard_assignments = + Plugin.RPC.Dal.dal_shards + cctxt + (cctxt#chain, `Level level) + ~level:raw_level + () + in + return + @@ List.map + (fun Plugin.RPC.Dal.S.{delegate; indexes} -> + Data.Dal. + { + delegate = Tezos_crypto.Signature.Of_V1.public_key_hash delegate; + assigned_shard_indices = indexes; + }) + shard_assignments + let raw_block_round shell_header = let wrap = Environment.wrap_tzresult in let open Result_syntax in diff --git a/teztale/bin_teztale_archiver/PsQuebec_machine.real.ml b/teztale/bin_teztale_archiver/PsQuebec_machine.real.ml index e1aaeb4369af27770faee55b043f7a5932934882..5538c0f6f0f140dd50c0c903248c42af47aebb39 100644 --- a/teztale/bin_teztale_archiver/PsQuebec_machine.real.ml +++ b/teztale/bin_teztale_archiver/PsQuebec_machine.real.ml @@ -188,6 +188,25 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int round = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of cctxt level = + let raw_level = Protocol.Alpha_context.Raw_level.of_int32_exn level in + let* shard_assignments = + Plugin.RPC.Dal.dal_shards + cctxt + (cctxt#chain, `Level level) + ~level:raw_level + () + in + return + @@ List.map + (fun Plugin.RPC.Dal.S.{delegate; indexes} -> + Data.Dal. + { + delegate = Tezos_crypto.Signature.Of_V1.public_key_hash delegate; + assigned_shard_indices = indexes; + }) + shard_assignments + let raw_block_round shell_header = let wrap = Environment.wrap_tzresult in let open Result_syntax in diff --git a/teztale/bin_teztale_archiver/PsRiotum_machine.real.ml b/teztale/bin_teztale_archiver/PsRiotum_machine.real.ml index 1bf3f3f0a42bde7b007916b64cdb765a4bd8e624..16272dd52f668bd002ab217ea603ad2f6d3f7b49 100644 --- a/teztale/bin_teztale_archiver/PsRiotum_machine.real.ml +++ b/teztale/bin_teztale_archiver/PsRiotum_machine.real.ml @@ -188,6 +188,25 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int round = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of cctxt level = + let raw_level = Protocol.Alpha_context.Raw_level.of_int32_exn level in + let* shard_assignments = + Plugin.RPC.Dal.dal_shards + cctxt + (cctxt#chain, `Level level) + ~level:raw_level + () + in + return + @@ List.map + (fun Plugin.RPC.Dal.S.{delegate; indexes} -> + Data.Dal. + { + delegate = Tezos_crypto.Signature.Of_V1.public_key_hash delegate; + assigned_shard_indices = indexes; + }) + shard_assignments + let raw_block_round shell_header = let wrap = Environment.wrap_tzresult in let open Result_syntax in diff --git a/teztale/bin_teztale_archiver/PsYLVpVv_machine.real.ml b/teztale/bin_teztale_archiver/PsYLVpVv_machine.real.ml index d733fa70e4a9d1dd9a7da9f66102ae4cb8ba780b..4d887c96cd69ce1252e32a4faf2a6e2e30ef18ee 100644 --- a/teztale/bin_teztale_archiver/PsYLVpVv_machine.real.ml +++ b/teztale/bin_teztale_archiver/PsYLVpVv_machine.real.ml @@ -158,6 +158,8 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int priority = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of _cctxt _level = return_nil + let block_round header = match Data_encoding.Binary.of_bytes diff --git a/teztale/bin_teztale_archiver/PsddFKi3_machine.real.ml b/teztale/bin_teztale_archiver/PsddFKi3_machine.real.ml index c27d42260118e663b1dfba33a3289a244474a05e..cb087bae500bef69da9b917af6ee216240f41f1e 100644 --- a/teztale/bin_teztale_archiver/PsddFKi3_machine.real.ml +++ b/teztale/bin_teztale_archiver/PsddFKi3_machine.real.ml @@ -158,6 +158,8 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int priority = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of _cctxt _level = return_nil + let block_round header = match Data_encoding.Binary.of_bytes diff --git a/teztale/bin_teztale_archiver/Psithaca_machine.real.ml b/teztale/bin_teztale_archiver/Psithaca_machine.real.ml index 93f3b93d751371fea031026498590c4508ca6250..61922d0a42ae9b1876686b50a4c4661c309bb5ea 100644 --- a/teztale/bin_teztale_archiver/Psithaca_machine.real.ml +++ b/teztale/bin_teztale_archiver/Psithaca_machine.real.ml @@ -186,6 +186,8 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int round = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of _cctxt _level = return_nil + let raw_block_round shell_header = let wrap = Environment.wrap_tzresult in let open Result_syntax in diff --git a/teztale/bin_teztale_archiver/Pt24m4xi_machine.real.ml b/teztale/bin_teztale_archiver/Pt24m4xi_machine.real.ml index 742789b46c8229dc432a10bf51628c7a7758e0ea..98474167ebd4a759a89b840999ab311f4f7e56a5 100644 --- a/teztale/bin_teztale_archiver/Pt24m4xi_machine.real.ml +++ b/teztale/bin_teztale_archiver/Pt24m4xi_machine.real.ml @@ -158,6 +158,8 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int priority = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of _cctxt _level = return_nil + let block_round header = match Data_encoding.Binary.of_bytes diff --git a/teztale/bin_teztale_archiver/PtCJ7pwo_machine.real.ml b/teztale/bin_teztale_archiver/PtCJ7pwo_machine.real.ml index a7b59b162a6c5e4b05d72a72f6562fa7eaf21a71..44933f66a11d5b83042964e9236037fda41ee54f 100644 --- a/teztale/bin_teztale_archiver/PtCJ7pwo_machine.real.ml +++ b/teztale/bin_teztale_archiver/PtCJ7pwo_machine.real.ml @@ -158,6 +158,8 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int priority = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of _cctxt _level = return_nil + let block_round header = match Data_encoding.Binary.of_bytes diff --git a/teztale/bin_teztale_archiver/PtEdo2Zk_machine.real.ml b/teztale/bin_teztale_archiver/PtEdo2Zk_machine.real.ml index 6867a25add3407cadcfb276747cc178fa8664bd8..d897c2ed52379540f638d0950a2ce9544576e760 100644 --- a/teztale/bin_teztale_archiver/PtEdo2Zk_machine.real.ml +++ b/teztale/bin_teztale_archiver/PtEdo2Zk_machine.real.ml @@ -158,6 +158,8 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int priority = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of _cctxt _level = return_nil + let block_round header = match Data_encoding.Binary.of_bytes diff --git a/teztale/bin_teztale_archiver/PtGRANAD_machine.real.ml b/teztale/bin_teztale_archiver/PtGRANAD_machine.real.ml index 035f46b9718775fce7a69f791f808711d31cba1f..708099c5f9e6383c5af633711137ce5e7a1a7d0a 100644 --- a/teztale/bin_teztale_archiver/PtGRANAD_machine.real.ml +++ b/teztale/bin_teztale_archiver/PtGRANAD_machine.real.ml @@ -172,6 +172,8 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int priority = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of _cctxt _level = return_nil + let block_round header = match Data_encoding.Binary.of_bytes diff --git a/teztale/bin_teztale_archiver/PtHangz2_machine.real.ml b/teztale/bin_teztale_archiver/PtHangz2_machine.real.ml index 532b155847afabe2e9e3ccf59f79c8417cb164f3..5f40c4ca95761ca8d81361d5efa2804f9ec488fc 100644 --- a/teztale/bin_teztale_archiver/PtHangz2_machine.real.ml +++ b/teztale/bin_teztale_archiver/PtHangz2_machine.real.ml @@ -172,6 +172,8 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int priority = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of _cctxt _level = return_nil + let block_round header = match Data_encoding.Binary.of_bytes diff --git a/teztale/bin_teztale_archiver/PtJakart_machine.real.ml b/teztale/bin_teztale_archiver/PtJakart_machine.real.ml index 336d920146837011c19c12bb0478a8729c81b710..8f9165ee1f69b87204848b4a4caa48793fa21b4a 100644 --- a/teztale/bin_teztale_archiver/PtJakart_machine.real.ml +++ b/teztale/bin_teztale_archiver/PtJakart_machine.real.ml @@ -186,6 +186,8 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int round = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of _cctxt _level = return_nil + let raw_block_round shell_header = let wrap = Environment.wrap_tzresult in let open Result_syntax in diff --git a/teztale/bin_teztale_archiver/PtKathma_machine.real.ml b/teztale/bin_teztale_archiver/PtKathma_machine.real.ml index 9c7c2084e38d9ad5738fb691278588f622ba5d0d..935c66a8c3567d2199eb6fa30b41082aed3d8019 100644 --- a/teztale/bin_teztale_archiver/PtKathma_machine.real.ml +++ b/teztale/bin_teztale_archiver/PtKathma_machine.real.ml @@ -186,6 +186,8 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int round = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of _cctxt _level = return_nil + let raw_block_round shell_header = let wrap = Environment.wrap_tzresult in let open Result_syntax in diff --git a/teztale/bin_teztale_archiver/PtLimaPt_machine.real.ml b/teztale/bin_teztale_archiver/PtLimaPt_machine.real.ml index b67d6a929b2e54168c3370cfffe0d45317d4c250..61af5e1fdf98a63fc93b5b8522da848d7a7023df 100644 --- a/teztale/bin_teztale_archiver/PtLimaPt_machine.real.ml +++ b/teztale/bin_teztale_archiver/PtLimaPt_machine.real.ml @@ -187,6 +187,8 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int round = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of _cctxt _level = return_nil + let raw_block_round shell_header = let wrap = Environment.wrap_tzresult in let open Result_syntax in diff --git a/teztale/bin_teztale_archiver/PtMumbai_machine.real.ml b/teztale/bin_teztale_archiver/PtMumbai_machine.real.ml index 49b1da0c503720bf36b7164d0fc5ba27e362f1a4..d1bd33c43392787da722aab5d9e64aade112aac1 100644 --- a/teztale/bin_teztale_archiver/PtMumbai_machine.real.ml +++ b/teztale/bin_teztale_archiver/PtMumbai_machine.real.ml @@ -183,6 +183,8 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int round = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of _cctxt _level = return_nil + let raw_block_round shell_header = let wrap = Environment.wrap_tzresult in let open Result_syntax in diff --git a/teztale/bin_teztale_archiver/PtNairob_machine.real.ml b/teztale/bin_teztale_archiver/PtNairob_machine.real.ml index ecfeebc08199c00b17221e988f8c152bf3e388c4..217ba7938735f11b7315bcc4a518fd6666c258b6 100644 --- a/teztale/bin_teztale_archiver/PtNairob_machine.real.ml +++ b/teztale/bin_teztale_archiver/PtNairob_machine.real.ml @@ -183,6 +183,8 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int round = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of _cctxt _level = return_nil + let raw_block_round shell_header = let wrap = Environment.wrap_tzresult in let open Result_syntax in diff --git a/teztale/bin_teztale_archiver/PtParisB_machine.real.ml b/teztale/bin_teztale_archiver/PtParisB_machine.real.ml index 899876b0687e5934df7e1f60d45daa321d67f2b7..23447e7c9ae55a37f30a44b716dbe800375f84f0 100644 --- a/teztale/bin_teztale_archiver/PtParisB_machine.real.ml +++ b/teztale/bin_teztale_archiver/PtParisB_machine.real.ml @@ -188,6 +188,25 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int round = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of cctxt level = + let raw_level = Protocol.Alpha_context.Raw_level.of_int32_exn level in + let* shard_assignments = + Plugin.RPC.Dal.dal_shards + cctxt + (cctxt#chain, `Level level) + ~level:raw_level + () + in + return + @@ List.map + (fun Plugin.RPC.Dal.S.{delegate; indexes} -> + Data.Dal. + { + delegate = Tezos_crypto.Signature.Of_V1.public_key_hash delegate; + assigned_shard_indices = indexes; + }) + shard_assignments + let raw_block_round shell_header = let wrap = Environment.wrap_tzresult in let open Result_syntax in diff --git a/teztale/bin_teztale_archiver/PtSeouLo_machine.real.ml b/teztale/bin_teztale_archiver/PtSeouLo_machine.real.ml index eaddbfa2acdc90b2003415a7f3058eb4d0d7e769..180d52a80a0e6cbea77adf69eef013907f79dd33 100644 --- a/teztale/bin_teztale_archiver/PtSeouLo_machine.real.ml +++ b/teztale/bin_teztale_archiver/PtSeouLo_machine.real.ml @@ -188,6 +188,21 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int round = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of cctxt level = + let raw_level = Protocol.Alpha_context.Raw_level.of_int32_exn level in + let* shard_assignments = + Plugin.RPC.Dal.dal_shards + cctxt + (cctxt#chain, `Level level) + ~level:raw_level + () + in + return + @@ List.map + (fun Plugin.RPC.Dal.S.{delegate; indexes} -> + Data.Dal.{delegate; assigned_shard_indices = indexes}) + shard_assignments + let raw_block_round shell_header = let wrap = Environment.wrap_tzresult in let open Result_syntax in diff --git a/teztale/bin_teztale_archiver/PtTALLiN_machine.real.ml b/teztale/bin_teztale_archiver/PtTALLiN_machine.real.ml index b5a0faf178ac82ad72813b2906c4fe1fdd6e9125..ed842de7ba4d38e98d37902c1166fd37e0ce06fe 100644 --- a/teztale/bin_teztale_archiver/PtTALLiN_machine.real.ml +++ b/teztale/bin_teztale_archiver/PtTALLiN_machine.real.ml @@ -188,6 +188,21 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int round = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of cctxt level = + let raw_level = Protocol.Alpha_context.Raw_level.of_int32_exn level in + let* shard_assignments = + Plugin.RPC.Dal.dal_shards + cctxt + (cctxt#chain, `Level level) + ~level:raw_level + () + in + return + @@ List.map + (fun Plugin.RPC.Dal.S.{delegate; indexes} -> + Data.Dal.{delegate; assigned_shard_indices = indexes}) + shard_assignments + let raw_block_round shell_header = let wrap = Environment.wrap_tzresult in let open Result_syntax in diff --git a/teztale/bin_teztale_archiver/alpha_machine.real.ml b/teztale/bin_teztale_archiver/alpha_machine.real.ml index 5c655ce7fd7a803efaacf2ac1fbc3fab30c797b3..cd31a5f9c488e8631c3c3a5965913bb5a1b3b2e9 100644 --- a/teztale/bin_teztale_archiver/alpha_machine.real.ml +++ b/teztale/bin_teztale_archiver/alpha_machine.real.ml @@ -188,6 +188,21 @@ module Services : Protocol_machinery.PROTOCOL_SERVICES = struct assert (Int32.of_int round = hd.round) ; return (hd.delegate, baking_rights) + let dal_shards_of cctxt level = + let raw_level = Protocol.Alpha_context.Raw_level.of_int32_exn level in + let* shard_assignments = + Plugin.RPC.Dal.dal_shards + cctxt + (cctxt#chain, `Level level) + ~level:raw_level + () + in + return + @@ List.map + (fun Plugin.RPC.Dal.S.{delegate; indexes} -> + Data.Dal.{delegate; assigned_shard_indices = indexes}) + shard_assignments + let raw_block_round shell_header = let wrap = Environment.wrap_tzresult in let open Result_syntax in diff --git a/teztale/bin_teztale_archiver/archiver.ml b/teztale/bin_teztale_archiver/archiver.ml index 3efe026672a716265f948f740b31c87644fc9fb2..df097473dd1a9efd088b271fea6e136d61bcb53e 100644 --- a/teztale/bin_teztale_archiver/archiver.ml +++ b/teztale/bin_teztale_archiver/archiver.ml @@ -35,4 +35,6 @@ module type S = sig unit val add_rights : level:Int32.t -> Consensus_ops.rights -> unit + + val add_dal_shards : level:Int32.t -> Data.Dal.shard_assignment list -> unit end diff --git a/teztale/bin_teztale_archiver/archiver.mli b/teztale/bin_teztale_archiver/archiver.mli index 3efe026672a716265f948f740b31c87644fc9fb2..df097473dd1a9efd088b271fea6e136d61bcb53e 100644 --- a/teztale/bin_teztale_archiver/archiver.mli +++ b/teztale/bin_teztale_archiver/archiver.mli @@ -35,4 +35,6 @@ module type S = sig unit val add_rights : level:Int32.t -> Consensus_ops.rights -> unit + + val add_dal_shards : level:Int32.t -> Data.Dal.shard_assignment list -> unit end diff --git a/teztale/bin_teztale_archiver/converter.ml b/teztale/bin_teztale_archiver/converter.ml index f5900c85c098665ffad67c63f1d2ed919ad5666f..a8d21ac288d3c60926474c2ee590f1182f931b81 100644 --- a/teztale/bin_teztale_archiver/converter.ml +++ b/teztale/bin_teztale_archiver/converter.ml @@ -20,7 +20,13 @@ let to_received_ops ctx endpoint auth level data = let received_ops = List.map (fun Data.Delegate_operations. - {delegate; first_slot; attesting_power; operations} + { + delegate; + first_slot; + attesting_power; + operations; + assigned_shard_indices = _; + } -> ( Consensus_ops.{address = delegate; first_slot; power = attesting_power}, List.flatten @@ -86,7 +92,13 @@ let included_ops_map level data = List.fold_left (fun acc Data.Delegate_operations. - {delegate; first_slot = _; attesting_power; operations} + { + delegate; + first_slot = _; + attesting_power; + operations; + assigned_shard_indices = _; + } -> List.fold_left (fun acc diff --git a/teztale/bin_teztale_archiver/general_archiver.ml b/teztale/bin_teztale_archiver/general_archiver.ml index 9b0bcb120f0ae33850be77b0f14189eb621affab..ef09c18d6ffaa095ddb05980ce18135c0df5c2c5 100644 --- a/teztale/bin_teztale_archiver/general_archiver.ml +++ b/teztale/bin_teztale_archiver/general_archiver.ml @@ -41,6 +41,8 @@ let supported_protocols = ref [] let registered_rights_levels = Ring.create 120 +let registered_dal_shards_levels = Ring.create 120 + let rights_machine = Protocol_hash.Table.create 10 let past_block_machine = Protocol_hash.Table.create 10 @@ -57,6 +59,12 @@ let maybe_add_rights (module A : Archiver.S) level rights = let () = Ring.add registered_rights_levels level in A.add_rights ~level rights +let maybe_add_dal_shards (module A : Archiver.S) level shard_assignments = + if Ring.mem registered_dal_shards_levels level then () + else ( + Ring.add registered_dal_shards_levels level ; + A.add_dal_shards ~level shard_assignments) + let dump_my_current_attestations (module A : Archiver.S) ~unaccurate ~level rights attestations = let () = maybe_add_rights (module A) level rights in @@ -137,11 +145,15 @@ module Define (Services : Protocol_machinery.PROTOCOL_SERVICES) = struct (Some cycle_info) reception_times + let dal_shards_of ctxt level = + let cctx = Services.wrap_full ctxt in + Services.dal_shards_of cctx level + let () = Protocol_hash.Table.add live_block_machine Services.hash - (rights_of, get_applied_block) + (rights_of, get_applied_block, dal_shards_of) let rec pack_by_slot i e = function | ((i', l) as x) :: t -> @@ -333,6 +345,17 @@ module Loops (Archiver : Archiver.S) = struct in Lwt.return_unit + let maybe_register_rights_and_dal_shards ~rights_of ~dal_shards_of cctx ~level + ops = + if List.is_empty ops then return_unit + else + let* rights = rights_of cctx (Int32.pred level) in + let pred_level = Int32.pred level in + let () = maybe_add_rights (module Archiver) pred_level rights in + let* dal_shards = dal_shards_of cctx pred_level in + let () = maybe_add_dal_shards (module Archiver) pred_level dal_shards in + return_unit + let reception_blocks_loop cctx = let logger = Log.logger () in let*! block_stream = @@ -381,7 +404,8 @@ module Loops (Archiver : Archiver.S) = struct current_protocol) in Lwt.return ((fun _ _ _ _ _ -> return_unit), None) - | Some (rights_of, get_applied_block) -> + | Some (rights_of, get_applied_block, dal_shards_of) + -> let recorder cctx level hash header reception_time = let* (( _block_info, @@ -395,31 +419,23 @@ module Loops (Archiver : Archiver.S) = struct reception_time in let* () = - if List.is_empty attestations then return_unit - else - let* rights = - rights_of cctx (Int32.pred level) - in - let () = - maybe_add_rights - (module Archiver) - (Int32.pred level) - rights - in - return_unit + maybe_register_rights_and_dal_shards + ~rights_of + ~dal_shards_of + cctx + ~level:(Int32.pred level) + attestations in let* () = if List.is_empty preattestations then return_unit else - let* rights = rights_of cctx level in - let () = - maybe_add_rights - (module Archiver) - level - rights - in - return_unit + maybe_register_rights_and_dal_shards + ~rights_of + ~dal_shards_of + cctx + ~level + preattestations in let () = Archiver.add_block ~level block_data in return_unit diff --git a/teztale/bin_teztale_archiver/json_archiver.ml b/teztale/bin_teztale_archiver/json_archiver.ml index 1f7334248ed0dad9474b48911d563522280616e0..47218b92e0019ed18130cbfe985bfa8a6ea65e62 100644 --- a/teztale/bin_teztale_archiver/json_archiver.ml +++ b/teztale/bin_teztale_archiver/json_archiver.ml @@ -109,7 +109,13 @@ let add_inclusion_in_block block_hash validators delegate_operations = List.fold_left (fun (acc, missing) Data.Delegate_operations.( - {delegate; first_slot; attesting_power; operations} as delegate_ops) + { + delegate; + first_slot; + attesting_power; + operations; + assigned_shard_indices; + } as delegate_ops) -> match List.partition @@ -133,6 +139,7 @@ let add_inclusion_in_block block_hash validators delegate_operations = op.op.kind ?ops_round:op.op.round operations; + assigned_shard_indices; } :: acc, missing' ) @@ -161,6 +168,7 @@ let add_inclusion_in_block block_hash validators delegate_operations = block_inclusion = [block_hash]; }; ]; + assigned_shard_indices = []; } :: acc) updated_known @@ -353,8 +361,13 @@ let dump_received logger path ?unaccurate level received_ops = List.fold_left (fun (acc, missing) Data.Delegate_operations.( - {delegate; first_slot; attesting_power; operations} as - delegate_ops) + { + delegate; + first_slot; + attesting_power; + operations; + assigned_shard_indices; + } as delegate_ops) -> match List.partition @@ -372,6 +385,7 @@ let dump_received logger path ?unaccurate level received_ops = first_slot; attesting_power; operations = merge_operations operations new_operations; + assigned_shard_indices; } :: acc, missing' ) @@ -415,6 +429,7 @@ let dump_received logger path ?unaccurate level received_ops = block_inclusion = []; }) ops; + assigned_shard_indices = []; } :: acc) updated_known @@ -514,3 +529,6 @@ let add_block ~level (block, cycle_info, (endos, preendos), baking_rights) = (* not used *) let add_rights ~level:_ _rights = () + +(* not used *) +let add_dal_shards ~level:_ _shard_assignments = () diff --git a/teztale/bin_teztale_archiver/protocol_machinery.ml b/teztale/bin_teztale_archiver/protocol_machinery.ml index fd98bcb031fadecb4f42613bf02340c641d75bd5..6743947fdded9021167b2ded6f206042a1e91aba 100644 --- a/teztale/bin_teztale_archiver/protocol_machinery.ml +++ b/teztale/bin_teztale_archiver/protocol_machinery.ml @@ -41,6 +41,10 @@ module type PROTOCOL_SERVICES = sig (Tezos_crypto.Signature.public_key_hash * Data.baking_right list) tzresult Lwt.t + (* dal_shards_of _ level *) + val dal_shards_of : + wrap_full -> Int32.t -> Data.Dal.shard_assignment list tzresult Lwt.t + val baker_and_cycle : wrap_full -> Block_hash.t -> diff --git a/teztale/bin_teztale_archiver/protocol_machinery.mli b/teztale/bin_teztale_archiver/protocol_machinery.mli index fd98bcb031fadecb4f42613bf02340c641d75bd5..6743947fdded9021167b2ded6f206042a1e91aba 100644 --- a/teztale/bin_teztale_archiver/protocol_machinery.mli +++ b/teztale/bin_teztale_archiver/protocol_machinery.mli @@ -41,6 +41,10 @@ module type PROTOCOL_SERVICES = sig (Tezos_crypto.Signature.public_key_hash * Data.baking_right list) tzresult Lwt.t + (* dal_shards_of _ level *) + val dal_shards_of : + wrap_full -> Int32.t -> Data.Dal.shard_assignment list tzresult Lwt.t + val baker_and_cycle : wrap_full -> Block_hash.t -> diff --git a/teztale/bin_teztale_archiver/server_archiver.ml b/teztale/bin_teztale_archiver/server_archiver.ml index ff8e9bd3823996d53995ceb8e5c3e2809abcf6de..9a950cd2af4381634b723a832798969b9ff61e9f 100644 --- a/teztale/bin_teztale_archiver/server_archiver.ml +++ b/teztale/bin_teztale_archiver/server_archiver.ml @@ -19,6 +19,7 @@ type chunk = * Data.baking_right list) | Mempool of Int32.t (* level *) * Consensus_ops.delegate_ops | Rights of (Int32.t (* level *) * Consensus_ops.rights) + | Dal_shards of Int32.t (* level *) * Data.Dal.shard_assignment list type ctx = { cohttp_ctx : Cohttp_lwt_unix.Net.ctx; @@ -99,12 +100,25 @@ let send_mempool ctx level ops = let path = Int32.to_string level ^ "/mempool" in send_something ctx path body +let send_dal_shards ctx level shard_assignments = + let body = + `String + (Ezjsonm.value_to_string + (Data_encoding.Json.construct + Data.Dal.shard_assignments_encoding + shard_assignments)) + in + let path = Int32.to_string level ^ "/dal_shards" in + send_something ctx path body + let chunk_stream, chunk_feeder = Lwt_stream.create () let send actx = function | Block (level, block) -> send_block actx level block | Mempool (level, ops) -> send_mempool actx level ops | Rights (level, rights) -> send_rights actx level rights + | Dal_shards (level, shard_assignments) -> + send_dal_shards actx level shard_assignments let launch actx _source = Lwt_stream.iter_p @@ -120,3 +134,6 @@ let add_mempool ?unaccurate:(_ : bool option) ~level items = let add_block ~level block = chunk_feeder (Some (Block (level, block))) let add_rights ~level rights = chunk_feeder (Some (Rights (level, rights))) + +let add_dal_shards ~level shard_assignments = + chunk_feeder (Some (Dal_shards (level, shard_assignments))) diff --git a/teztale/bin_teztale_archiver/server_archiver.mli b/teztale/bin_teztale_archiver/server_archiver.mli index ca945541cebaf62bce7aeeb7408c0f98eecda141..24c61502ab1d1764de9a3ba031ec6e216207ed8a 100644 --- a/teztale/bin_teztale_archiver/server_archiver.mli +++ b/teztale/bin_teztale_archiver/server_archiver.mli @@ -19,6 +19,7 @@ type chunk = * Data.baking_right list) | Mempool of Int32.t (* level *) * Consensus_ops.delegate_ops | Rights of (Int32.t (* level *) * Consensus_ops.rights) + | Dal_shards of Int32.t (* level *) * Data.Dal.shard_assignment list type ctx = { cohttp_ctx : Cohttp_lwt_unix.Net.ctx; diff --git a/teztale/bin_teztale_archiver/teztale_archiver_main.ml b/teztale/bin_teztale_archiver/teztale_archiver_main.ml index d9478c20197601a34bbbed3259cc9a2a226ff673..267b72b934d204b6f9f0959643559a4da5b4729f 100644 --- a/teztale/bin_teztale_archiver/teztale_archiver_main.ml +++ b/teztale/bin_teztale_archiver/teztale_archiver_main.ml @@ -90,6 +90,7 @@ let server_to_json_chunk : Server_archiver.chunk -> Json_archiver.chunk option = baking_rights )) | Mempool (level, ops) -> Some (Mempool (None, level, ops)) | Rights (_, _) -> None + | Dal_shards (_, _) -> None let select_commands _ctxt Client_config.{chain; _} = return @@ -232,6 +233,7 @@ let select_commands _ctxt Client_config.{chain; _} = | Block (level, _) -> (level, "/block") | Mempool (level, _) -> (level, "/mempool") | Rights (level, _) -> (level, "/rights") + | Dal_shards (level, _) -> (level, "/dal_shards") in Printf.sprintf "(%ld) Failed to send %s data and --backup-dir is \ diff --git a/teztale/bin_teztale_server/exporter.ml b/teztale/bin_teztale_server/exporter.ml index 8c2f98905c1c3f799a041f81436fafbd4da7fc17..af32b638c7e4ff1ce721cdd1231355f95bfc2c86 100644 --- a/teztale/bin_teztale_server/exporter.ml +++ b/teztale/bin_teztale_server/exporter.ml @@ -150,6 +150,20 @@ let select_missing_blocks conf db_pool boundaries = Int32Map.empty) db_pool +let select_dal_shard_assignments = + Caqti_request.Infix.( + Caqti_type.(t2 int32 int32) + ->* Caqti_type.(t3 int32 Sql_requests.Type.public_key_hash int)) + "SELECT\n\ + \ er.level,\n\ + \ delegates.address,\n\ + \ dsa.shard_index\n\ + FROM endorsing_rights er\n\ + JOIN dal_shard_assignments dsa ON er.id = dsa.endorsing_right\n\ + JOIN delegates ON delegates.id = er.delegate\n\ + WHERE er.level >= $1\n\ + AND er.level <= $2" + let select_blocks conf db_pool boundaries = let block_request = Caqti_request.Infix.( @@ -257,7 +271,10 @@ let select_ops conf db_pool boundaries = let ops = Ops.add delegate - (first_slot, power, Tezos_crypto.Hashed.Operation_hash.Map.empty) + ( first_slot, + power, + Tezos_crypto.Hashed.Operation_hash.Map.empty, + [] (* assigned_shard_indices *) ) ops in Int32Map.add level ops info @@ -272,7 +289,7 @@ let select_ops conf db_pool boundaries = Ops.update delegate (function - | Some (first_slot, power, ops) -> + | Some (first_slot, power, ops, assigned_shard_indices) -> let op = match Tezos_crypto.Hashed.Operation_hash.Map.find_opt op_hash ops @@ -284,7 +301,7 @@ let select_ops conf db_pool boundaries = let ops' = Tezos_crypto.Hashed.Operation_hash.Map.add op_hash op ops in - Some (first_slot, power, ops') + Some (first_slot, power, ops', assigned_shard_indices) | None -> None) ops in @@ -305,7 +322,7 @@ let select_ops conf db_pool boundaries = Ops.update delegate (function - | Some (first_slot, power, ops) -> + | Some (first_slot, power, ops, assigned_shard_indices) -> let op = match Tezos_crypto.Hashed.Operation_hash.Map.find_opt op_hash ops @@ -318,12 +335,43 @@ let select_ops conf db_pool boundaries = let ops' = Tezos_crypto.Hashed.Operation_hash.Map.add op_hash op ops in - Some (first_slot, power, ops') + Some (first_slot, power, ops', assigned_shard_indices) | None -> None) ops in Int32Map.add level ops info in + let cb_shards (level, delegate, shard_index) info = + let ops = + match Int32Map.find_opt level info with Some m -> m | None -> Ops.empty + in + let ops = + Ops.update + delegate + (function + | Some (first_slot, power, op_map, assigned_shard_indices) -> + let assigned_shard_indices = + if List.mem shard_index assigned_shard_indices then + assigned_shard_indices + else shard_index :: assigned_shard_indices + in + Some (first_slot, power, op_map, assigned_shard_indices) + | None -> + let logger = Log.logger () in + Lib_teztale_base.Log.warning logger (fun () -> + Format.asprintf + "DAL: received shard index %d for delegate %a at level \ + %ld, but no rights were recorded for that delegate, level \ + pair. Ignoring." + shard_index + Tezos_crypto.Signature.Public_key_hash.pp + delegate + level) ; + None) + ops + in + Int32Map.add level ops info + in let* out = Caqti_lwt_unix.Pool.use (fun (module Db : Caqti_lwt.CONNECTION) -> @@ -338,10 +386,17 @@ let select_ops conf db_pool boundaries = Db.fold q_included cb_included boundaries out) db_pool in + let* out = + Caqti_lwt_unix.Pool.use + (fun (module Db : Caqti_lwt.CONNECTION) -> + maybe_with_metrics conf "select_operations_reception" @@ fun () -> + Db.fold q_received cb_received boundaries out) + db_pool + in Caqti_lwt_unix.Pool.use (fun (module Db : Caqti_lwt.CONNECTION) -> - maybe_with_metrics conf "select_operations_reception" @@ fun () -> - Db.fold q_received cb_received boundaries out) + maybe_with_metrics conf "select_dal_shard_assignments" @@ fun () -> + Db.fold select_dal_shard_assignments cb_shards boundaries out) db_pool let translate_ops info = @@ -363,13 +418,15 @@ let translate_ops info = Int32Map.map (fun info -> Tezos_crypto.Signature.Public_key_hash.Map.fold - (fun pkh (first_slot, power, pkh_ops) acc -> + (fun pkh (first_slot, power, pkh_ops, assigned_shard_indices) acc -> Lib_teztale_base.Data.Delegate_operations. { delegate = pkh; first_slot; attesting_power = power; operations = translate pkh_ops; + assigned_shard_indices = + List.sort_uniq compare assigned_shard_indices; } :: acc) info diff --git a/teztale/bin_teztale_server/sql_requests.ml b/teztale/bin_teztale_server/sql_requests.ml index 87946cfa9da2bd37f22ffe61dcfc7794e47e9ecc..dd7d54207e77a716f59b2a6e04bcb41191d9e7fb 100644 --- a/teztale/bin_teztale_server/sql_requests.ml +++ b/teztale/bin_teztale_server/sql_requests.ml @@ -117,6 +117,14 @@ let create_missing_blocks = \ FOREIGN KEY (baker) REFERENCES delegates(id),\n\ \ UNIQUE (source, level, round))" +let create_dal_shard_assignments = + "CREATE TABLE IF NOT EXISTS dal_shard_assignments(\n\ + \ id $(PRIMARY_INCREMENTING_INT) PRIMARY KEY,\n\ + \ endorsing_right $(PRIMARY_INCREMENTING_INT_REF) NOT NULL,\n\ + \ shard_index INTEGER NOT NULL,\n\ + \ FOREIGN KEY (endorsing_right) REFERENCES endorsing_rights(id),\n\ + \ UNIQUE (endorsing_right, shard_index))" + module Mutex = struct let delegates = Lwt_mutex.create () @@ -137,6 +145,8 @@ module Mutex = struct let cycles = Lwt_mutex.create () let missing_blocks = Lwt_mutex.create () + + let dal_shard_assignments = Lwt_mutex.create () end let create_endorsing_rights_level_idx = @@ -167,6 +177,10 @@ let create_cycles_level_idx = let create_missing_blocks_level_idx = "CREATE INDEX IF NOT EXISTS missing_blocks_level_idx ON missing_blocks(level)" +let create_dal_shard_assignments_endorsing_right_idx = + "CREATE INDEX IF NOT EXISTS dal_shard_assignments_endorsing_right_idx ON \ + dal_shard_assignments(endorsing_right)" + let create_tables = [ create_delegates; @@ -179,6 +193,7 @@ let create_tables = create_endorsing_rights; create_cycles; create_missing_blocks; + create_dal_shard_assignments; create_endorsing_rights_level_idx; create_blocks_level_idx; create_operations_level_idx; @@ -187,6 +202,7 @@ let create_tables = create_operations_inclusion_operation_idx; create_cycles_level_idx; create_missing_blocks_level_idx; + create_dal_shard_assignments_endorsing_right_idx; ] let alter_blocks = @@ -399,5 +415,21 @@ let insert_received_block = COALESCE(blocks_reception.validation_timestamp, \ excluded.validation_timestamp)" +let insert_dal_shard_assignment = + Caqti_request.Infix.( + Caqti_type.( + t3 + (* $1 level *) int32 + (* $2 delegate *) Type.public_key_hash + (* $3 shard_index *) int + ->. unit)) + "INSERT INTO dal_shard_assignments (endorsing_right, shard_index)\n\ + SELECT er.id AS endorsing_right, $3 AS shard_index\n\ + FROM endorsing_rights er\n\ + JOIN delegates ON er.delegate = delegates.id\n\ + WHERE er.level = $1\n\ + AND delegates.address = $2\n\ + ON CONFLICT DO NOTHING" + let maybe_with_metrics (c : Config.t) (name : string) (f : unit -> 'a Lwt.t) = if c.with_metrics then Metrics.sql name f else f () diff --git a/teztale/bin_teztale_server/sql_requests.mli b/teztale/bin_teztale_server/sql_requests.mli index e9293af51ee30575c2b0ef812a487c03701fa929..5d8dbe93f5348648448b04dbfc723ee496754bd6 100644 --- a/teztale/bin_teztale_server/sql_requests.mli +++ b/teztale/bin_teztale_server/sql_requests.mli @@ -41,6 +41,8 @@ module Mutex : sig val cycles : Lwt_mutex.t val missing_blocks : Lwt_mutex.t + + val dal_shard_assignments : Lwt_mutex.t end val create_tables : string list @@ -120,4 +122,10 @@ val insert_received_block : [`Zero] ) Caqti_request.t +val insert_dal_shard_assignment : + ( int32 * Tezos_crypto.Signature.public_key_hash * int, + unit, + [`Zero] ) + Caqti_request.t + val maybe_with_metrics : Config.t -> string -> (unit -> 'a Lwt.t) -> 'a Lwt.t diff --git a/teztale/bin_teztale_server/teztale_server_main.ml b/teztale/bin_teztale_server/teztale_server_main.ml index fc21d2ffc3a8e017fc06810b506793132f0deacf..bf1b9a9ade5ff58b1e10f94a53224488123be7fa 100644 --- a/teztale/bin_teztale_server/teztale_server_main.ml +++ b/teztale/bin_teztale_server/teztale_server_main.ml @@ -897,6 +897,20 @@ let import_callback ~logger conf db_pool g data = (level, first_slot, attesting_power, delegate)) data.Lib_teztale_base.Data.delegate_operations in + (* DAL shard assignments *) + let* () = + Tezos_lwt_result_stdlib.Lwtreslib.Bare.List.iter_es + (fun Lib_teztale_base.Data.Delegate_operations. + {delegate; assigned_shard_indices; _} + -> + Tezos_lwt_result_stdlib.Lwtreslib.Bare.List.iter_es + (fun shard_index -> + Db.exec + Sql_requests.insert_dal_shard_assignment + (level, delegate, shard_index)) + assigned_shard_indices) + data.Lib_teztale_base.Data.delegate_operations + in (* blocks *) let* () = Tezos_lwt_result_stdlib.Lwtreslib.Bare.List.iter_es @@ -1003,6 +1017,45 @@ let import_callback ~logger conf db_pool g data = ~body:"Level imported" ()) +let dal_shards_callback ~logger conf db_pool g shard_assignments = + let level = Int32.of_string (Re.Group.get g 1) in + let out = + Caqti_lwt_unix.Pool.use + (fun (module Db : Caqti_lwt.CONNECTION) -> + let open Tezos_lwt_result_stdlib.Lwtreslib.Bare.Monad.Lwt_result_syntax in + let* () = + let delegates = + List.map + (fun Lib_teztale_base.Data.Dal.{delegate; _} -> delegate) + shard_assignments + in + may_insert_delegates (module Db) conf delegates + in + let rows = + List.concat_map + (fun Lib_teztale_base.Data.Dal.{delegate; assigned_shard_indices} -> + List.map + (fun shard_index -> (level, delegate, shard_index)) + assigned_shard_indices) + shard_assignments + in + maybe_with_metrics conf "insert_dal_shard_assignments" @@ fun () -> + without_cache + Sql_requests.Mutex.dal_shard_assignments + Sql_requests.insert_dal_shard_assignment + (module Db) + conf + rows) + db_pool + in + with_caqti_error ~logger out (fun () -> + Cohttp_lwt_unix.Server.respond_string + ~headers: + (Cohttp.Header.init_with "content-type" "text/plain; charset=UTF-8") + ~status:`OK + ~body:"DAL shard assignments" + ()) + let extract_boundaries g = let min = Re.Group.get g 1 in let max = Re.Group.get g 2 in @@ -1016,7 +1069,9 @@ let extract_boundaries g = - //mempool Used by archiver to send data about consensus operations for a given level. - //import - Use by archiver to import past data recorded locally. + Used by archiver to import past data recorded locally. + - //dal_shards + Used by archiver to send DAL shard assignments per delegate for a given level. - /timestamp/ Get the levels (if any) before and after a given timestamp. - /ping @@ -1082,6 +1137,13 @@ let routes : Lib_teztale_base.Data.encoding body (import_callback ~logger conf db_pool g)) ); + ( Re.seq [Re.str "/"; Re.group (Re.rep1 Re.digit); Re.str "/dal_shards"], + fun g ~logger ~conf ~admins:_ ~users db_pool header meth body -> + post_only_endpoint !users header meth (fun _source -> + with_data + Lib_teztale_base.Data.Dal.shard_assignments_encoding + body + (dal_shards_callback ~logger conf db_pool g)) ); ( Re.seq [Re.str "/timestamp/"; Re.group (Re.rep1 Re.digit)], fun g ~logger ~conf:_ ~admins:_ ~users:_ db_pool _header meth _body -> get_only_endpoint meth (fun () -> diff --git a/teztale/lib_teztale_base/data.ml b/teztale/lib_teztale_base/data.ml index c6d0e986af7befff295a9de3efca96cd4ae5a8e3..779a6bb12dc31539d2a8d368606ab7ff3e2a76d5 100644 --- a/teztale/lib_teztale_base/data.ml +++ b/teztale/lib_teztale_base/data.ml @@ -101,6 +101,7 @@ module Delegate_operations = struct first_slot : int; attesting_power : int; operations : operation list; + assigned_shard_indices : int list; } let legacy_encoding = @@ -110,7 +111,13 @@ module Delegate_operations = struct (fun (delegate, reception_time, errors, block_inclusion) -> match (reception_time, block_inclusion) with | None, [] -> - {delegate; first_slot = 0; attesting_power = 0; operations = []} + { + delegate; + first_slot = 0; + attesting_power = 0; + operations = []; + assigned_shard_indices = []; + } | _, _ -> let mempool_inclusion = match reception_time with @@ -132,6 +139,7 @@ module Delegate_operations = struct block_inclusion; }; ]; + assigned_shard_indices = []; }) (obj4 (req "delegate" Tezos_crypto.Signature.Public_key_hash.encoding) @@ -142,15 +150,38 @@ module Delegate_operations = struct let encoding = let open Data_encoding in conv - (fun {delegate; first_slot; attesting_power; operations} -> - (delegate, first_slot, attesting_power, operations)) - (fun (delegate, first_slot, attesting_power, operations) -> - {delegate; first_slot; attesting_power; operations}) - (obj4 + (fun { + delegate; + first_slot; + attesting_power; + operations; + assigned_shard_indices; + } + -> + ( delegate, + first_slot, + attesting_power, + operations, + assigned_shard_indices )) + (fun ( delegate, + first_slot, + attesting_power, + operations, + assigned_shard_indices ) + -> + { + delegate; + first_slot; + attesting_power; + operations; + assigned_shard_indices; + }) + (obj5 (req "delegate" Tezos_crypto.Signature.Public_key_hash.encoding) (dft "first_slot" int16 0) (dft "endorsing_power" int16 0) - (dft "operations" (list operation_encoding) [])) + (dft "operations" (list operation_encoding) []) + (dft "assigned_shard_indices" (list int16) [])) let encoding = let open Data_encoding in @@ -360,3 +391,25 @@ module Archiver = struct (dft "preendorsements" (list Consensus_ops.block_op_encoding) []) (dft "baking_rights" (list baking_right_encoding) []))) end + +module Dal = struct + type shard_assignment = { + delegate : Tezos_crypto.Signature.public_key_hash; + assigned_shard_indices : int list; + } + + type shard_assignments = shard_assignment list + + let shard_assignment_encoding = + let open Data_encoding in + conv + (fun {delegate; assigned_shard_indices} -> + (delegate, assigned_shard_indices)) + (fun (delegate, assigned_shard_indices) -> + {delegate; assigned_shard_indices}) + (obj2 + (req "delegate" Tezos_crypto.Signature.Public_key_hash.encoding) + (dft "assigned_shard_indices" (list int16) [])) + + let shard_assignments_encoding = Data_encoding.list shard_assignment_encoding +end diff --git a/teztale/scripts/db/dump_month.sh b/teztale/scripts/db/dump_month.sh old mode 100644 new mode 100755 index d164382e32fa92b918fcd84788c8a5daa92ed24f..d7df29597338ac795cf4551df5edeb1ea2344c39 --- a/teztale/scripts/db/dump_month.sh +++ b/teztale/scripts/db/dump_month.sh @@ -30,8 +30,15 @@ mkdir -p "$OUTPUT_DIR" DUMP_FILE_PREFIX="teztale_${DATABASE}_dump" # Convert date to start and end timestamps (epoch) -START_TS=$(date -d "$DATE-01" +%s) -END_TS=$(date -d "$DATE-01 +1 month" +%s) +# Use GNU date if available (gdate on macOS), otherwise plain date +if command -v gdate >/dev/null 2>&1; then + DATE_BIN=gdate +else + DATE_BIN=date +fi + +START_TS=$("$DATE_BIN" -d "$DATE-01" +%s) +END_TS=$("$DATE_BIN" -d "$DATE-01 +1 month" +%s) echo "Dumping data from timestamp $START_TS to $END_TS [$DATABASE]" @@ -45,10 +52,11 @@ DROP VIEW IF EXISTS monthly_view_blocks_reception CASCADE; DROP VIEW IF EXISTS monthly_view_operations CASCADE; DROP VIEW IF EXISTS monthly_view_operations_reception CASCADE; DROP VIEW IF EXISTS monthly_view_operations_inclusion CASCADE; +DROP VIEW IF EXISTS monthly_view_endorsing_rights CASCADE; +DROP VIEW IF EXISTS monthly_view_dal_shard_assignments CASCADE; DROP VIEW IF EXISTS monthly_view_delegates CASCADE; DROP VIEW IF EXISTS monthly_view_cycles CASCADE; DROP VIEW IF EXISTS monthly_view_missing_blocks CASCADE; -DROP VIEW IF EXISTS monthly_view_endorsing_rights CASCADE; DROP VIEW IF EXISTS monthly_view_nodes CASCADE; -- Create view for blocks in the specified month @@ -78,6 +86,17 @@ CREATE VIEW monthly_view_operations_inclusion AS SELECT ops_inclusion.* FROM operations_inclusion ops_inclusion INNER JOIN monthly_view_blocks mb ON ops_inclusion.block = mb.id; +-- Create view for related endorsing_rights +CREATE VIEW monthly_view_endorsing_rights AS +SELECT er.* FROM endorsing_rights er +WHERE er.level >= (SELECT MIN(level) FROM monthly_view_blocks) +AND er.level <= (SELECT MAX(level) FROM monthly_view_blocks); + +-- Create view for related DAL shard assignments +CREATE VIEW monthly_view_dal_shard_assignments AS +SELECT dsa.* FROM dal_shard_assignments dsa +JOIN monthly_view_endorsing_rights er ON er.id = dsa.endorsing_right; + -- Create view for related delegates CREATE VIEW monthly_view_delegates AS SELECT DISTINCT d.* FROM delegates d @@ -85,6 +104,8 @@ WHERE d.id IN ( SELECT baker FROM monthly_view_blocks UNION SELECT endorser FROM monthly_view_operations + UNION + SELECT delegate FROM monthly_view_endorsing_rights ); -- Create view for related cycles @@ -100,12 +121,6 @@ SELECT mb.* FROM missing_blocks mb WHERE mb.level >= (SELECT MIN(level) FROM monthly_view_blocks) AND mb.level <= (SELECT MAX(level) FROM monthly_view_blocks); --- Create view for related endorsing_rights -CREATE VIEW monthly_view_endorsing_rights AS -SELECT er.* FROM endorsing_rights er -WHERE er.level >= (SELECT MIN(level) FROM monthly_view_blocks) -AND er.level <= (SELECT MAX(level) FROM monthly_view_blocks); - -- Create view for related nodes CREATE VIEW monthly_view_nodes AS SELECT DISTINCT n.* FROM nodes n @@ -124,10 +139,11 @@ SELECT (SELECT COUNT(*) FROM monthly_view_blocks_reception) as block_reception_count, (SELECT COUNT(*) FROM monthly_view_operations) as operation_count, (SELECT COUNT(*) FROM monthly_view_operations_reception) as operation_reception_count, + (SELECT COUNT(*) FROM monthly_view_endorsing_rights) as endorsing_rights_count, + (SELECT COUNT(*) FROM monthly_view_dal_shard_assignments) as dal_shard_assignments_count, (SELECT COUNT(*) FROM monthly_view_delegates) as delegate_count, (SELECT COUNT(*) FROM monthly_view_cycles) as cycle_count, (SELECT COUNT(*) FROM monthly_view_missing_blocks) as missing_block_count, - (SELECT COUNT(*) FROM monthly_view_endorsing_rights) as endorsing_rights_count, (SELECT COUNT(*) FROM monthly_view_nodes) as nodes_count; COMMIT; @@ -160,6 +176,12 @@ CREATE TABLE temp_monthly_view_operations_reception AS TABLE monthly_view_operat DROP TABLE IF EXISTS temp_monthly_view_operations_inclusion; CREATE TABLE temp_monthly_view_operations_inclusion AS TABLE monthly_view_operations_inclusion; +DROP TABLE IF EXISTS temp_monthly_view_endorsing_rights; +CREATE TABLE temp_monthly_view_endorsing_rights AS TABLE monthly_view_endorsing_rights; + +DROP TABLE IF EXISTS temp_monthly_view_dal_shard_assignments; +CREATE TABLE temp_monthly_view_dal_shard_assignments AS TABLE monthly_view_dal_shard_assignments; + DROP TABLE IF EXISTS temp_monthly_view_delegates; CREATE TABLE temp_monthly_view_delegates AS TABLE monthly_view_delegates; @@ -169,9 +191,6 @@ CREATE TABLE temp_monthly_view_cycles AS TABLE monthly_view_cycles; DROP TABLE IF EXISTS temp_monthly_view_missing_blocks; CREATE TABLE temp_monthly_view_missing_blocks AS TABLE monthly_view_missing_blocks; -DROP TABLE IF EXISTS temp_monthly_view_endorsing_rights; -CREATE TABLE temp_monthly_view_endorsing_rights AS TABLE monthly_view_endorsing_rights; - DROP TABLE IF EXISTS temp_monthly_view_nodes; CREATE TABLE temp_monthly_view_nodes AS TABLE monthly_view_nodes; @@ -203,10 +222,11 @@ pg_dump \ --table=temp_monthly_view_operations \ --table=temp_monthly_view_operations_reception \ --table=temp_monthly_view_operations_inclusion \ + --table=temp_monthly_view_endorsing_rights \ + --table=temp_monthly_view_dal_shard_assignments \ --table=temp_monthly_view_delegates \ --table=temp_monthly_view_cycles \ --table=temp_monthly_view_missing_blocks \ - --table=temp_monthly_view_endorsing_rights \ --table=temp_monthly_view_nodes exit_code_3=$? @@ -218,10 +238,11 @@ DROP VIEW IF EXISTS monthly_view_blocks_reception CASCADE; DROP VIEW IF EXISTS monthly_view_operations CASCADE; DROP VIEW IF EXISTS monthly_view_operations_reception CASCADE; DROP VIEW IF EXISTS monthly_view_operations_inclusion CASCADE; +DROP VIEW IF EXISTS monthly_view_endorsing_rights CASCADE; +DROP VIEW IF EXISTS monthly_view_dal_shard_assignments CASCADE; DROP VIEW IF EXISTS monthly_view_delegates CASCADE; DROP VIEW IF EXISTS monthly_view_cycles CASCADE; DROP VIEW IF EXISTS monthly_view_missing_blocks CASCADE; -DROP VIEW IF EXISTS monthly_view_endorsing_rights CASCADE; DROP VIEW IF EXISTS monthly_view_nodes CASCADE; DROP VIEW IF EXISTS monthly_view_data_summary; @@ -230,10 +251,11 @@ DROP TABLE IF EXISTS temp_monthly_view_blocks_reception; DROP TABLE IF EXISTS temp_monthly_view_operations; DROP TABLE IF EXISTS temp_monthly_view_operations_reception; DROP TABLE IF EXISTS temp_monthly_view_operations_inclusion; +DROP TABLE IF EXISTS temp_monthly_view_endorsing_rights; +DROP TABLE IF EXISTS temp_monthly_view_dal_shard_assignments; DROP TABLE IF EXISTS temp_monthly_view_delegates; DROP TABLE IF EXISTS temp_monthly_view_cycles; DROP TABLE IF EXISTS temp_monthly_view_missing_blocks; -DROP TABLE IF EXISTS temp_monthly_view_endorsing_rights; DROP TABLE IF EXISTS temp_monthly_view_nodes; EOF