diff --git a/.gitignore b/.gitignore index 6cc22a249cd0979eda4e2568bd9af2f1e2f7e02e..b4ec24c046e078872363905be38d7379619ecc0a 100644 --- a/.gitignore +++ b/.gitignore @@ -127,6 +127,7 @@ tx_kernel.wasm tx_kernel_dal.wasm tx-demo-collector dal_echo_kernel.wasm +dal_echo_kernel_bandwidth.wasm sequencer.wasm evm_benchmark_kernel.wasm evm_unstripped_installer.wasm diff --git a/kernels.mk b/kernels.mk index aab344ef36641ec2d98a3b1e28bd73644ab9ba69..3a4f6114f3932c45bad81c3013f88c23eacbf8ce 100644 --- a/kernels.mk +++ b/kernels.mk @@ -4,7 +4,7 @@ # # SPDX-License-Identifier: MIT -KERNELS=tx_kernel.wasm tx_kernel_dal.wasm dal_echo_kernel.wasm +KERNELS=tx_kernel.wasm tx_kernel_dal.wasm dal_echo_kernel.wasm dal_echo_kernel_bandwidth.wasm SDK_DIR=src/kernel_sdk DEMO_DIR=src/kernel_tx_demo @@ -39,6 +39,10 @@ dal_echo_kernel.wasm: @cp src/kernel_dal_echo/target/wasm32-unknown-unknown/release/dal_echo_kernel.wasm $@ @wasm-strip $@ +dal_echo_kernel_bandwidth.wasm: + @make -C src/kernel_dal_echo dal_echo_kernel_bandwidth + @cp src/kernel_dal_echo/target/wasm32-unknown-unknown/release/dal_echo_kernel_bandwidth.wasm $@ + @wasm-strip $@ .PHONY: build build: ${KERNELS} kernel_sdk diff --git a/src/kernel_dal_echo/Cargo.lock b/src/kernel_dal_echo/Cargo.lock index 4152a7b2997b84d03f5c31b840a02390abd00f10..a897664e98bb35b7ad02019a3e58844cb46b9e84 100644 --- a/src/kernel_dal_echo/Cargo.lock +++ b/src/kernel_dal_echo/Cargo.lock @@ -210,6 +210,20 @@ dependencies = [ "tezos-smart-rollup-storage", ] +[[package]] +name = "dal_echo_kernel_bandwidth" +version = "0.1.0" +dependencies = [ + "tezos-smart-rollup", + "tezos-smart-rollup-core", + "tezos-smart-rollup-debug", + "tezos-smart-rollup-encoding", + "tezos-smart-rollup-entrypoint", + "tezos-smart-rollup-host", + "tezos-smart-rollup-mock", + "tezos-smart-rollup-storage", +] + [[package]] name = "der" version = "0.4.5" diff --git a/src/kernel_dal_echo/Cargo.toml b/src/kernel_dal_echo/Cargo.toml index c3e632078ee50a074f62bff1a1d3ffcd9bd9bc51..02ebbcc07f7110685a083f7a945a508032e78698 100644 --- a/src/kernel_dal_echo/Cargo.toml +++ b/src/kernel_dal_echo/Cargo.toml @@ -5,7 +5,7 @@ [workspace] -members = ["kernel"] +members = ["kernel", "kernel_bandwidth"] [workspace.dependencies] tezos-smart-rollup = { path = "../kernel_sdk/sdk", features = ["proto-alpha"] } diff --git a/src/kernel_dal_echo/Makefile b/src/kernel_dal_echo/Makefile index f189754ec759813bc850894a4fbb8025f0b8215a..731253ecfa7d94714fb4ce01a5b25dbeb4358549 100644 --- a/src/kernel_dal_echo/Makefile +++ b/src/kernel_dal_echo/Makefile @@ -14,6 +14,10 @@ all: build test check dal_echo_kernel: @cargo build --target wasm32-unknown-unknown --release -p dal_echo_kernel +.PHONY: dal_echo_kernel_bandwidth +dal_echo_kernel_bandwidth: + @cargo build --target wasm32-unknown-unknown --release -p dal_echo_kernel_bandwidth + .PHONY: build build: dal_echo_kernel diff --git a/src/kernel_dal_echo/README.md b/src/kernel_dal_echo/README.md index 59bfb0fa413497bd2f3fcc6dac4d3cdcb35f171e..bc485b1f401ad83f45a26bc609181c854ee1c163 100644 --- a/src/kernel_dal_echo/README.md +++ b/src/kernel_dal_echo/README.md @@ -7,3 +7,19 @@ A simple kernel for testing the DAL. At each level, it will download all pages p ``` sh make build ``` + +## Bandwidth + +`kernel_bandwidth` provides an alternative version of the DAL echo kernel that +reads slots but only writes the number of bytes read per slot instead of the +content. Slots to be read are set through a configuration file of the form: + +``` +instructions: + - set: + value: + to: /slots +``` + +where the bit vector is an hexadecimal encoding of an integer in little-endian, +where each of its bits represent a DAL slot. diff --git a/src/kernel_dal_echo/kernel/src/lib.rs b/src/kernel_dal_echo/kernel/src/lib.rs index b1083f538ea0809b9394a4fa552f670b48d79872..7efaee46e297ae14e4527e142c5bbb5ebbcbd70f 100644 --- a/src/kernel_dal_echo/kernel/src/lib.rs +++ b/src/kernel_dal_echo/kernel/src/lib.rs @@ -48,6 +48,7 @@ fn process_slot( ); let slot_path = format!("/output/slot-{}", slot_index); let path: OwnedPath = slot_path.as_bytes().to_vec().try_into().unwrap(); + host.store_write(&path, &buffer, 0) .map_err(|_| "Error writing to storage".to_string()) .unwrap_or_default(); @@ -87,7 +88,7 @@ pub fn entry(host: &mut impl Runtime) { let parameters = host.reveal_dal_parameters(); debug_msg!(host, "Running kernel with parameters: {:?}\n", parameters); let RollupDalParameters { - number_of_slots: _, + number_of_slots: _number_of_slots, attestation_lag, slot_size, page_size, diff --git a/src/kernel_dal_echo/kernel_bandwidth/Cargo.toml b/src/kernel_dal_echo/kernel_bandwidth/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..c0cb384fd8050d379d46b23806be6601b1f3d0f4 --- /dev/null +++ b/src/kernel_dal_echo/kernel_bandwidth/Cargo.toml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: 2022-2023 TriliTech +# +# SPDX-License-Identifier: MIT + +[package] +name = "dal_echo_kernel_bandwidth" +version = "0.1.0" +edition = "2021" +license = "MIT" + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +tezos-smart-rollup.workspace = true +tezos-smart-rollup-core.workspace = true +tezos-smart-rollup-host.workspace = true +tezos-smart-rollup-debug.workspace = true +tezos-smart-rollup-entrypoint.workspace = true +tezos-smart-rollup-storage.workspace = true +tezos-smart-rollup-encoding.workspace = true +tezos-smart-rollup-mock.workspace = true + +[features] +default = [] +testing = [] diff --git a/src/kernel_dal_echo/kernel_bandwidth/src/lib.rs b/src/kernel_dal_echo/kernel_bandwidth/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..563b057ed255a464fdc2262f7f8116a1fd157c54 --- /dev/null +++ b/src/kernel_dal_echo/kernel_bandwidth/src/lib.rs @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: 2023 Marigold +// SPDX-FileCopyrightText: 2023 Nomadic Labs +// +// SPDX-License-Identifier: MIT + +use tezos_smart_rollup::entrypoint; +use tezos_smart_rollup_debug::debug_msg; +use tezos_smart_rollup_host::dal_parameters::RollupDalParameters; +use tezos_smart_rollup_host::path::{OwnedPath, RefPath}; +use tezos_smart_rollup_host::runtime::Runtime; + +fn process_slot( + host: &mut impl Runtime, + published_level: i32, + num_pages: usize, + page_size: usize, + slot_index: u8, +) { + let mut buffer = vec![0u8; page_size * num_pages]; + + let mut read_bytes = 0; + + for page_index in 0..num_pages { + let result = host.reveal_dal_page( + published_level, + slot_index, + page_index.try_into().unwrap(), + &mut buffer[page_index * page_size..(page_index + 1) * page_size], + ); + + match result { + Ok(0) => { + // Currently, we send empty pages to kernels for non-attested slots. + debug_msg!( + host, + "Slot {} not attested for level {}\n", + slot_index, + published_level + ); + return; + } + Ok(num) => { + debug_msg!( + host, + "Retrieved page {} for level {}, slot index {} successfully. {} bytes read\n", + page_index, + published_level, + slot_index, + num + ); + read_bytes += num; + } + Err(err) => { + debug_msg!( + host, + "Failed to retrieve one of the pages. Slot {} not processed. Error: {}\n", + slot_index, + &err.to_string() + ); + // Stop fetching pages on error + return; + } + } + } + + let slot_path = format!("/output/slot-{}", slot_index); + let path: OwnedPath = slot_path.as_bytes().to_vec().try_into().unwrap(); + host.store_write(&path, &read_bytes.to_le_bytes(), 0) + .map_err(|_| "Error writing to storage".to_string()) + .unwrap_or_default(); +} + +fn get_slots_from_storage(host: &impl Runtime, number_of_slots: u64) -> Vec { + let path = RefPath::assert_from("/slots".as_bytes()); + let bitvec_bytes = host.store_read_all(&path).unwrap(); + + let mut slots = Vec::new(); + for (i, byte) in bitvec_bytes.iter().enumerate() { + let slots_start = i * 8; + // Stop early if the slots are out of bounds, which can happen with + // padding in the bitvector. + if slots_start >= number_of_slots as usize { + continue; + } + for bit in 0..8 { + let slot = slots_start + bit; + // Stop early if the slot is out of bound, due to padding again. + if slot >= number_of_slots as usize { + continue; + } else if byte >> bit & 1 > 0 { + slots.push(slot as u8) + } + } + } + slots +} + +#[entrypoint::main] +pub fn entry(host: &mut impl Runtime) { + let parameters = host.reveal_dal_parameters(); + debug_msg!(host, "Running kernel with parameters: {:?}\n", parameters); + let RollupDalParameters { + number_of_slots, + attestation_lag, + slot_size, + page_size, + } = parameters; + + let slot_indexes: Vec = get_slots_from_storage(host, number_of_slots); + + match host.read_input() { + Ok(Some(message)) => { + let level = message.level; + let published_level = (level as i32) - (attestation_lag as i32); + let num_pages = (slot_size / page_size) as usize; + for slot_index in slot_indexes { + process_slot( + host, + published_level, + num_pages, + page_size as usize, + slot_index, + ); + } + } + Ok(None) => { + debug_msg!(host, "Input message was empty"); + } + Err(_) => { + debug_msg!(host, "Failed to read input message"); + } + } +} diff --git a/tezt/lib_tezos/constant.ml b/tezt/lib_tezos/constant.ml index 4430fe62b2d3c0557a96dff1ee28e1a6ed841ff4..d1108954dfd5a33121c9d93c8f6aae20f979f8a0 100644 --- a/tezt/lib_tezos/constant.ml +++ b/tezt/lib_tezos/constant.ml @@ -115,6 +115,12 @@ module WASM = struct let dal_echo_kernel = Uses.make ~tag:"dal_echo_kernel" ~path:"dal_echo_kernel.wasm" () + let dal_echo_kernel_bandwidth = + Uses.make + ~tag:"dal_echo_kernel_bandwidth" + ~path:"dal_echo_kernel_bandwidth.wasm" + () + let debug_kernel = Uses.make ~tag:"debug_kernel" diff --git a/tezt/lib_wrapper/expected/tezt_wrapper.ml/runtime-dependency-tags.out b/tezt/lib_wrapper/expected/tezt_wrapper.ml/runtime-dependency-tags.out index 286edf2100cf49b20fb7fba3a13ec820aae23e0f..d06d56ee55dcb6e0c9ea86429ec8e739f48ba81e 100644 --- a/tezt/lib_wrapper/expected/tezt_wrapper.ml/runtime-dependency-tags.out +++ b/tezt/lib_wrapper/expected/tezt_wrapper.ml/runtime-dependency-tags.out @@ -2,6 +2,7 @@ injector_server: _build/default/contrib/octez_injector_server/octez_injector_ser yes_wallet: _build/default/devtools/yes_wallet/yes_wallet.exe p2p_node: _build/default/src/bin_p2p_node/main_p2p_node.exe dal_echo_kernel: dal_echo_kernel.wasm +dal_echo_kernel_bandwidth: dal_echo_kernel_bandwidth.wasm etherlink_governance_observer: etherlink-governance-observer debug_kernel: etherlink/kernel_latest/kernel/tests/resources/debug_kernel.wasm failed_migration: etherlink/kernel_latest/kernel/tests/resources/failed_migration.wasm diff --git a/tezt/tests/dal.ml b/tezt/tests/dal.ml index bcd5ccfead5aadf1420db132d40c1cd878d916cd..3e07433abe0c9e4438cf91222b31b628cf9ff12b 100644 --- a/tezt/tests/dal.ml +++ b/tezt/tests/dal.ml @@ -7286,7 +7286,7 @@ module Tx_kernel_e2e = struct Check.( (String.length value = parameters.Dal.Parameters.cryptobox.slot_size) int - ~error_msg:"Expected a value of size %R. Got $L") ; + ~error_msg:"Expected a value of size %R. Got %L") ; if String.starts_with ~prefix:payload value then unit else let message = @@ -7297,6 +7297,83 @@ module Tx_kernel_e2e = struct (String.sub value 0 (String.length payload)) in Test.fail "%s" message + + let test_manual_echo_kernel_for_bandwidth _protocol parameters dal_node + sc_rollup_node _sc_rollup_address node client pvm_name = + Log.info "Originate the echo kernel." ; + let config = + Sc_rollup_helpers.Installer_kernel_config. + [ + Set + { + to_ = "/slots"; + (* Only the slot 0 *) + value = Z.to_bits Z.one |> Hex.of_string |> Hex.show; + }; + ] + in + let* {boot_sector; _} = + Sc_rollup_helpers.prepare_installer_kernel + ~config:(`Config config) + ~preimages_dir: + (Filename.concat (Sc_rollup_node.data_dir sc_rollup_node) pvm_name) + Constant.WASM.dal_echo_kernel_bandwidth + in + let* sc_rollup_address = + Client.Sc_rollup.originate + ~burn_cap:Tez.(of_int 9999999) + ~alias:"dal_echo_kernel_bandwidth" + ~src:Constant.bootstrap1.public_key_hash + ~kind:pvm_name + ~boot_sector + ~parameters_ty:"unit" + client + in + let* () = bake_for client in + let* () = + Sc_rollup_node.run sc_rollup_node sc_rollup_address [Log_kernel_debug] + in + let* current_level = Node.get_level node in + let target_level = + current_level + parameters.Dal.Parameters.attestation_lag + 1 + in + let payload = "hello" in + let* () = + publish_store_and_attest_slot + client + node + dal_node + Constant.bootstrap1 + ~index:0 + ~content: + (Helpers.make_slot + ~slot_size:parameters.Dal.Parameters.cryptobox.slot_size + payload) + ~attestation_lag:parameters.attestation_lag + ~number_of_slots:parameters.number_of_slots + in + Log.info "Wait for the rollup node to catch up." ; + let* _level = + Sc_rollup_node.wait_for_level ~timeout:30. sc_rollup_node target_level + in + let key = "/output/slot-0" in + let* value_written = + Sc_rollup_node.RPC.call sc_rollup_node ~rpc_hooks + @@ Sc_rollup_rpc.get_global_block_durable_state_value + ~pvm_kind:pvm_name + ~operation:Sc_rollup_rpc.Value + ~key + () + in + match value_written with + | None -> Test.fail "Expected a value to be found. But none was found." + | Some value -> + let value = `Hex value |> Hex.to_string |> Z.of_bits in + Check.( + (Z.to_int value = parameters.cryptobox.slot_size) + int + ~error_msg:"Expected number of bytes length %R. Got %L") ; + unit end module Profiler = Tezos_profiler.Profiler @@ -11066,6 +11143,21 @@ let register ~protocols = ~operator_profiles:[0] Tx_kernel_e2e.test_echo_kernel_e2e protocols ; + (* This test only asserts the echo kernel used by tezt-cloud scenarios is + correct. It isn't meant to be ran by the CI. *) + scenario_with_all_nodes + "test echo_kernel_for_bandwidth" + ~uses:(fun _protocol -> + [Constant.smart_rollup_installer; Constant.WASM.dal_echo_kernel_bandwidth]) + ~pvm_name:"wasm_2_0_0" + ~slot_size:2048 + ~page_size:256 + ~number_of_shards:64 + ~operator_profiles:[0] + ~tags:[Tag.ci_disabled] + ~regression:false + Tx_kernel_e2e.test_manual_echo_kernel_for_bandwidth + protocols ; (* Register tutorial test *) scenario_tutorial_dal_baker protocols ;