diff --git a/etherlink/lib_wasm_runtime/Cargo.lock b/etherlink/lib_wasm_runtime/Cargo.lock index 96888c0cd6f5711581ec15054b6eb40ce4d37500..c75c812e5c85f195ad16e60297616719d6de93e5 100644 --- a/etherlink/lib_wasm_runtime/Cargo.lock +++ b/etherlink/lib_wasm_runtime/Cargo.lock @@ -3335,6 +3335,7 @@ dependencies = [ "hex", "ieee-apsqrt", "itertools", + "lazy_static", "num_enum", "paste", "range-collections", diff --git a/src/riscv/Cargo.lock b/src/riscv/Cargo.lock index 7ae6ba9938c31187778a93c6173dc088e19c12ea..e7015f3a6ab4d029bccd1dc2ef71aa3272849cbe 100644 --- a/src/riscv/Cargo.lock +++ b/src/riscv/Cargo.lock @@ -1579,6 +1579,7 @@ dependencies = [ "thiserror", "tracing", "try-blocks", + "tuples", "vm-fdt", ] @@ -2833,6 +2834,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "296cc7892cc05ae83e113b20c113d3fd9020eac7abbbaaeaf69c424ef872be7a" +[[package]] +name = "tuples" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dd6c24c6d32b904abc35343bf05fde6e4a88980b3a7331a85c2a0d5447d53e2" + [[package]] name = "typenum" version = "1.17.0" diff --git a/src/riscv/api/src/lib.rs b/src/riscv/api/src/lib.rs index 89391d2f137cec1449050e692e5c12c41c42db6b..2810dff3af9f473f51b27e2bb4b8f00f085e6f9e 100644 --- a/src/riscv/api/src/lib.rs +++ b/src/riscv/api/src/lib.rs @@ -10,6 +10,7 @@ use std::fs; use std::ops::Deref; use std::ops::DerefMut; use std::str; +use std::str::Bytes; use arbitrary_int::u31; use move_semantics::ImmutableState; @@ -22,11 +23,20 @@ use octez_riscv::pvm::PvmHooks; use octez_riscv::pvm::PvmInput; use octez_riscv::pvm::PvmStatus; use octez_riscv::pvm::node_pvm::NodePvm; +use octez_riscv::pvm::node_pvm::NodePvmLayout; use octez_riscv::pvm::node_pvm::PvmStorage; use octez_riscv::pvm::node_pvm::PvmStorageError; +use octez_riscv::state_backend::ProofLayout; +use octez_riscv::state_backend::ProofPart; +use octez_riscv::state_backend::ProofTree; use octez_riscv::state_backend::owned_backend::Owned; use octez_riscv::state_backend::proof_backend::proof; +use octez_riscv::state_backend::proof_backend::proof::MerkleProof; +use octez_riscv::state_backend::proof_backend::proof::MerkleProofLeaf; +use octez_riscv::state_backend::proof_backend::proof::deser_stream::StreamShapeContext; +use octez_riscv::state_backend::proof_backend::proof::deserialise_into_backend; use octez_riscv::state_backend::proof_backend::proof::serialise_proof; +use octez_riscv::state_backend::verify_backend::Verifier; use octez_riscv::storage; use octez_riscv::storage::StorageError; use pointer_apply::ApplyReadOnly; @@ -490,22 +500,21 @@ pub unsafe fn octez_riscv_storage_export_snapshot( }) } -/// Proofs -#[ocaml::sig] -pub struct Proof(proof::Proof); +use crate::storage::Hash; -impl Deref for Proof { - type Target = proof::Proof; - - fn deref(&self) -> &Self::Target { - &self.0 - } +struct ProtocolProof { + pvm: NodePvm, + merkle_proof_tree: MerkleProof, + serialisation: Box<[u8]>, + final_state_hash: Hash, } -impl DerefMut for Proof { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } +// TODO: Move splitting implementations at the backend.ml layer +/// Proofs +#[ocaml::sig] +enum Proof { + Protocol(ProtocolProof), + RollupNode(proof::Proof), } ocaml::custom!(Proof); @@ -513,13 +522,19 @@ ocaml::custom!(Proof); #[ocaml::func] #[ocaml::sig("proof -> bytes")] pub fn octez_riscv_proof_start_state(proof: Pointer) -> [u8; 32] { - proof.as_ref().initial_state_hash().into() + match proof.as_ref() { + Proof::Protocol(_) => todo!("RV-XXX: Protocol can't obtain start state of proof"), + Proof::RollupNode(proof) => proof.initial_state_hash().into(), + } } #[ocaml::func] #[ocaml::sig("proof -> bytes")] pub fn octez_riscv_proof_stop_state(proof: Pointer) -> [u8; 32] { - proof.as_ref().final_state_hash().into() + match proof.as_ref() { + Proof::Protocol(proof) => proof.final_state_hash.into(), + Proof::RollupNode(proof) => proof.final_state_hash().into(), + } } #[ocaml::func] @@ -530,7 +545,7 @@ pub fn octez_riscv_produce_proof( ) -> Option> { let input = input.map(|i| i.into()); let proof = state.apply_ro(|pvm| NodePvm::produce_proof(pvm, input, &mut PvmHooks::default())); - proof.map(|p| Pointer::from(Proof(p))) + proof.map(|proof| Pointer::from(Proof::RollupNode(proof))) } #[ocaml::func] @@ -545,7 +560,20 @@ pub fn octez_riscv_verify_proof( proof: Pointer, ) -> Option> { let input = input.map(|i| i.into()); - let input_request = NodePvm::verify_proof(proof.as_ref(), input, &mut PvmHooks::default()); + let (mut pvm, final_state_hash, merkle_tree) = match proof.as_ref() { + Proof::RollupNode(proof) => todo!(), + Proof::Protocol(proof) => ( + proof.pvm.clone(), + proof.final_state_hash, + &proof.merkle_proof_tree, + ), + }; + let input_request = pvm.verify( + input, + final_state_hash, + ProofPart::Present(merkle_tree), + &mut PvmHooks::default(), + ); // TODO: RV-556: Return an `InputRequest` todo!() @@ -553,17 +581,31 @@ pub fn octez_riscv_verify_proof( #[ocaml::func] #[ocaml::sig("proof -> bytes")] -pub unsafe fn octez_riscv_serialise_proof(proof: Pointer) -> ocaml::Value { +pub unsafe fn octez_riscv_serialise_proof(proof: Pointer) -> BytesWrapper { // Safety: the function needs to return the same `ocaml::Value` as in the signature. // In this case, an OCaml bytes value has to be returned. - let serialisation: Vec = serialise_proof(proof.as_ref()).collect(); - unsafe { ocaml::Value::bytes(serialisation) } + let bytes = match proof.as_ref() { + Proof::Protocol(proof) => proof.serialisation.clone(), + Proof::RollupNode(proof) => serialise_proof(proof).collect(), + }; + BytesWrapper(bytes) } #[ocaml::func] #[ocaml::sig("bytes -> (proof, string) Result.t")] -pub fn octez_riscv_deserialise_proof(_bytes: &[u8]) -> Result, String> { - todo!("RV-555") +pub fn octez_riscv_deserialise_proof(bytes: &[u8]) -> Result, String> { + let (final_state_hash, space) = + deserialise_into_backend::(bytes.into_iter().cloned()) + .map_err(|e| e.to_string())?; + let pvm = NodePvm::bind(space); + let proof = Proof::Protocol(ProtocolProof { + pvm, + // TODO: RV-XXX: ProofTree should either be obtained here (by parsing) or made part of the verifier backend + merkle_proof_tree: MerkleProof::Leaf(MerkleProofLeaf::Read(vec![])), + serialisation: bytes.into(), + final_state_hash, + }); + Ok(proof.into()) } #[ocaml::func] diff --git a/src/riscv/lib/Cargo.toml b/src/riscv/lib/Cargo.toml index ad714349f1a3854ea59247efd78709f0d0459259..e45a290f2c9744f329a73f531d43ed2cfafb3f30 100644 --- a/src/riscv/lib/Cargo.toml +++ b/src/riscv/lib/Cargo.toml @@ -35,6 +35,7 @@ vm-fdt.workspace = true itertools.workspace = true range-collections.workspace = true elf.workspace = true +tuples = "1.16.0" [dependencies.__tracing_do_not_use_directly] workspace = true diff --git a/src/riscv/lib/src/cache_utils.rs b/src/riscv/lib/src/cache_utils.rs index 6398316f708b4dca9620d41e8ff52bfb0252f2e8..d1945c614dd779b02a208c8fd60425051d449b75 100644 --- a/src/riscv/lib/src/cache_utils.rs +++ b/src/riscv/lib/src/cache_utils.rs @@ -10,13 +10,17 @@ use crate::default::ConstDefault; use crate::machine_state::memory::Address; use crate::state_backend::AllocatedOf; use crate::state_backend::CommitmentLayout; +use crate::state_backend::FromProofError; use crate::state_backend::FromProofResult; use crate::state_backend::Layout; use crate::state_backend::ManagerBase; use crate::state_backend::ManagerSerialise; use crate::state_backend::Many; +use crate::state_backend::ProofContextLayout; use crate::state_backend::ProofLayout; use crate::state_backend::ProofTree; +use crate::state_backend::proof_backend::proof::deser_proof::DeserProof; +use crate::state_backend::verify_backend::Verifier; use crate::storage::Hash; use crate::storage::HashError; @@ -120,3 +124,15 @@ impl ProofLayou Many::::partial_state_hash(state, proof) } } + +impl ProofContextLayout + for Sizes +where + AllocatedOf: 'static, +{ + fn from_proof_comp( + proof: DP, + ) -> Result>, FromProofError> { + as ProofContextLayout>::from_proof_comp(proof) + } +} diff --git a/src/riscv/lib/src/machine_state/block_cache.rs b/src/riscv/lib/src/machine_state/block_cache.rs index 3239ddede7d70a2a62435b923741ca220ce710cf..2b51784f716a9769c02b39f8d67d6efafe0414b8 100644 --- a/src/riscv/lib/src/machine_state/block_cache.rs +++ b/src/riscv/lib/src/machine_state/block_cache.rs @@ -115,6 +115,7 @@ use crate::state_backend::ManagerSerialise; use crate::state_backend::ManagerWrite; use crate::state_backend::Ref; use crate::state_backend::proof_backend; +use crate::state_backend::proof_backend::proof::deser_proof::DeserProof; use crate::storage::Hash; use crate::storage::HashError; use crate::traps::EnvironException; @@ -208,6 +209,14 @@ impl state_backend::ProofLayout for AddressCellLayout { } } +impl state_backend::ProofContextLayout for AddressCellLayout { + fn from_proof_comp( + proof: DP, + ) -> Result>, state_backend::FromProofError> { + as state_backend::ProofContextLayout>::from_proof_comp(proof) + } +} + /// The layout of block cache entries, see [`Cached`] for more information. pub type CachedLayout = ( AddressCellLayout, diff --git a/src/riscv/lib/src/machine_state/csregisters/values.rs b/src/riscv/lib/src/machine_state/csregisters/values.rs index 95a114a7de7e8236cbdd9810db86694197b29a96..752de3f96e96d3756cfd35fc183483cb23191171 100644 --- a/src/riscv/lib/src/machine_state/csregisters/values.rs +++ b/src/riscv/lib/src/machine_state/csregisters/values.rs @@ -43,6 +43,8 @@ use crate::state_backend::hash::HashError; use crate::state_backend::owned_backend::Owned; use crate::state_backend::proof_backend::merkle::AccessInfoAggregatable; use crate::state_backend::proof_backend::merkle::MerkleTree; +use crate::state_backend::proof_backend::proof::deser_proof::DeserProof; +use crate::state_backend::proof_backend::proof::deser_proof::PartialResult; use crate::state_backend::verify_backend; use crate::storage::binary; @@ -171,6 +173,36 @@ impl CSRegisters { /// Layout for the values of CSRs pub struct CSRValuesLayout; +impl CSRValuesLayout { + fn make_absent() -> AllocatedOf { + CSRValuesF::new_with( + || mstatus::MStatusLayoutF { + sie: Cell::absent(), + mie: Cell::absent(), + spie: Cell::absent(), + ube: Cell::absent(), + mpie: Cell::absent(), + spp: Cell::absent(), + mpp: Cell::absent(), + fs: Cell::absent(), + xs: Cell::absent(), + mprv: Cell::absent(), + sum: Cell::absent(), + mxr: Cell::absent(), + tvm: Cell::absent(), + tw: Cell::absent(), + tsr: Cell::absent(), + uxl: Cell::absent(), + sxl: Cell::absent(), + sbe: Cell::absent(), + mbe: Cell::absent(), + }, + || (), + || Cell::bind(verify_backend::Region::Absent), + ) + } +} + impl Layout for CSRValuesLayout { type Allocated = CSRValuesF< AllocatedOf, M>, @@ -192,37 +224,9 @@ impl ProofLayout for CSRValuesLayout { } fn from_proof(proof: ProofTree) -> FromProofResult { - fn make_absent() -> AllocatedOf { - CSRValuesF::new_with( - || mstatus::MStatusLayoutF { - sie: Cell::absent(), - mie: Cell::absent(), - spie: Cell::absent(), - ube: Cell::absent(), - mpie: Cell::absent(), - spp: Cell::absent(), - mpp: Cell::absent(), - fs: Cell::absent(), - xs: Cell::absent(), - mprv: Cell::absent(), - sum: Cell::absent(), - mxr: Cell::absent(), - tvm: Cell::absent(), - tw: Cell::absent(), - tsr: Cell::absent(), - uxl: Cell::absent(), - sxl: Cell::absent(), - sbe: Cell::absent(), - mbe: Cell::absent(), - }, - || (), - || Cell::bind(verify_backend::Region::Absent), - ) - } - let leaf = proof.into_leaf()?; let cell = match leaf { - ProofPart::Absent => make_absent(), + ProofPart::Absent => Self::make_absent(), ProofPart::Present(data) => { let values: AllocatedOf = binary::deserialise(data)?; values.map( @@ -293,6 +297,49 @@ impl ProofLayout for CSRValuesLayout { } } +use crate::state_backend::ProofContextLayout; +use crate::state_backend::proof_layout; + +impl ProofContextLayout for CSRValuesLayout { + fn from_proof_comp( + proof: DP, + ) -> Result>, proof_layout::FromProofError> { + use crate::state_backend::proof_backend::proof::deser_proof::Suspended; + let ctx = proof.as_leaf::>()?; + + let handler = ctx.map(move |data| match data { + PartialResult::Absent | PartialResult::Blinded(_) => Ok(Self::make_absent()), + PartialResult::Present(data) => Ok(data.map( + |mstatus| mstatus::MStatusLayoutF { + sie: Cell::from_owned(mstatus.sie), + mie: Cell::from_owned(mstatus.mie), + spie: Cell::from_owned(mstatus.spie), + ube: Cell::from_owned(mstatus.ube), + mpie: Cell::from_owned(mstatus.mpie), + spp: Cell::from_owned(mstatus.spp), + mpp: Cell::from_owned(mstatus.mpp), + fs: Cell::from_owned(mstatus.fs), + xs: Cell::from_owned(mstatus.xs), + mprv: Cell::from_owned(mstatus.mprv), + sum: Cell::from_owned(mstatus.sum), + mxr: Cell::from_owned(mstatus.mxr), + tvm: Cell::from_owned(mstatus.tvm), + tw: Cell::from_owned(mstatus.tw), + tsr: Cell::from_owned(mstatus.tsr), + uxl: Cell::from_owned(mstatus.uxl), + sxl: Cell::from_owned(mstatus.sxl), + sbe: Cell::from_owned(mstatus.sbe), + mbe: Cell::from_owned(mstatus.mbe), + }, + |()| (), + Cell::from_owned, + )), + }); + + Ok(handler) + } +} + #[derive(Default, Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct CSRValuesF { pub mstatus: MStatus, diff --git a/src/riscv/lib/src/machine_state/memory/buddy.rs b/src/riscv/lib/src/machine_state/memory/buddy.rs index 8d7016f984ba44a0bed18653af1d395f767f2344..c36ee84e9721802cf295ab6536d3af051afe62d5 100644 --- a/src/riscv/lib/src/machine_state/memory/buddy.rs +++ b/src/riscv/lib/src/machine_state/memory/buddy.rs @@ -31,11 +31,12 @@ use crate::state_backend::ManagerBase; use crate::state_backend::ManagerClone; use crate::state_backend::ManagerRead; use crate::state_backend::ManagerReadWrite; +use crate::state_backend::ProofContextLayout; use crate::state_backend::ProofLayout; use crate::state_backend::Ref; /// Layout for a Buddy-style memory manager -pub trait BuddyLayout: CommitmentLayout + ProofLayout { +pub trait BuddyLayout: CommitmentLayout + ProofLayout + ProofContextLayout { /// Buddy-style memory manager implementation type Buddy: Buddy; diff --git a/src/riscv/lib/src/machine_state/memory/buddy/branch.rs b/src/riscv/lib/src/machine_state/memory/buddy/branch.rs index 56bc99a48a2587bea5a43f08f050063c32c8f066..61717abe04230cf2c9c5da23f06406a83b3fed14 100644 --- a/src/riscv/lib/src/machine_state/memory/buddy/branch.rs +++ b/src/riscv/lib/src/machine_state/memory/buddy/branch.rs @@ -10,6 +10,7 @@ use serde::Serialize; use super::Buddy; use super::BuddyLayout; use crate::state::NewState; +use crate::state_backend::AllocatedOf; use crate::state_backend::Atom; use crate::state_backend::Cell; use crate::state_backend::FnManager; @@ -21,6 +22,7 @@ use crate::state_backend::ManagerRead; use crate::state_backend::ManagerReadWrite; use crate::state_backend::ManagerSerialise; use crate::state_backend::Ref; +use crate::state_backend::verify_backend::Verifier; use crate::struct_layout; /// Information about what is free in each buddy @@ -53,7 +55,11 @@ struct_layout! { } } -impl BuddyLayout for BuddyBranch2Layout { +impl BuddyLayout for BuddyBranch2Layout +where + B: 'static, + AllocatedOf: 'static, // B: 'static, +{ type Buddy = BuddyBranch2, M>; fn bind(space: Self::Allocated) -> Self::Buddy { diff --git a/src/riscv/lib/src/machine_state/memory/buddy/branch_combinations.rs b/src/riscv/lib/src/machine_state/memory/buddy/branch_combinations.rs index c880c1262e31772014cfa7c540db1391fc3b20ba..77a58494e5ed8677c8688d13cfcccc256549b59a 100644 --- a/src/riscv/lib/src/machine_state/memory/buddy/branch_combinations.rs +++ b/src/riscv/lib/src/machine_state/memory/buddy/branch_combinations.rs @@ -28,12 +28,17 @@ use crate::state_backend::ManagerRead; use crate::state_backend::ManagerReadWrite; use crate::state_backend::ManagerSerialise; use crate::state_backend::PartialHashError; +use crate::state_backend::ProofContextLayout; use crate::state_backend::ProofLayout; use crate::state_backend::ProofTree; use crate::state_backend::Ref; use crate::state_backend::RefProofGenOwnedAlloc; use crate::state_backend::RefVerifierAlloc; use crate::state_backend::proof_backend::merkle::MerkleTree; +use crate::state_backend::proof_backend::proof::deser_proof::DeserProof; +use crate::state_backend::proof_backend::proof::deser_proof::DeserResult; +use crate::state_backend::proof_backend::proof::deser_proof::Suspended; +use crate::state_backend::verify_backend::Verifier; use crate::storage::Hash; use crate::storage::HashError; @@ -123,7 +128,24 @@ macro_rules! combined_buddy_branch { } } - impl BuddyLayout for [<$name Layout>] { + impl ProofContextLayout for [<$name Layout>] + where + AllocatedOf<[<$buddy1 Layout>]<[<$buddy2 Layout>]>, Verifier>: 'static, + // AllocatedOf: 'static, + // AllocatedOf: 'static, + [<$buddy1 Layout>]<[<$buddy2 Layout>]>: ProofContextLayout + { + fn from_proof_comp(proof: DP) -> DeserResult>> { + let inner = <[<$buddy1 Layout>]<[<$buddy2 Layout>]>>::from_proof_comp(proof)?; + Ok(inner.map(|inner| Ok([<$name Alloc>](inner?)))) + } + } + + impl BuddyLayout for [<$name Layout>] + where [<$buddy1 Layout>]<[<$buddy2 Layout>]>: 'static, + + // where B: serde::de::DeserializeOwned + 'static + { type Buddy = $name, M>; fn bind(space: Self::Allocated) -> Self::Buddy { @@ -139,6 +161,7 @@ macro_rules! combined_buddy_branch { [<$name Alloc>](inner) } } + } /// Combined Buddy branch diff --git a/src/riscv/lib/src/machine_state/memory/buddy/leaf.rs b/src/riscv/lib/src/machine_state/memory/buddy/leaf.rs index a252d71c3eba80766a390efc1c6c016ba598ce84..dfbb3dba71e8ee2242057b3cf45a9bd2dd279ddd 100644 --- a/src/riscv/lib/src/machine_state/memory/buddy/leaf.rs +++ b/src/riscv/lib/src/machine_state/memory/buddy/leaf.rs @@ -12,6 +12,7 @@ use super::BuddyLayout; use crate::bits::ones; use crate::state::NewState; use crate::state_backend::AllocatedOf; +use crate::state_backend::Array; use crate::state_backend::Atom; use crate::state_backend::Cell; use crate::state_backend::CommitmentLayout; @@ -26,12 +27,14 @@ use crate::state_backend::ManagerRead; use crate::state_backend::ManagerReadWrite; use crate::state_backend::ManagerSerialise; use crate::state_backend::PartialHashError; +use crate::state_backend::ProofContextLayout; use crate::state_backend::ProofLayout; use crate::state_backend::ProofTree; use crate::state_backend::Ref; use crate::state_backend::RefProofGenOwnedAlloc; use crate::state_backend::RefVerifierAlloc; use crate::state_backend::proof_backend::merkle::MerkleTree; +use crate::state_backend::proof_backend::proof::deser_proof::Suspended; use crate::storage::Hash; use crate::storage::HashError; @@ -55,7 +58,7 @@ impl ProofLayout for BuddyLeafLayout { fn from_proof(proof: ProofTree) -> FromProofResult { Ok(Self::Allocated { - set: Atom::from_proof(proof)?, + set: as ProofLayout>::from_proof(proof)?, }) } @@ -67,6 +70,17 @@ impl ProofLayout for BuddyLeafLayout { } } +impl ProofContextLayout for BuddyLeafLayout { + fn from_proof_comp( + proof: DP, + ) -> crate::state_backend::proof_backend::proof::deser_proof::DeserResult< + DP::Susp>, + > { + let f = Atom::from_proof_comp(proof)?; + Ok(f.map(|cell| Ok(Self::Allocated { set: cell? }))) + } +} + impl BuddyLayout for BuddyLeafLayout { type Buddy = BuddyLeaf; diff --git a/src/riscv/lib/src/machine_state/memory/buddy/proxy.rs b/src/riscv/lib/src/machine_state/memory/buddy/proxy.rs index e871caab97267f8de31818970650e33c889524e7..afb0e1350b1eadb49249c612c27d714f6620a53c 100644 --- a/src/riscv/lib/src/machine_state/memory/buddy/proxy.rs +++ b/src/riscv/lib/src/machine_state/memory/buddy/proxy.rs @@ -18,6 +18,7 @@ use crate::state_backend::Layout; use crate::state_backend::ManagerBase; use crate::state_backend::ManagerSerialise; use crate::state_backend::PartialHashError; +use crate::state_backend::ProofContextLayout; use crate::state_backend::ProofLayout; use crate::state_backend::ProofTree; use crate::state_backend::Ref; @@ -66,6 +67,20 @@ where } } +impl ProofContextLayout for BuddyLayoutProxy +where + (): BuddyLayoutMatch, + // PickLayout: ProofContextLayout, +{ + fn from_proof_comp( + proof: DP, + ) -> crate::state_backend::proof_backend::proof::deser_proof::DeserResult< + DP::Susp>, + > { + as ProofContextLayout>::from_proof_comp(proof) + } +} + impl BuddyLayout for BuddyLayoutProxy where (): BuddyLayoutMatch, diff --git a/src/riscv/lib/src/pvm/common.rs b/src/riscv/lib/src/pvm/common.rs index 24bdb74843cc063202c07e6ae040f0070bd82ea5..4bfc3496371bbb556b80b820ba163f9e96120659 100644 --- a/src/riscv/lib/src/pvm/common.rs +++ b/src/riscv/lib/src/pvm/common.rs @@ -212,7 +212,7 @@ impl< /// Bind the block cache to the given allocated state and the given [block builder]. /// /// [block builder]: block::Block::BlockBuilder - pub(crate) fn bind( + pub fn bind( space: state_backend::AllocatedOf, M>, block_builder: B::BlockBuilder, ) -> Self diff --git a/src/riscv/lib/src/pvm/node_pvm.rs b/src/riscv/lib/src/pvm/node_pvm.rs index bdc102f59422bdcbf937f04991d536711b136445..b21797307369aa22c5b344ac65cfcbd469476ea2 100644 --- a/src/riscv/lib/src/pvm/node_pvm.rs +++ b/src/riscv/lib/src/pvm/node_pvm.rs @@ -22,10 +22,13 @@ use crate::state::NewState; use crate::state_backend; use crate::state_backend::AllocatedOf; use crate::state_backend::FnManagerIdent; +use crate::state_backend::FromProofError; +use crate::state_backend::ProofContextLayout; use crate::state_backend::ProofLayout; use crate::state_backend::ProofTree; use crate::state_backend::owned_backend::Owned; use crate::state_backend::proof_backend::proof::Proof; +use crate::state_backend::proof_backend::proof::deser_proof::DeserProof; use crate::state_backend::verify_backend::Verifier; use crate::storage; use crate::storage::Hash; @@ -39,7 +42,7 @@ pub enum PvmError { type NodePvmMemConfig = crate::machine_state::memory::M64M; -type NodePvmLayout = PvmLayout; +pub type NodePvmLayout = PvmLayout; type NodePvmState = Pvm, M>; @@ -197,17 +200,32 @@ impl NodePvm { impl NodePvm { /// Verify the proof with the given input. Upon success, return the input /// request which corresponds to the initial state of the proof. - pub fn verify_proof( - proof: &Proof, + // pub fn verify_proof( + // proof: &Proof, + // input: Option, + // pvm_hooks: &mut PvmHooks, + // ) -> Option<()> { + // let proof_tree = proof.tree(); + // let mut pvm = Pvm::from_proof(proof_tree, InterpretedBlockBuilder).map(|state| Self { + // state: Box::new(state), + // })?; + + // pvm.verify( + // input, + // proof.final_state_hash(), + // ProofTree::Present(proof_tree), + // pvm_hooks, + // ) + // } + + pub fn verify( + &mut self, input: Option, + expected_final_hash: Hash, + proof_tree: ProofTree, pvm_hooks: &mut PvmHooks, ) -> Option<()> { - let proof_tree = proof.tree(); - let mut pvm = Pvm::from_proof(proof_tree, InterpretedBlockBuilder).map(|state| Self { - state: Box::new(state), - })?; - - pvm.with_backend_mut(|pvm| { + self.with_backend_mut(|pvm| { match input { None => pvm.eval_one(pvm_hooks), Some(input) => { @@ -218,9 +236,8 @@ impl NodePvm { }; let refs = pvm.struct_ref::(); - let final_hash = - NodePvmLayout::partial_state_hash(refs, ProofTree::Present(proof_tree)).ok()?; - if final_hash != proof.final_state_hash() { + let final_hash = NodePvmLayout::partial_state_hash(refs, proof_tree).ok()?; + if final_hash != expected_final_hash { return None; } diff --git a/src/riscv/lib/src/state_backend/layout.rs b/src/riscv/lib/src/state_backend/layout.rs index 5b4f602c900b3f4d3e9d4155ab9268b16fc41d4d..e9db24b1fe1bab0bde87b0cb97223d560d43c71a 100644 --- a/src/riscv/lib/src/state_backend/layout.rs +++ b/src/riscv/lib/src/state_backend/layout.rs @@ -148,6 +148,49 @@ macro_rules! struct_layout { } } + impl < + $( + [<$field_name:camel>]: $crate::state_backend::ProofContextLayout + ),+ + > $crate::state_backend::ProofContextLayout for [<$layout_t F>]< + $( + [<$field_name:camel>] + ),+ + > + where + $( + $crate::state_backend::AllocatedOf<[<$field_name:camel>], $crate::state_backend::verify_backend::Verifier>: 'static + ),+ + { + #[inline] + fn from_proof_comp( + proof: DP, + ) -> Result< + DP::Susp<$crate::state_backend::FromProofResult>, + $crate::state_backend::FromProofError> { + + use $crate::tuple_branches_proof_context_layout; + use $crate::state_backend::proof_backend::proof::deser_proof::DeserError; + use $crate::state_backend::proof_backend::proof::deser_proof::DeserBranch; + + let ctx = proof + .as_node()?.map(|_unit_partial_res| Ok::<_, DeserError>(())); + + let ctx = tuple_branches_proof_context_layout!(ctx, [], $([<$field_name:camel>],)+); + + let ctx = ctx.map(|res| -> Result<_, DeserError> { + let ( $($field_name,)+ ) = res?; + Ok(Self::Allocated { + $( + $field_name + ),+ + }) + }); + + Ok(ctx.done()) + } + } + impl < $( [<$field_name:camel>]: $crate::state_backend::ProofLayout diff --git a/src/riscv/lib/src/state_backend/proof_backend/proof.rs b/src/riscv/lib/src/state_backend/proof_backend/proof.rs index 379bbae406d2a599fcf074fcc62c5df1cf4e2297..71e934191485db839cffd7b1f6e2e61d378a5b47 100644 --- a/src/riscv/lib/src/state_backend/proof_backend/proof.rs +++ b/src/riscv/lib/src/state_backend/proof_backend/proof.rs @@ -14,15 +14,25 @@ //! //! - Convert [`super::merkle::MerkleTree`] to [`MerkleProof`] +use deser_stream::StreamShapeContext; use itertools::Itertools; use super::tree::ModifyResult; use super::tree::Tree; use super::tree::impl_modify_map_collect; +use crate::state_backend::FromProofError; +use crate::state_backend::FromProofResult; +use crate::state_backend::Layout; +use crate::state_backend::ProofContextLayout; use crate::state_backend::hash::Hash; +use crate::state_backend::verify_backend::Verifier; use crate::storage::DIGEST_SIZE; use crate::storage::HashError; +pub mod deser_owned; +pub mod deser_proof; +pub mod deser_stream; + /// Structure of a proof transitioning from state A to state B. /// /// The proof needs to be able to: @@ -186,6 +196,15 @@ impl From for Tag { } } +impl Tag { + pub fn ordered_tags_from_u8(byte: u8) -> Vec> { + [0, 2, 4, 6] + .map(|offset| (byte >> offset) & TAG_MASK) + .map(Tag::try_from) + .into() + } +} + fn serialise_raw_tags(raw_tags: impl Iterator) -> Vec { // Tag serialisation to bytes depends on the number of bits required to hold a raw tag // Here, a raw tag is 2 bits wide, hence we use 8 / 2 = 4 chunks @@ -232,11 +251,15 @@ pub fn serialise_proof(proof: &Proof) -> impl Iterator + '_ { .chain(nodes_encoding) } -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, thiserror::Error)] pub enum DeserialiseError { - ExpectedLeaf, + #[error("Expected a node Tag, but got leaf")] + UnexpectedNode, + #[error("Invalid tag")] InvalidTag, + #[error("Not enough bytes")] NotEnoughBytes, + #[error("Too many bytes")] TooManyBytes, } @@ -301,7 +324,7 @@ fn deserialise_merkle_proof_tags( }, Tag::Node => match subtree { Tree::Node(children) => Ok(ModifyResult::NodeContinue((), children)), - Tree::Leaf(_) => Err(DeserialiseError::ExpectedLeaf), + Tree::Leaf(_) => Err(DeserialiseError::UnexpectedNode), }, } }) @@ -342,7 +365,7 @@ fn deserialise_merkle_proof( /// /// Ideally, the shape should be a compile time type/argument. /// However, for ease of implementation currently it is given in the form of a [`Shape`] argument -pub fn deserialise_proof( +pub fn deserialise_into_proof( mut bytes: impl Iterator, full_merkle_tree_shape: Shape, ) -> Result { @@ -357,6 +380,15 @@ pub fn deserialise_proof( Ok(Proof::new(merkle_proof, final_hash)) } +pub fn deserialise_into_backend( + mut bytes: impl Iterator, +) -> Result<(Hash, ::Allocated), FromProofError> { + let final_hash = deserialise_final_hash_encoding(&mut bytes)?; + let bytes = bytes.collect::>(); + let verify_backend = L::from_proof(StreamShapeContext::from_bytes(&bytes))?; + Ok((final_hash, verify_backend)) +} + #[cfg(test)] mod tests { use proptest::proptest; @@ -372,7 +404,7 @@ mod tests { use super::TAG_READ; use super::serialise_proof; use crate::state_backend::proof_backend::proof::Proof; - use crate::state_backend::proof_backend::proof::deserialise_proof; + use crate::state_backend::proof_backend::proof::deserialise_into_proof; use crate::storage::DIGEST_SIZE; use crate::storage::Hash; @@ -602,7 +634,7 @@ mod tests { } fn check_deserialisation(raw_bytes: &[u8], full_shape: Shape, expected_proof: Proof) { - let proof = deserialise_proof(raw_bytes.iter().copied(), full_shape).unwrap(); + let proof = deserialise_into_proof(raw_bytes.iter().copied(), full_shape).unwrap(); assert_eq!(proof, expected_proof); } @@ -611,7 +643,7 @@ mod tests { full_shape: Shape, expected_error: DeserialiseError, ) { - let proof = deserialise_proof(raw_bytes.iter().copied(), full_shape); + let proof = deserialise_into_proof(raw_bytes.iter().copied(), full_shape); assert_eq!(proof, Err(expected_error)); } @@ -799,14 +831,14 @@ mod tests { let raw_bytes = [fh.as_ref(), &tag_bytes, &val_bytes].concat(); check_deserialisation(&raw_bytes, root.clone(), Proof::new(merkle_proof, fh)); - // ExpectedLeaf tag error + // UnexpectedNode tag error let tag_bytes_bad = [ (TAG_NODE << 6) | (TAG_NODE << 4) | (TAG_NODE << 2) | (TAG_READ << 0), (TAG_BLIND << 6) | (TAG_NODE << 4) | (TAG_READ << 2) | (TAG_NODE << 0), (TAG_READ << 6) | (TAG_BLIND << 4), ]; let raw_bytes = [fh.as_ref(), &tag_bytes_bad, &val_bytes].concat(); - check_bad_deserialisation(&raw_bytes, root.clone(), DeserialiseError::ExpectedLeaf); + check_bad_deserialisation(&raw_bytes, root.clone(), DeserialiseError::UnexpectedNode); } #[test] diff --git a/src/riscv/lib/src/state_backend/proof_backend/proof/deser_owned.rs b/src/riscv/lib/src/state_backend/proof_backend/proof/deser_owned.rs new file mode 100644 index 0000000000000000000000000000000000000000..8e529568bdcd2f81eef94e06dcee4c242f7553cd --- /dev/null +++ b/src/riscv/lib/src/state_backend/proof_backend/proof/deser_owned.rs @@ -0,0 +1,233 @@ +use std::collections::VecDeque; +use std::marker::PhantomData; + +use serde::de::DeserializeOwned; + +use super::deser_proof::DeserBranch; +use super::deser_proof::DeserError; +use super::deser_proof::DeserProof; +use super::deser_proof::DeserResult; +use super::deser_proof::PartialResult; +use super::deser_proof::Suspended; +use crate::state_backend::FromProofError; +use crate::state_backend::ProofTree; +use crate::state_backend::proof_backend::proof::MerkleProofLeaf; +use crate::storage::binary; + +// Owns info about shape and contained data in proof +// Leafs provide DummyInput +pub type OwnedShapeContext<'t> = ProofTree<'t>; + +impl<'t> OwnedShapeContext<'t> { + pub fn deser_as_leaf(self) -> DeserResult>> { + use crate::state_backend::ProofPart; + use crate::state_backend::proof_backend::tree::Tree; + + let partial_result = match self { + ProofPart::Absent => PartialResult::Absent, + ProofPart::Present(Tree::Node(_)) => return Err(FromProofError::UnexpectedNode), + ProofPart::Present(Tree::Leaf(MerkleProofLeaf::Blind(hash))) => { + PartialResult::Blinded(*hash) + } + ProofPart::Present(Tree::Leaf(MerkleProofLeaf::Read(items))) => { + PartialResult::Present(items.clone()) + } + }; + + Ok(partial_result) + } + + pub fn deser_as_node(self) -> DeserResult>>> { + use crate::state_backend::ProofPart; + use crate::state_backend::proof_backend::tree::Tree; + + let partial_result = match self { + ProofPart::Absent => PartialResult::Absent, + ProofPart::Present(Tree::Leaf(_)) => return Err(FromProofError::UnexpectedNode), + ProofPart::Present(Tree::Node(trees)) => PartialResult::Present( + trees + .into_iter() + .map(|tree| ProofPart::Present(tree)) + .collect(), + ), + }; + + Ok(partial_result) + } +} + +pub struct OwnedParserComb<'t, R> { + f: Box Result + 'static>, + _pd: PhantomData)>, +} + +pub struct OwnedBranchComb<'p, R, B> { + f: OwnedParserComb<'p, R>, + branches: PartialResult>, +} + +impl<'t, R> OwnedParserComb<'t, R> { + fn new(f: impl FnOnce() -> Result + 'static) -> Self { + Self { + f: Box::new(f), + _pd: PhantomData, + } + } +} + +impl<'t> DeserProof for OwnedShapeContext<'t> { + type Susp = OwnedParserComb<'t, R>; + + fn as_leaf_raw( + self, + ) -> DeserResult>>> { + let leaf = self.deser_as_leaf()?; + + let bytes: Box<[u8; LEN]> = match leaf { + PartialResult::Absent => return Ok(OwnedParserComb::new(|| Ok(PartialResult::Absent))), + PartialResult::Blinded(hash) => { + return Ok(OwnedParserComb::new(move || { + Ok(PartialResult::Blinded(hash)) + })); + } + PartialResult::Present(data) => { + let data_len = data.len(); + data.try_into() + .map_err(|_| DeserError::UnexpectedLeafSize { + expected: LEN, + got: data_len, + })? + } + }; + Ok(OwnedParserComb::new(|| Ok(PartialResult::Present(bytes)))) + } + + // The leaf case is not that simple + // for blind case, the cursor needs to pass over the hash + // And anyway, other usecases may want different return values depending on + // blind/absent/hasdata + fn as_leaf(self) -> DeserResult>> + where + T: DeserializeOwned + 'static, + { + let leaf = self.deser_as_leaf()?; + let leaf = match leaf { + PartialResult::Absent => PartialResult::Absent, + PartialResult::Blinded(hash) => PartialResult::Blinded(hash), + PartialResult::Present(data) => { + PartialResult::Present(binary::deserialise::(data.as_ref())?) + } + }; + Ok(OwnedParserComb::new(move || Ok(leaf))) + } + + type DeserBr = OwnedBranchComb<'t, R, Self>; + + fn as_node(self) -> DeserResult>> { + let branches = self.deser_as_node()?; + let branches = + branches.and_then(|branches| PartialResult::Present(VecDeque::from(branches))); + Ok(OwnedBranchComb { + f: OwnedParserComb::new(|| Ok(PartialResult::Present(()))), + branches, + }) + } +} + +impl<'t, R> DeserBranch for OwnedBranchComb<'t, R, OwnedShapeContext<'t>> { + type Parent = OwnedShapeContext<'t>; + + fn branch( + mut self, + br_deser: impl FnOnce(Self::Parent) -> DeserResult<::Susp>, + ) -> DeserResult<::DeserBr<(R, T)>> + where + R: 'static, + T: 'static, + { + let next_branch = match self.branches { + PartialResult::Absent => OwnedShapeContext::Absent, + PartialResult::Blinded(_node_hash) => OwnedShapeContext::Absent, + PartialResult::Present(ref mut branches) => { + branches.pop_front().ok_or(DeserError::UnexpectedBranch)? + } + }; + let br_comb = br_deser(next_branch)?; + + Ok(OwnedBranchComb { + f: self.f.zip(br_comb), + branches: self.branches, + }) + } + + fn map(self, f: impl FnOnce(R) -> T + 'static) -> ::DeserBr + where + T: 'static, + R: 'static, + { + OwnedBranchComb { + f: self.f.map(f), + branches: self.branches, + } + } + + fn done(self) -> ::Susp { + self.f + } +} + +impl<'t, R> Suspended for OwnedParserComb<'t, R> { + type R = R; + + type Parent = OwnedShapeContext<'t>; + + fn map(self, f: impl FnOnce(Self::R) -> T + 'static) -> ::Susp + where + Self::R: 'static, + { + OwnedParserComb::new(move || Ok(f((self.f)()?))) + } + + fn zip( + self, + other: ::Susp, + ) -> ::Susp<(Self::R, T)> + where + Self::R: 'static, + T: 'static, + { + OwnedParserComb::new(move || Ok(((self.f)()?, (other.f)()?))) + } + + fn exec(self) -> DeserResult { + (self.f)() + } +} + +impl<'p, R: 'static> Suspended for OwnedBranchComb<'p, R, OwnedShapeContext<'p>> { + type R = R; + + type Parent = OwnedShapeContext<'p>; + + fn map(self, f: impl FnOnce(Self::R) -> T + 'static) -> ::Susp + where + Self::R: 'static, + { + self.f.map(f) + } + + fn zip( + self, + other: ::Susp, + ) -> ::Susp<(Self::R, T)> + where + Self::R: 'static, + T: 'static, + { + self.f.zip(other) + } + + fn exec(self) -> DeserResult { + self.f.exec() + } +} diff --git a/src/riscv/lib/src/state_backend/proof_backend/proof/deser_proof.rs b/src/riscv/lib/src/state_backend/proof_backend/proof/deser_proof.rs new file mode 100644 index 0000000000000000000000000000000000000000..ed7b2f7b3aff83c0a44e673e2d19d8cb0accc309 --- /dev/null +++ b/src/riscv/lib/src/state_backend/proof_backend/proof/deser_proof.rs @@ -0,0 +1,158 @@ +#![allow(dead_code)] + +use serde::de::DeserializeOwned; + +use crate::state_backend::FromProofError; +use crate::storage::Hash; + +pub type DeserError = FromProofError; + +pub type DeserResult = Result; + +pub enum PartialResult { + Absent, + Blinded(Hash), + Present(T), +} + +impl PartialResult { + pub fn and_then(self, f: impl FnOnce(T) -> PartialResult) -> PartialResult { + match self { + PartialResult::Absent => PartialResult::Absent, + PartialResult::Blinded(hash) => PartialResult::Blinded(hash), + PartialResult::Present(data) => f(data), + } + } +} + +pub trait DeserProof { + type Susp: Suspended; + + type DeserBr: DeserBranch; + + fn as_leaf_raw( + self, + ) -> DeserResult>>>; + + fn as_leaf(self) -> DeserResult>>; + + fn as_node(self) -> DeserResult>>; +} + +pub trait DeserBranch { + type Parent: DeserProof; + + fn branch( + self, + br_deser: impl FnOnce(Self::Parent) -> DeserResult<::Susp>, + ) -> DeserResult<::DeserBr<(R, T)>> + where + R: 'static, + T: 'static; + + fn map(self, f: impl FnOnce(R) -> T + 'static) -> ::DeserBr + where + T: 'static, + R: 'static; + + fn done(self) -> ::Susp; +} + +pub trait Suspended { + type R; + + type Parent: DeserProof; + + fn map( + self, + f: impl FnOnce(Self::R) -> T + 'static, + ) -> ::Susp + where + Self::R: 'static; + + fn zip( + self, + other: ::Susp, + ) -> ::Susp<(Self::R, T)> + where + Self::R: 'static, + T: 'static; + + fn exec(self) -> DeserResult; +} + +#[cfg(test)] +mod tests { + // Test code + + use super::DeserBranch; + use super::DeserProof; + use super::DeserResult; + use super::PartialResult; + use super::Suspended; + use crate::storage::Hash; + + fn create_computation(proof: D) -> DeserResult<::Susp> { + // allocate vec + + // The tree structure: + // Root + // ├── Leaf (Hash) + // └── Node + // └── Branch + // └── Leaf (u32) + + let ctx = proof.as_node()?; + let r = ctx + .branch(|br_proof| br_proof.as_leaf::())? + .branch(|br_proof| { + Ok(br_proof + .as_node()? + .branch(|pr| pr.as_leaf::())? + .map(|(_unit, br)| br) + .done()) + })? + .done(); + + Ok(r.map(|(_left, right)| match right { + PartialResult::Absent => 0, + PartialResult::Blinded(_hash) => -1, + PartialResult::Present(nr) => nr as i32, + })) + } + + fn use_comp(proof: D) -> DeserResult { + let comp_fn = create_computation(proof)?; + comp_fn.exec() + } + + fn create_computation_2(proof: D) -> DeserResult<::Susp> { + // allocate vec + + // The tree structure + // Root (Node) + // ├── Branch 1 (Leaf: i32) + // ├── Branch 2 (Leaf: i32) + // ├── Branch 3 (Leaf: i32) + // └── Branch 4 (Leaf: i32) + + let mut ctx = proof + .as_node()? + .map(|_| PartialResult::Present(Vec::::new())); + + for _ in 0..4 { + ctx = ctx + .branch(|br_proof| br_proof.as_leaf::())? + .map(|(acc, val)| { + val.and_then(|val| { + acc.and_then(|mut acc| { + acc.push(val); + PartialResult::Present(acc) + }) + }) + }) + } + + Ok(ctx.done().map(|_| 3)) + } +} diff --git a/src/riscv/lib/src/state_backend/proof_backend/proof/deser_stream.rs b/src/riscv/lib/src/state_backend/proof_backend/proof/deser_stream.rs new file mode 100644 index 0000000000000000000000000000000000000000..f6f50d6feee1f5fcfbc255467e644b98d4810c11 --- /dev/null +++ b/src/riscv/lib/src/state_backend/proof_backend/proof/deser_stream.rs @@ -0,0 +1,228 @@ +use std::cell::RefCell; +use std::collections::VecDeque; +use std::io::Read; +use std::rc::Rc; + +use serde::de::DeserializeOwned; + +use super::DeserialiseError; +use super::LeafTag; +use super::Tag; +use super::deser_proof::DeserBranch; +use super::deser_proof::DeserError; +use super::deser_proof::DeserProof; +use super::deser_proof::DeserResult; +use super::deser_proof::PartialResult; +use super::deser_proof::Suspended; +use crate::storage::Hash; + +struct StreamInput<'cd> { + cursor: std::io::Cursor<&'cd [u8]>, +} + +impl<'cd> StreamInput<'cd> { + fn deserialise(&mut self) -> DeserResult { + Ok(crate::storage::binary::deserialise_from(&mut self.cursor)?) + } +} + +// Can only first be parsed to get shape data +// Only after it can be used to obtain DummyInput +#[derive(Clone)] +pub struct StreamShapeContext<'t> { + tags: Rc>>, +} + +impl<'t> StreamShapeContext<'t> { + pub fn from_bytes(bytes: &'t [u8]) -> Self { + Self { + tags: Rc::new(RefCell::new(TagIter::from_bytes(bytes))), + } + } +} + +impl<'t> DeserProof for StreamShapeContext<'t> { + type Susp = StreamParserComb<'t, R>; + + type DeserBr = StreamBranchComb<'t, R>; + + fn as_leaf_raw( + self, + ) -> DeserResult>>> { + let mut ts = self.tags.borrow_mut(); + let tag = ts.next().ok_or(DeserialiseError::NotEnoughBytes)?; + let f = match tag { + Tag::Node => return Err(DeserError::UnexpectedNode), + Tag::Leaf(leaf_type) => Box::new(move |input: &mut StreamInput| match leaf_type { + LeafTag::Blind => Ok(PartialResult::Blinded(input.deserialise::()?)), + LeafTag::Read => { + let mut data = Box::new([0u8; LEN]); + match input.cursor.read_exact(data.as_mut()) { + Ok(()) => Ok(PartialResult::Present(data)), + Err(_eof) => Err(DeserialiseError::NotEnoughBytes.into()), + } + } + }), + }; + Ok(StreamParserComb { + f, + tags_ctx: self.clone(), + }) + } + + fn as_leaf(self) -> DeserResult>> { + // Panic: Never panic becase the borrow is firist used only while deserialising, + // and then it is used when running the Suspended computation + let mut ts = self.tags.borrow_mut(); + let tag = ts.next().ok_or(DeserialiseError::NotEnoughBytes)?; + let f = match tag { + Tag::Node => return Err(DeserError::UnexpectedNode), + Tag::Leaf(leaf_type) => Box::new(move |input: &mut StreamInput| match leaf_type { + LeafTag::Blind => Ok(PartialResult::Blinded(input.deserialise::()?)), + LeafTag::Read => Ok(PartialResult::Present(input.deserialise::()?)), + }), + }; + Ok(StreamParserComb { + f, + tags_ctx: self.clone(), + }) + } + + fn as_node(self) -> DeserResult>> { + let mut ts = self.tags.borrow_mut(); + let tag = ts.next().ok_or(DeserialiseError::NotEnoughBytes)?; + let f = match tag { + Tag::Leaf(_) => return Err(DeserError::UnexpectedLeaf), + Tag::Node => Box::new(|_: &mut StreamInput| Ok(PartialResult::Present(()))), + }; + Ok(StreamBranchComb { + f: StreamParserComb { + f, + tags_ctx: StreamShapeContext { + tags: self.tags.clone(), + }, + }, + }) + } +} + +pub struct TagIter<'i> { + buffered_tags: VecDeque>, + raw_data_iter: std::io::Cursor<&'i [u8]>, +} + +impl<'i> Iterator for TagIter<'i> { + type Item = Tag; + + fn next(&mut self) -> Option { + if self.buffered_tags.is_empty() { + // We have to consume the next byte + let mut next_byte = [0u8]; + let byte = match self.raw_data_iter.read_exact(&mut next_byte) { + Ok(()) => next_byte[0], + Err(_eof) => return None, + }; + + let tags = Tag::ordered_tags_from_u8(byte); + self.buffered_tags.append(&mut tags.into()); + } + + self.buffered_tags.pop_front().map(Result::ok).flatten() + } +} + +impl<'i> TagIter<'i> { + fn remaining_to_cursor(&'i self) -> std::io::Cursor<&'i [u8]> { + self.raw_data_iter.clone() + } + + fn from_bytes(bytes: &'i [u8]) -> Self { + Self { + buffered_tags: VecDeque::new(), + raw_data_iter: std::io::Cursor::new(bytes), + } + } +} + +pub struct StreamBranchComb<'t, R> { + f: StreamParserComb<'t, R>, +} + +impl<'t, R> DeserBranch for StreamBranchComb<'t, R> { + type Parent = StreamShapeContext<'t>; + + fn branch( + self, + br_deser: impl FnOnce(Self::Parent) -> DeserResult<::Susp>, + ) -> DeserResult<::DeserBr<(R, T)>> + where + R: 'static, + T: 'static, + { + let br = br_deser(StreamShapeContext { + tags: self.f.tags_ctx.tags.clone(), + })?; + + Ok(StreamBranchComb { f: self.f.zip(br) }) + } + + fn map(self, f: impl FnOnce(R) -> T + 'static) -> ::DeserBr + where + T: 'static, + R: 'static, + { + StreamBranchComb { f: self.f.map(f) } + } + + fn done(self) -> ::Susp { + self.f + } +} + +pub struct StreamParserComb<'t, R> { + f: Box Result + 'static>, + tags_ctx: StreamShapeContext<'t>, +} + +impl<'t, R> Suspended for StreamParserComb<'t, R> { + type R = R; + + type Parent = StreamShapeContext<'t>; + + fn map(self, f: impl FnOnce(Self::R) -> T + 'static) -> ::Susp + where + Self::R: 'static, + { + StreamParserComb { + f: Box::new(|input| { + let r = (self.f)(input)?; + Ok(f(r)) + }), + tags_ctx: self.tags_ctx, + } + } + + fn zip( + self, + other: ::Susp, + ) -> ::Susp<(Self::R, T)> + where + Self::R: 'static, + T: 'static, + { + StreamParserComb { + f: Box::new(|input| { + let r = (self.f)(input)?; + let t = (other.f)(input)?; + Ok((r, t)) + }), + tags_ctx: self.tags_ctx, + } + } + + fn exec(self) -> DeserResult { + (self.f)(&mut StreamInput { + cursor: self.tags_ctx.tags.borrow().remaining_to_cursor(), + }) + } +} diff --git a/src/riscv/lib/src/state_backend/proof_layout.rs b/src/riscv/lib/src/state_backend/proof_layout.rs index dc3513ac516dd257e5f807cf468e33a9bbb7b28c..2828ad7ca3a1d93d58a78d5e4489a9fab01bdd1c 100644 --- a/src/riscv/lib/src/state_backend/proof_layout.rs +++ b/src/riscv/lib/src/state_backend/proof_layout.rs @@ -4,7 +4,11 @@ // SPDX-License-Identifier: MIT use std::collections::VecDeque; +use std::marker::PhantomData; +use tuples::CombinRight; + +use super::AllocatedOf; use super::Array; use super::Atom; use super::DynArray; @@ -22,12 +26,19 @@ use super::proof_backend::merkle::MerkleTree; use super::proof_backend::merkle::MerkleWriter; use super::proof_backend::merkle::build_custom_merkle_tree; use super::proof_backend::merkle::chunks_to_writer; +use super::proof_backend::proof::DeserialiseError; use super::proof_backend::proof::MerkleProof; use super::proof_backend::proof::MerkleProofLeaf; +use super::proof_backend::proof::deser_proof::DeserBranch; +use super::proof_backend::proof::deser_proof::DeserProof; +use super::proof_backend::proof::deser_proof::DeserResult; +use super::proof_backend::proof::deser_proof::Suspended; use super::proof_backend::tree::Tree; use super::verify_backend::PartialState; +use super::verify_backend::Verifier; use super::verify_backend::{self}; use crate::array_utils::boxed_array; +use crate::state_backend::proof_backend::proof::deser_proof::PartialResult; use crate::state_backend::verify_backend::PageId; use crate::storage::binary; @@ -37,6 +48,9 @@ pub enum FromProofError { #[error("Error during hashing: {0}")] Hash(#[from] HashError), + #[error("Error during stream parsing")] + Parsing(#[from] DeserialiseError), + #[error("Error during deserialisation: {0}")] Deserialise(#[from] bincode::Error), @@ -54,6 +68,9 @@ pub enum FromProofError { #[error("Encountered a node where a leaf was expected")] UnexpectedNode, + + #[error("Too few branches were present in the tree")] + UnexpectedBranch, } type Result = std::result::Result; @@ -83,6 +100,8 @@ pub enum PartialHashError { /// Common result type for parsing a Merkle proof pub type FromProofResult = Result<::Allocated>; +pub type AllocVerif = ::Allocated; + /// Part of a tree that may be absent pub enum ProofPart<'a, T: ?Sized> { /// This part of the tree is absent. @@ -254,6 +273,15 @@ impl ProofLayout for Box { } } +impl ProofContextLayout for Box +where + T: 'static, +{ + fn from_proof_comp(proof: DP) -> DeserResult>> { + Ok(T::from_proof_comp(proof)?.map(|data| Ok(Box::new(data?)))) + } +} + impl ProofLayout for Atom where T: serde::Serialize + serde::de::DeserializeOwned + 'static, @@ -270,7 +298,7 @@ where } fn from_proof(proof: ProofTree) -> FromProofResult { - >::from_proof(proof).map(super::Cell::from) + as ProofLayout>::from_proof(proof).map(super::Cell::from) } fn partial_state_hash( @@ -286,6 +314,18 @@ where } } +// todo: Write proofcontextlayout with Deser framework + +impl ProofContextLayout for Atom +where + T: serde::Serialize + serde::de::DeserializeOwned + 'static, +{ + fn from_proof_comp(proof: DP) -> DeserResult>> { + let f = as ProofContextLayout>::from_proof_comp(proof)?; + Ok(f.map(|cells| Ok(super::Cell::from(cells?)))) + } +} + impl ProofLayout for Array where T: serde::Serialize + serde::de::DeserializeOwned + 'static, @@ -339,6 +379,57 @@ where } } +// implement ProofContextLayout for DynArray +impl ProofContextLayout for DynArray { + fn from_proof_comp(proof: DP) -> DeserResult>> { + type PageData = ( + PageId<{ MERKLE_LEAF_SIZE.get() }>, + Box<[u8; MERKLE_LEAF_SIZE.get()]>, + ); + + fn parse_pages_fn_getter( + start: usize, + left_length: usize, + ) -> impl FnOnce(DP) -> DeserResult>> { + let page = verify_backend::PageId::from_address(start); + + move |proof: DP| { + if left_length <= MERKLE_LEAF_SIZE.get() { + let ctx = proof.as_leaf_raw()?; + let ctx = ctx.map(move |data| match data { + PartialResult::Blinded(_) | PartialResult::Absent => vec![], + PartialResult::Present(data) => vec![(page, data)], + }); + Ok(ctx) + } else { + let ctx: ::DeserBr> = + proof.as_node()?.map(|_| vec![]); + let ctx = Ok::<_, FromProofError>(ctx); + let r = work_merkle_params::(start, left_length).fold( + ctx, + |ctx, (start, length)| { + Ok(ctx?.branch(parse_pages_fn_getter(start, length))?.map( + |(mut acc, mut br_data)| { + acc.append(&mut br_data); + acc + }, + )) + }, + ); + Ok(r?.done()) + } + } + } + + let pages_handler = parse_pages_fn_getter::(0, LEN)(proof)?; + Ok(pages_handler.map(|pages| { + let region = verify_backend::DynRegion::from_pages(pages); + let cells = super::DynCells::bind(region); + Ok(cells) + })) + } +} + impl ProofLayout for DynArray { fn to_merkle_tree(state: RefProofGenOwnedAlloc) -> Result { let region = state.region_ref(); @@ -689,6 +780,33 @@ where } } +impl ProofContextLayout for [T; LEN] +where + T: ProofContextLayout, + AllocatedOf: 'static, +{ + fn from_proof_comp(proof: DP) -> DeserResult>> { + let mut ctx: DP::DeserBr>>> = proof.as_node()?.map(|_| Ok(vec![])); + + for _ in 0..LEN { + ctx = ctx.branch(T::from_proof_comp)?.map(|(acc, br_res)| { + let mut acc = acc?; + acc.push(br_res?); + Ok(acc) + }); + } + + let ctx = ctx.map(|data| { + data?.try_into().map_err(|_| { + // We can't use expected because the error can't be displayed + unreachable!("Conversion to array of fixed length doesn't fail") + }) + }); + + Ok(ctx.done()) + } +} + impl ProofLayout for [T; LEN] where T: ProofLayout, @@ -729,6 +847,328 @@ where } } +#[derive(Copy, Clone)] +pub enum LeafProofCtx { + Absent, + Blind, + HasData, +} + +impl From<&MerkleProofLeaf> for LeafProofCtx { + fn from(value: &MerkleProofLeaf) -> Self { + match value { + MerkleProofLeaf::Blind(_) => LeafProofCtx::Blind, + MerkleProofLeaf::Read(_) => LeafProofCtx::HasData, + } + } +} + +pub trait DataStream { + fn parse_bytes(&mut self) -> Result; + + fn parse_raw_bytes(&mut self) -> Result>; +} + +pub type Handler = Box FnOnce(&'d mut DS) -> R>; + +pub type ResultHandler = Handler, DS>; + +pub trait ProofContext +where + Self: Sized, +{ + fn as_leaf(&mut self) -> Result; + + fn as_branches<'c, const LEN: usize, DS: DataStream + 'static>( + &'c mut self, + ) -> Result>; +} + +struct OwnedBranchCtx<'c, R, DS, PC: ProofContext + ?Sized, BI: Iterator> { + pub handler: ResultHandler, + branches: BI, + branches_cnt: usize, + _pd: PhantomData<&'c ()>, +} + +impl<'c, DS, PC: ProofContext, BI: Iterator> OwnedBranchCtx<'c, (), DS, PC, BI> { + fn empty(iter: BI) -> Self { + Self { + handler: Box::new(|_| Ok(())), + branches: iter, + branches_cnt: 0, + _pd: PhantomData, + } + } +} + +impl<'c, R: 'static, DS: DataStream + 'static, PC: ProofContext, BI: Iterator> + OwnedBranchCtx<'c, R, DS, PC, BI> +{ + fn branch( + mut self, + branch_fn: impl FnOnce(&mut PC) -> Result>, + combine: impl FnOnce(R, BranchRes) -> NewRes + 'static, + ) -> Result> { + let mut child_ctx = self + .branches + .next() + .ok_or(FromProofError::BadNumberOfBranches { + expected: self.branches_cnt, + got: self.branches_cnt + 1, + })?; + let branch_handler = branch_fn(&mut child_ctx)?; + + let f: Box FnOnce(&'dd mut DS) -> _> = Box::new(move |data_stream| { + let res = (self.handler)(data_stream)?; + let branch_res = branch_handler(data_stream)?; + // let (res, branch_res) = use_handlers(data_stream, self.handler, branch_handler); + + Ok(combine(res, branch_res)) + }); + Ok(OwnedBranchCtx { + handler: f, + branches: self.branches, + branches_cnt: self.branches_cnt + 1, + _pd: self._pd, + }) + } +} + +impl<'c, R: 'static, DS: DataStream + 'static, PC: ProofContext, BI: Iterator> + BranchCtx<'c, R, DS, PC> for OwnedBranchCtx<'c, R, DS, PC, BI> +{ + fn branch( + self, + branch_fn: impl FnOnce(&mut PC) -> Result>, + ) -> Result> { + self.branch(branch_fn, Box::new(|res, branch_res| (res, branch_res))) + } + + fn map_handler( + self, + map_fn: impl FnOnce(R) -> NewR + 'static, + ) -> impl BranchCtx<'c, NewR, DS, PC> { + let new_handler: Box FnOnce(&'ds mut DS) -> _> = + Box::new(|data_stream| Ok(map_fn((self.handler)(data_stream)?))); + OwnedBranchCtx { + handler: new_handler, + branches: self.branches, + branches_cnt: self.branches_cnt, + _pd: PhantomData, + } + } + + fn handler(self) -> ResultHandler { + self.handler + } + + fn branch_same( + self, + branch_fn: impl FnOnce(&mut PC) -> Result>, + combine: impl FnOnce(R, R) -> R + 'static, + ) -> Result { + self.branch(branch_fn, combine) + } +} + +pub trait BranchCtx<'c, R, DS, PC> +where + Self: Sized, + R: 'static, + DS: DataStream + 'static, + PC: ProofContext, +{ + fn branch( + self, + branch_fn: impl FnOnce(&mut PC) -> Result>, + ) -> Result>; + + fn branch_same( + self, + branch_fn: impl for<'pc> FnOnce(&'pc mut PC) -> Result>, + combine: impl FnOnce(R, R) -> R + 'static, + ) -> Result; + + fn map_handler( + self, + map_fn: impl FnOnce(R) -> NewR + 'static, + ) -> impl BranchCtx<'c, NewR, DS, PC>; + + fn handler(self) -> ResultHandler; +} + +pub trait ProofContextLayout: Layout { + /// For From_proof, we want the same proof context to work as a data stream as well + fn from_proof<'c, DP: DeserProof + 'c>(proof: DP) -> FromProofResult { + let comp_fn = Self::from_proof_comp(proof)?; + comp_fn.exec()? + } + + fn from_proof_comp(proof: DP) -> DeserResult>>; +} + +impl ProofContextLayout for Array +where + T: serde::de::DeserializeOwned + 'static, +{ + fn from_proof_comp(proof: DP) -> DeserResult>> { + use super::proof_backend::proof::deser_proof::PartialResult; + + Ok(proof + .as_leaf::>()? + .map(|stuff| { + let region = match stuff { + PartialResult::Absent | PartialResult::Blinded(_) => { + verify_backend::Region::Absent + } + PartialResult::Present(cells) => { + let arr: Box<[Option; LEN]> = Box::new(cells.into_region().map(Some)); + verify_backend::Region::Partial(arr) + } + }; + Ok(super::Cells::bind(region)) + })) + } +} + +#[macro_export] +macro_rules! count_args { + ($($arg:tt),+) => { + <[()]>::len(&[$(count_args!(@sub $arg)),+]) + }; + (@sub $_arg:tt) => { () }; +} + +#[macro_export] +macro_rules! tuple_branches_proof_context_layout { + ($ctx:expr, [$($acc_br:ident,)*], $curr_br:ident, $($rest_br:ident,)*) => { + { + use $crate::state_backend::proof_backend::proof::deser_proof::DeserBranch; + use tuples::CombinRight; + + let ctx = $ctx + .branch(<$curr_br>::from_proof_comp)? + .map(|(acc, a)| -> Result<_, DeserError> { + let acc = acc?; + let a = a?; + Ok(acc.push_right(a)) + }); + + tuple_branches_proof_context_layout!(ctx, [ $($acc_br,)* $curr_br, ], $($rest_br,)*) + } + }; + ($ctx:expr, [$($acc_br:ident,)*],) => { + $ctx + }; +} + +#[macro_export] +macro_rules! impl_proof_context_layout_for_tuple { + ($($name:ident),+) => { + impl<$($name: ProofContextLayout),+> ProofContextLayout for ($($name,)+) + where + $( + AllocatedOf<$name, Verifier>: 'static, + )+ + { + fn from_proof_comp( + proof: DP, + ) -> DeserResult>> { + + use $crate::state_backend::proof_backend::proof::deser_proof::DeserError; + + const CNT: usize = count_args!($($name),+); + + let ctx = proof + .as_node()?.map(|_unit_partial_res| Ok::<_, DeserError>(())); + + let ctx = tuple_branches_proof_context_layout!(ctx, [], $($name,)+); + + Ok(ctx.done()) + } + } + }; +} + +impl_proof_context_layout_for_tuple!(A, B); +impl_proof_context_layout_for_tuple!(A, B, C); +impl_proof_context_layout_for_tuple!(A, B, C, D); +impl_proof_context_layout_for_tuple!(A, B, C, D, E); +impl_proof_context_layout_for_tuple!(A, B, C, D, E, F); + +const fn const_child_length() -> usize { + let branch_max_length = LEN.div_ceil(CHILDREN); + if LEN < branch_max_length { + LEN + } else { + branch_max_length + } +} + +const fn child_length(length: usize) -> usize { + let branch_max_length = length.div_ceil(CHILDREN); + if length < branch_max_length { + length + } else { + branch_max_length + } +} + +fn work_merkle_params( + mut branch_start: usize, + mut length_left: usize, +) -> impl Iterator { + (0..CHILDREN).map(move |_| { + let this_branch_length = child_length::(length_left); + + let item = (branch_start, this_branch_length); + + branch_start = branch_start.saturating_add(this_branch_length); + length_left = length_left.saturating_sub(this_branch_length); + + item + }) +} + +impl ProofContextLayout for Many +where + T: ProofContextLayout, + AllocatedOf: 'static, +{ + fn from_proof_comp(proof: DP) -> DeserResult>> { + // let mut _data_fn = Vec::with_capacity(LEN); + fn parametrised_deserialiser( + length: usize, + ) -> impl FnOnce(DP) -> DeserResult>>>> + where + AllocatedOf: 'static, + { + let child_length = child_length::(length); + + Box::new(move |proof: DP| { + if length == 1 { + Ok(T::from_proof_comp(proof)?.map(|data| data.map(|data| vec![data]))) + } else { + let mut ctx = proof.as_node()?.map(|_| Ok(vec![])); + for _ in 0..MERKLE_ARITY { + ctx = ctx + .branch(parametrised_deserialiser::(child_length))? + .map(|(acc, br_res)| { + let mut acc = acc?; + acc.append(&mut br_res?); + Ok(acc) + }); + } + Ok(ctx.done()) + } + }) + } + + parametrised_deserialiser::(LEN)(proof) + } +} + impl ProofLayout for Many where T: ProofLayout, diff --git a/src/riscv/lib/src/stepper/pvm.rs b/src/riscv/lib/src/stepper/pvm.rs index a0e1cd1efeb145f2d585828327a1415318349df2..238ba0420040d90af5eef30d9c8b0b8af869b24e 100644 --- a/src/riscv/lib/src/stepper/pvm.rs +++ b/src/riscv/lib/src/stepper/pvm.rs @@ -34,6 +34,7 @@ use crate::range_utils::bound_saturating_sub; use crate::state_backend::AllocatedOf; use crate::state_backend::FnManagerIdent; use crate::state_backend::ManagerBase; +use crate::state_backend::ManagerRead; use crate::state_backend::ManagerReadWrite; use crate::state_backend::ProofLayout; use crate::state_backend::ProofTree; diff --git a/src/riscv/lib/tests/test_proofs.rs b/src/riscv/lib/tests/test_proofs.rs index fa13d942b315931df4c6d5cfd2f5df6b6ae9dc75..68cdc03a8200288afad46103e7fb4f2d4ac519eb 100644 --- a/src/riscv/lib/tests/test_proofs.rs +++ b/src/riscv/lib/tests/test_proofs.rs @@ -13,10 +13,16 @@ use common::*; use octez_riscv::machine_state::CacheLayouts; use octez_riscv::machine_state::DefaultCacheLayouts; use octez_riscv::machine_state::TestCacheLayouts; +use octez_riscv::machine_state::block_cache::block; use octez_riscv::machine_state::memory::M64M; use octez_riscv::machine_state::memory::MemoryConfig; +use octez_riscv::pvm::Pvm; +use octez_riscv::pvm::PvmLayout; +use octez_riscv::state_backend::CommitmentLayout; +use octez_riscv::state_backend::ProofLayout; use octez_riscv::state_backend::hash; use octez_riscv::state_backend::proof_backend::proof::Proof; +use octez_riscv::state_backend::proof_backend::proof::deserialise_into_proof; use octez_riscv::state_backend::proof_backend::proof::serialise_proof; use octez_riscv::state_backend::verify_backend::ProofVerificationFailure; use octez_riscv::stepper::Stepper; @@ -179,6 +185,63 @@ fn basic_invalid_proofs_are_rejected( ); } +#[test] +#[ignore] +fn test_proof_serialisation() { + // We type the Layout type parameters to ensure the correct shape is generated + let mut stepper: PvmStepper = make_stepper_factory()(); + type PvmStepperLayout = PvmLayout; + + let ladder = dissect_steps(200_000, 50_000); + let expected_steps = ladder.iter().sum::(); + + let mut shape = None; + + let mut steps_done = 0; + for &steps in &ladder { + // Run one step short of `steps`, then produce a proof of the following step. + let steps = steps.checked_sub(1).expect("minimum step size is 1"); + let result = stepper.step_max(Bound::Included(steps)); + steps_done += steps; + + if steps_done != expected_steps { + assert!(matches!(result, StepperStatus::Running { .. })); + + let hash_from_stepper = stepper.hash(); + let proof = stepper.produce_proof().unwrap(); + let hash_from_proof = proof.initial_state_hash(); + assert_eq!(hash_from_proof, hash_from_stepper); + + let serialisation = serialise_proof(&proof); + + // Small optimisation to avoid creating the shape for every proof generation. + // it takes over 30 seconds to compute the (same) shape + let actual_shape = shape.unwrap_or_else(|| { + // Even though `start_proof_mode` gets called after `produce_proof` + // this is actually fine since this is a read-only function + let stepper = stepper.start_proof_mode(); + let refs = stepper.struct_ref(); + // layout_shape::(refs) + }); + shape = Some(actual_shape.clone()); + + // TODO: Use DeserProof to deserialise proof into backend and check initial hash with commitment hash + + stepper.eval_one(); + steps_done += 1; + } else { + // Can't generate a proof on the next step if execution has ended + assert!(matches!(result, StepperStatus::Exited { .. })); + assert!(ladder.is_empty()) + }; + + eprintln!( + "> Done {:.2}%", + (steps_done as f64 / expected_steps as f64) * 100.0 + ); + } +} + mod proof_helpers { use octez_riscv::state_backend::hash::Hash; use octez_riscv::state_backend::proof_backend::proof::MerkleProofLeaf; diff --git a/src/rust_deps/Cargo.lock b/src/rust_deps/Cargo.lock index caca4c10c263af66f7cf4aa4da153fd263190f4b..051e63699339af22e494e6930c2771af1e4135d1 100644 --- a/src/rust_deps/Cargo.lock +++ b/src/rust_deps/Cargo.lock @@ -2569,6 +2569,7 @@ dependencies = [ "hex", "ieee-apsqrt", "itertools", + "lazy_static", "num_enum", "paste", "range-collections", diff --git a/victor.log b/victor.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391