From 35ca86a031992d9184447a211baca9c2cedb5bd2 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Mon, 30 Oct 2023 12:00:20 -0400 Subject: [PATCH 001/166] DefautBlockBuilder scaffold --- block-producer/src/block_builder/mod.rs | 44 ++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index bbd8fb529..596683466 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -1,6 +1,7 @@ use async_trait::async_trait; +use miden_objects::{Digest, Felt}; -use crate::SharedTxBatch; +use crate::{store::Store, SharedTxBatch}; #[derive(Debug, PartialEq)] pub enum BuildBlockError { @@ -19,3 +20,44 @@ pub trait BlockBuilder: Send + Sync + 'static { batches: Vec, ) -> Result<(), BuildBlockError>; } + +pub struct DefaultBlockBuilder { + store: S, + prev_header_hash: Digest, + prev_block_hash: Digest, + prev_block_num: Felt, +} + +impl DefaultBlockBuilder +where + S: Store, +{ + pub fn new( + store: S, + prev_header_hash: Digest, + prev_block_hash: Digest, + prev_block_num: Felt, + ) -> Self { + Self { + store, + prev_header_hash, + prev_block_hash, + prev_block_num, + } + } +} + +#[async_trait] +impl BlockBuilder for DefaultBlockBuilder +where + S: Store, +{ + async fn build_block( + &self, + batches: Vec, + ) -> Result<(), BuildBlockError> { + // TODO: call store.get_block_inputs() + + todo!() + } +} From d81ae71cea2545d690f7753cb60da5bf5d45a64f Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Mon, 30 Oct 2023 15:49:03 -0400 Subject: [PATCH 002/166] build_block: non-header fields --- block-producer/src/batch_builder/mod.rs | 4 ++-- block-producer/src/block_builder/mod.rs | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/block-producer/src/batch_builder/mod.rs b/block-producer/src/batch_builder/mod.rs index aa7cd8da4..998a0a95b 100644 --- a/block-producer/src/batch_builder/mod.rs +++ b/block-producer/src/batch_builder/mod.rs @@ -48,7 +48,7 @@ impl TransactionBatch { } /// Returns the nullifier of all consumed notes - pub fn consumed_notes_nullifiers(&self) -> impl Iterator + '_ { + pub fn produced_nullifiers(&self) -> impl Iterator + '_ { self.txs .iter() .flat_map(|tx| tx.consumed_notes()) @@ -56,7 +56,7 @@ impl TransactionBatch { } /// Returns the hash of created notes - pub fn created_notes_hashes(&self) -> impl Iterator + '_ { + pub fn created_notes(&self) -> impl Iterator + '_ { self.txs .iter() .flat_map(|tx| tx.created_notes()) diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index 596683466..e6fa2578a 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -1,5 +1,5 @@ use async_trait::async_trait; -use miden_objects::{Digest, Felt}; +use miden_objects::{accounts::AccountId, Digest, Felt}; use crate::{store::Store, SharedTxBatch}; @@ -21,6 +21,7 @@ pub trait BlockBuilder: Send + Sync + 'static { ) -> Result<(), BuildBlockError>; } +#[derive(Debug)] pub struct DefaultBlockBuilder { store: S, prev_header_hash: Digest, @@ -58,6 +59,13 @@ where ) -> Result<(), BuildBlockError> { // TODO: call store.get_block_inputs() + let updated_accounts: Vec<(AccountId, Digest)> = + batches.iter().map(|batch| batch.updated_accounts()).flatten().collect(); + let created_notes: Vec = + batches.iter().map(|batch| batch.created_notes()).flatten().collect(); + let produced_nullifiers: Vec = + batches.iter().map(|batch| batch.produced_nullifiers()).flatten().collect(); + todo!() } } From b1e574f453547d846682b7d33b7a85ab9e30e63c Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Mon, 30 Oct 2023 16:20:55 -0400 Subject: [PATCH 003/166] build_block() scaffold --- block-producer/src/block.rs | 6 ++ block-producer/src/block_builder/mod.rs | 79 +++++++++++++++++++++---- 2 files changed, 73 insertions(+), 12 deletions(-) diff --git a/block-producer/src/block.rs b/block-producer/src/block.rs index 0ef2706c4..1096949cc 100644 --- a/block-producer/src/block.rs +++ b/block-producer/src/block.rs @@ -11,3 +11,9 @@ pub struct Block { // - full states for created public notes // - zk proof } + +impl Block { + pub fn hash(&self) -> Digest { + todo!() + } +} diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index e6fa2578a..77622506b 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -1,7 +1,13 @@ +use std::{ + sync::Arc, + time::{SystemTime, UNIX_EPOCH}, +}; + use async_trait::async_trait; -use miden_objects::{accounts::AccountId, Digest, Felt}; +use miden_objects::{accounts::AccountId, BlockHeader, Digest, Felt}; +use tokio::sync::RwLock; -use crate::{store::Store, SharedTxBatch}; +use crate::{block::Block, store::Store, SharedTxBatch}; #[derive(Debug, PartialEq)] pub enum BuildBlockError { @@ -23,10 +29,14 @@ pub trait BlockBuilder: Send + Sync + 'static { #[derive(Debug)] pub struct DefaultBlockBuilder { - store: S, - prev_header_hash: Digest, - prev_block_hash: Digest, - prev_block_num: Felt, + store: Arc, + protocol_version: Felt, + /// The hash of the previous header + prev_header_hash: Arc>, + /// The hash of the previous block + prev_block_hash: Arc>, + /// The block number of the next block to build + next_block_num: Arc>, } impl DefaultBlockBuilder @@ -34,16 +44,18 @@ where S: Store, { pub fn new( - store: S, + store: Arc, + protocol_version: Felt, prev_header_hash: Digest, prev_block_hash: Digest, prev_block_num: Felt, ) -> Self { Self { store, - prev_header_hash, - prev_block_hash, - prev_block_num, + protocol_version, + prev_header_hash: Arc::new(RwLock::new(prev_header_hash)), + prev_block_hash: Arc::new(RwLock::new(prev_block_hash)), + next_block_num: Arc::new(RwLock::new(prev_block_num + 1u32.into())), } } } @@ -57,7 +69,7 @@ where &self, batches: Vec, ) -> Result<(), BuildBlockError> { - // TODO: call store.get_block_inputs() + let current_block_num = *self.next_block_num.read().await; let updated_accounts: Vec<(AccountId, Digest)> = batches.iter().map(|batch| batch.updated_accounts()).flatten().collect(); @@ -66,6 +78,49 @@ where let produced_nullifiers: Vec = batches.iter().map(|batch| batch.produced_nullifiers()).flatten().collect(); - todo!() + let header = { + let prev_hash = *self.prev_header_hash.read().await; + let chain_root = Digest::default(); + let account_root = Digest::default(); + let nullifier_root = Digest::default(); + let note_root = Digest::default(); + let batch_root = Digest::default(); + let proof_hash = Digest::default(); + let timestamp: Felt = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("today is expected to be before 1970") + .as_millis() + .into(); + + BlockHeader::new( + prev_hash, + current_block_num, + chain_root, + account_root, + nullifier_root, + note_root, + batch_root, + proof_hash, + self.protocol_version, + timestamp, + ) + }; + + let block = Arc::new(Block { + header, + updated_accounts, + created_notes, + produced_nullifiers, + }); + + // TODO: properly handle + self.store.apply_block(block.clone()).await.expect("apply block failed"); + + // update fields + *self.prev_header_hash.write().await = block.header.hash(); + *self.prev_block_hash.write().await = block.hash(); + *self.next_block_num.write().await = current_block_num + 1u32.into(); + + Ok(()) } } From 2149014d1e5244f30c3231b86e3bf57a5e822ccb Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 31 Oct 2023 08:08:14 -0400 Subject: [PATCH 004/166] Store::get_block_inputs --- block-producer/src/state_view/tests/mod.rs | 21 ++++++++++++- block-producer/src/store/mod.rs | 35 +++++++++++++++++++++- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/block-producer/src/state_view/tests/mod.rs b/block-producer/src/state_view/tests/mod.rs index cb27bcaa5..a4610642b 100644 --- a/block-producer/src/state_view/tests/mod.rs +++ b/block-producer/src/state_view/tests/mod.rs @@ -6,7 +6,10 @@ use miden_objects::{ accounts::get_account_seed, transaction::ConsumedNoteInfo, BlockHeader, Felt, Hasher, }; -use crate::{store::TxInputsError, test_utils::DummyProvenTxGenerator}; +use crate::{ + store::{BlockInputs, BlockInputsError, TxInputsError}, + test_utils::DummyProvenTxGenerator, +}; mod apply_block; mod verify_tx; @@ -90,6 +93,14 @@ impl Store for MockStoreSuccess { nullifiers, }) } + + async fn get_block_inputs( + &self, + _modified_account_ids: &[AccountId], + _produced_nullifiers: &[Digest], + ) -> Result { + unimplemented!() + } } #[derive(Default)] @@ -113,6 +124,14 @@ impl Store for MockStoreFailure { ) -> Result { Err(TxInputsError::Dummy) } + + async fn get_block_inputs( + &self, + _modified_account_ids: &[AccountId], + _produced_nullifiers: &[Digest], + ) -> Result { + unimplemented!() + } } // MOCK PRIVATE ACCOUNT diff --git a/block-producer/src/store/mod.rs b/block-producer/src/store/mod.rs index 43a0346af..581786ce5 100644 --- a/block-producer/src/store/mod.rs +++ b/block-producer/src/store/mod.rs @@ -1,7 +1,11 @@ use std::{collections::BTreeMap, sync::Arc}; use async_trait::async_trait; -use miden_objects::Digest; +use miden_objects::{ + accounts::AccountId, + crypto::merkle::{MmrPeaks, PartialMerkleTree}, + BlockHeader, Digest, +}; use crate::{block::Block, SharedProvenTx}; @@ -10,6 +14,11 @@ pub enum TxInputsError { Dummy, } +#[derive(Debug, PartialEq)] +pub enum BlockInputsError { + Dummy, +} + #[derive(Debug, PartialEq)] pub enum ApplyBlockError { Dummy, @@ -32,10 +41,34 @@ pub struct TxInputs { pub nullifiers: BTreeMap, } +/// Information needed from the store to build a block +pub struct BlockInputs { + /// Previous block header + pub block_header: BlockHeader, + + /// MMR peaks for the current chain state + pub chain_peaks: MmrPeaks, + + /// latest account state hashes for all requested account IDs + pub account_states: Vec<(AccountId, Digest)>, + + /// a partial Merkle Tree with paths to all requested accounts in the account TSMT + pub account_proofs: PartialMerkleTree, + + /// a partial Merkle Tree with paths to all requested nullifiers in the account TSMT + pub nullifier_proofs: PartialMerkleTree, +} + #[async_trait] pub trait Store: ApplyBlock { async fn get_tx_inputs( &self, proven_tx: SharedProvenTx, ) -> Result; + + async fn get_block_inputs( + &self, + modified_account_ids: &[AccountId], + produced_nullifiers: &[Digest], + ) -> Result; } From b33878042c08b1cda954185dd2b7d870132360e7 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 31 Oct 2023 08:16:57 -0400 Subject: [PATCH 005/166] get_block_inputs: use iterators --- block-producer/src/block_builder/mod.rs | 9 +++++++++ block-producer/src/state_view/tests/mod.rs | 8 ++++---- block-producer/src/store/mod.rs | 5 +++-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index 77622506b..209e8940a 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -78,6 +78,15 @@ where let produced_nullifiers: Vec = batches.iter().map(|batch| batch.produced_nullifiers()).flatten().collect(); + let block_inputs = self + .store + .get_block_inputs( + updated_accounts.iter().map(|(account_id, _)| account_id), + produced_nullifiers.iter(), + ) + .await + .unwrap(); + let header = { let prev_hash = *self.prev_header_hash.read().await; let chain_root = Digest::default(); diff --git a/block-producer/src/state_view/tests/mod.rs b/block-producer/src/state_view/tests/mod.rs index a4610642b..91fd759e5 100644 --- a/block-producer/src/state_view/tests/mod.rs +++ b/block-producer/src/state_view/tests/mod.rs @@ -96,8 +96,8 @@ impl Store for MockStoreSuccess { async fn get_block_inputs( &self, - _modified_account_ids: &[AccountId], - _produced_nullifiers: &[Digest], + _updated_accounts: impl Iterator + Send, + _produced_nullifiers: impl Iterator + Send, ) -> Result { unimplemented!() } @@ -127,8 +127,8 @@ impl Store for MockStoreFailure { async fn get_block_inputs( &self, - _modified_account_ids: &[AccountId], - _produced_nullifiers: &[Digest], + _updated_accounts: impl Iterator + Send, + _produced_nullifiers: impl Iterator + Send, ) -> Result { unimplemented!() } diff --git a/block-producer/src/store/mod.rs b/block-producer/src/store/mod.rs index 581786ce5..f444a14a4 100644 --- a/block-producer/src/store/mod.rs +++ b/block-producer/src/store/mod.rs @@ -68,7 +68,8 @@ pub trait Store: ApplyBlock { async fn get_block_inputs( &self, - modified_account_ids: &[AccountId], - produced_nullifiers: &[Digest], + // updated_accounts: &[AccountId], + updated_accounts: impl Iterator + Send, + produced_nullifiers: impl Iterator + Send, ) -> Result; } From ea4f766de53e029083b89008fde2ab7fa845f9cc Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 31 Oct 2023 08:22:09 -0400 Subject: [PATCH 006/166] DefaultBlockBuilder: remove some state --- block-producer/src/block_builder/mod.rs | 55 +++++++++---------------- 1 file changed, 19 insertions(+), 36 deletions(-) diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index 209e8940a..6ca80d95c 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -5,9 +5,12 @@ use std::{ use async_trait::async_trait; use miden_objects::{accounts::AccountId, BlockHeader, Digest, Felt}; -use tokio::sync::RwLock; -use crate::{block::Block, store::Store, SharedTxBatch}; +use crate::{ + block::Block, + store::{BlockInputs, Store}, + SharedTxBatch, +}; #[derive(Debug, PartialEq)] pub enum BuildBlockError { @@ -30,33 +33,14 @@ pub trait BlockBuilder: Send + Sync + 'static { #[derive(Debug)] pub struct DefaultBlockBuilder { store: Arc, - protocol_version: Felt, - /// The hash of the previous header - prev_header_hash: Arc>, - /// The hash of the previous block - prev_block_hash: Arc>, - /// The block number of the next block to build - next_block_num: Arc>, } impl DefaultBlockBuilder where S: Store, { - pub fn new( - store: Arc, - protocol_version: Felt, - prev_header_hash: Digest, - prev_block_hash: Digest, - prev_block_num: Felt, - ) -> Self { - Self { - store, - protocol_version, - prev_header_hash: Arc::new(RwLock::new(prev_header_hash)), - prev_block_hash: Arc::new(RwLock::new(prev_block_hash)), - next_block_num: Arc::new(RwLock::new(prev_block_num + 1u32.into())), - } + pub fn new(store: Arc) -> Self { + Self { store } } } @@ -69,8 +53,6 @@ where &self, batches: Vec, ) -> Result<(), BuildBlockError> { - let current_block_num = *self.next_block_num.read().await; - let updated_accounts: Vec<(AccountId, Digest)> = batches.iter().map(|batch| batch.updated_accounts()).flatten().collect(); let created_notes: Vec = @@ -78,7 +60,13 @@ where let produced_nullifiers: Vec = batches.iter().map(|batch| batch.produced_nullifiers()).flatten().collect(); - let block_inputs = self + let BlockInputs { + block_header: prev_block_header, + chain_peaks, + account_states: account_states_in_store, + account_proofs, + nullifier_proofs, + } = self .store .get_block_inputs( updated_accounts.iter().map(|(account_id, _)| account_id), @@ -87,8 +75,8 @@ where .await .unwrap(); - let header = { - let prev_hash = *self.prev_header_hash.read().await; + let new_block_header = { + let prev_hash = prev_block_header.prev_hash(); let chain_root = Digest::default(); let account_root = Digest::default(); let nullifier_root = Digest::default(); @@ -103,20 +91,20 @@ where BlockHeader::new( prev_hash, - current_block_num, + prev_block_header.block_num(), chain_root, account_root, nullifier_root, note_root, batch_root, proof_hash, - self.protocol_version, + prev_block_header.version(), timestamp, ) }; let block = Arc::new(Block { - header, + header: new_block_header, updated_accounts, created_notes, produced_nullifiers, @@ -125,11 +113,6 @@ where // TODO: properly handle self.store.apply_block(block.clone()).await.expect("apply block failed"); - // update fields - *self.prev_header_hash.write().await = block.header.hash(); - *self.prev_block_hash.write().await = block.hash(); - *self.next_block_num.write().await = current_block_num + 1u32.into(); - Ok(()) } } From 4d660336c73127a81954b695ef09818da1b5d3ac Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 31 Oct 2023 09:48:13 -0400 Subject: [PATCH 007/166] clippy --- block-producer/src/block_builder/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index 6ca80d95c..290b127ac 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -54,11 +54,11 @@ where batches: Vec, ) -> Result<(), BuildBlockError> { let updated_accounts: Vec<(AccountId, Digest)> = - batches.iter().map(|batch| batch.updated_accounts()).flatten().collect(); + batches.iter().flat_map(|batch| batch.updated_accounts()).collect(); let created_notes: Vec = - batches.iter().map(|batch| batch.created_notes()).flatten().collect(); + batches.iter().flat_map(|batch| batch.created_notes()).collect(); let produced_nullifiers: Vec = - batches.iter().map(|batch| batch.produced_nullifiers()).flatten().collect(); + batches.iter().flat_map(|batch| batch.produced_nullifiers()).collect(); let BlockInputs { block_header: prev_block_header, From 88858cd5507fcd171251a1f7fb916a452986e5aa Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 31 Oct 2023 11:06:35 -0400 Subject: [PATCH 008/166] account program scaffold --- block-producer/Cargo.toml | 1 + block-producer/src/block_builder/mod.rs | 2 ++ block-producer/src/block_builder/vm.rs | 23 +++++++++++++++++++++++ 3 files changed, 26 insertions(+) create mode 100644 block-producer/src/block_builder/vm.rs diff --git a/block-producer/Cargo.toml b/block-producer/Cargo.toml index 1c9d8a599..a80e47774 100644 --- a/block-producer/Cargo.toml +++ b/block-producer/Cargo.toml @@ -18,4 +18,5 @@ winterfell = "0.6" async-trait = "0.1" itertools = "0.11" miden_objects = { workspace = true } +miden_stdlib = { package = "miden-stdlib", git = "https://github.com/0xPolygonMiden/miden-vm.git", branch = "next", default-features = false } tokio = { version = "1.29", features = ["rt-multi-thread", "macros", "sync", "time"] } diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index 290b127ac..6e60e306d 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -12,6 +12,8 @@ use crate::{ SharedTxBatch, }; +mod vm; + #[derive(Debug, PartialEq)] pub enum BuildBlockError { Dummy, diff --git a/block-producer/src/block_builder/vm.rs b/block-producer/src/block_builder/vm.rs new file mode 100644 index 000000000..60ea73db6 --- /dev/null +++ b/block-producer/src/block_builder/vm.rs @@ -0,0 +1,23 @@ +use miden_objects::{assembly::Assembler, Digest}; +use miden_stdlib::StdLibrary; + +const ACCOUNT_PROGRAM_SOURCE: &'static str = " + + begin + + end +"; + +pub fn compute_new_account_root() -> Digest { + let account_program = { + let assembler = Assembler::default() + .with_library(&StdLibrary::default()) + .expect("failed to load std-lib"); + + assembler + .compile(ACCOUNT_PROGRAM_SOURCE) + .expect("failed to load account update program") + }; + + todo!() +} From 5ac3971d39e8235b42303b637b89011ffae3cd47 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 31 Oct 2023 13:20:52 -0400 Subject: [PATCH 009/166] nullifier scaffold --- block-producer/src/block_builder/vm.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/block-producer/src/block_builder/vm.rs b/block-producer/src/block_builder/vm.rs index 60ea73db6..ee88670eb 100644 --- a/block-producer/src/block_builder/vm.rs +++ b/block-producer/src/block_builder/vm.rs @@ -1,21 +1,20 @@ use miden_objects::{assembly::Assembler, Digest}; use miden_stdlib::StdLibrary; -const ACCOUNT_PROGRAM_SOURCE: &'static str = " - +const NULLIFIER_PROGRAM_SOURCE: &'static str = " begin end "; -pub fn compute_new_account_root() -> Digest { - let account_program = { +pub fn compute_new_nullifier_root() -> Digest { + let nullifier_program = { let assembler = Assembler::default() .with_library(&StdLibrary::default()) .expect("failed to load std-lib"); assembler - .compile(ACCOUNT_PROGRAM_SOURCE) + .compile(NULLIFIER_PROGRAM_SOURCE) .expect("failed to load account update program") }; From db4335fe9e1ce5d921ae644ed437eb82a068db1f Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 31 Oct 2023 15:51:09 -0400 Subject: [PATCH 010/166] create host --- block-producer/Cargo.toml | 1 + block-producer/src/block_builder/vm.rs | 26 ++++++++++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/block-producer/Cargo.toml b/block-producer/Cargo.toml index a80e47774..9ca554aae 100644 --- a/block-producer/Cargo.toml +++ b/block-producer/Cargo.toml @@ -19,4 +19,5 @@ async-trait = "0.1" itertools = "0.11" miden_objects = { workspace = true } miden_stdlib = { package = "miden-stdlib", git = "https://github.com/0xPolygonMiden/miden-vm.git", branch = "next", default-features = false } +miden_vm = { package = "miden-vm", git = "https://github.com/0xPolygonMiden/miden-vm.git", branch = "next", default-features = false } tokio = { version = "1.29", features = ["rt-multi-thread", "macros", "sync", "time"] } diff --git a/block-producer/src/block_builder/vm.rs b/block-producer/src/block_builder/vm.rs index ee88670eb..02f46e30d 100644 --- a/block-producer/src/block_builder/vm.rs +++ b/block-producer/src/block_builder/vm.rs @@ -1,5 +1,10 @@ -use miden_objects::{assembly::Assembler, Digest}; +use miden_objects::{ + assembly::Assembler, + crypto::merkle::{MerkleStore, PartialMerkleTree}, + Digest, Felt, FieldElement, +}; use miden_stdlib::StdLibrary; +use miden_vm::{AdviceInputs, DefaultHost, MemAdviceProvider}; const NULLIFIER_PROGRAM_SOURCE: &'static str = " begin @@ -7,7 +12,11 @@ const NULLIFIER_PROGRAM_SOURCE: &'static str = " end "; -pub fn compute_new_nullifier_root() -> Digest { +/// This method assumes that all the notes associated with the nullifier have *not* been consumed. +pub fn compute_new_nullifier_root( + pmt: &PartialMerkleTree, + nullifiers: impl Iterator, +) -> Digest { let nullifier_program = { let assembler = Assembler::default() .with_library(&StdLibrary::default()) @@ -18,5 +27,18 @@ pub fn compute_new_nullifier_root() -> Digest { .expect("failed to load account update program") }; + let host = { + let advice_inputs = { + let merkle_store: MerkleStore = pmt.into(); + let map = nullifiers.map(|nullifier| (nullifier, vec![Felt::ZERO; 4])); + + AdviceInputs::default().with_merkle_store(merkle_store).with_map(map) + }; + + let advice_provider = MemAdviceProvider::from(advice_inputs); + + DefaultHost::new(advice_provider) + }; + todo!() } From 98ad2cefc17d78057387eb54c9c03c1c120cc7e0 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Mon, 6 Nov 2023 07:33:14 -0500 Subject: [PATCH 011/166] account root: set up MerkleStore --- block-producer/src/block_builder/vm.rs | 27 ++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/block-producer/src/block_builder/vm.rs b/block-producer/src/block_builder/vm.rs index 02f46e30d..06cb238bd 100644 --- a/block-producer/src/block_builder/vm.rs +++ b/block-producer/src/block_builder/vm.rs @@ -1,38 +1,41 @@ use miden_objects::{ assembly::Assembler, - crypto::merkle::{MerkleStore, PartialMerkleTree}, + crypto::merkle::{MerklePath, MerkleStore, PartialMerkleTree}, Digest, Felt, FieldElement, }; use miden_stdlib::StdLibrary; use miden_vm::{AdviceInputs, DefaultHost, MemAdviceProvider}; -const NULLIFIER_PROGRAM_SOURCE: &'static str = " +const ACCOUNT_PROGRAM_SOURCE: &'static str = " begin end "; -/// This method assumes that all the notes associated with the nullifier have *not* been consumed. -pub fn compute_new_nullifier_root( - pmt: &PartialMerkleTree, - nullifiers: impl Iterator, +/// Takes an iterator of (account id, node hash, Merkle path) +pub fn compute_new_account_root( + accounts: impl Iterator<(AccountId, Digest, MerklePath)> ) -> Digest { - let nullifier_program = { + let account_program = { let assembler = Assembler::default() .with_library(&StdLibrary::default()) .expect("failed to load std-lib"); assembler - .compile(NULLIFIER_PROGRAM_SOURCE) + .compile(ACCOUNT_PROGRAM_SOURCE) .expect("failed to load account update program") }; let host = { let advice_inputs = { - let merkle_store: MerkleStore = pmt.into(); - let map = nullifiers.map(|nullifier| (nullifier, vec![Felt::ZERO; 4])); - - AdviceInputs::default().with_merkle_store(merkle_store).with_map(map) + let merkle_store = + MerkleStore::default() + .add_merkle_paths(accounts.map(|(account_id, node_hash, path)| { + (u64::from(account_id), node_hash, path) + })) + .expect("Account SMT has depth 64; all keys are valid"); + + AdviceInputs::default().with_merkle_store(merkle_store) }; let advice_provider = MemAdviceProvider::from(advice_inputs); From 3a6ed99a47a7c5dd9e260a627f4c7687b1e667fd Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Mon, 6 Nov 2023 15:40:53 -0500 Subject: [PATCH 012/166] write account root update masm --- block-producer/src/block_builder/vm.rs | 41 ++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/block-producer/src/block_builder/vm.rs b/block-producer/src/block_builder/vm.rs index 06cb238bd..fbfc5ee86 100644 --- a/block-producer/src/block_builder/vm.rs +++ b/block-producer/src/block_builder/vm.rs @@ -6,15 +6,42 @@ use miden_objects::{ use miden_stdlib::StdLibrary; use miden_vm::{AdviceInputs, DefaultHost, MemAdviceProvider}; -const ACCOUNT_PROGRAM_SOURCE: &'static str = " - begin +/// Stack inputs: +/// [num_accounts_updated, +/// ACCOUNT_ROOT, +/// NEW_ACCOUNT_HASH_0, account_id_0, ... , NEW_ACCOUNT_HASH_n, account_id_n] +const ACCOUNT_UPDATE_ROOT_MASM: &'static str = " +use.std::collections::smt64 +begin + push.1 + while.true + # stack: [counter, ROOT_0, ..., NEW_ACCOUNT_HASH_i, account_id_i , ...] + + # Move counter down for next iteration + movdn.9 + # => [ROOT_i, NEW_ACCOUNT_HASH_i, account_id_i, counter, ...] + + # Move root down (for smt64.set) + movdn.8 movdn.8 movdn.8 movdn.8 + # => [NEW_ACCOUNT_HASH_i, account_id_i, ROOT_i, counter, ...] + + # set new value in SMT + smt64.set dropw + # => [ROOT_{i+1}, counter, ...] + + # loop counter + movup.4 sub.1 dup neq.0 + # => [0 or 1, counter-1, ROOT_{i+1}, ...] end +end "; -/// Takes an iterator of (account id, node hash, Merkle path) +/// `current_account_states`: iterator of (account id, node hash, Merkle path) +/// `account_updates`: iterator of (account id, new account hash) pub fn compute_new_account_root( - accounts: impl Iterator<(AccountId, Digest, MerklePath)> + current_account_states: impl Iterator, + account_updates: impl Iterator, ) -> Digest { let account_program = { let assembler = Assembler::default() @@ -22,7 +49,7 @@ pub fn compute_new_account_root( .expect("failed to load std-lib"); assembler - .compile(ACCOUNT_PROGRAM_SOURCE) + .compile(ACCOUNT_UPDATE_ROOT_MASM) .expect("failed to load account update program") }; @@ -30,7 +57,7 @@ pub fn compute_new_account_root( let advice_inputs = { let merkle_store = MerkleStore::default() - .add_merkle_paths(accounts.map(|(account_id, node_hash, path)| { + .add_merkle_paths(current_account_states.map(|(account_id, node_hash, path)| { (u64::from(account_id), node_hash, path) })) .expect("Account SMT has depth 64; all keys are valid"); @@ -43,5 +70,7 @@ pub fn compute_new_account_root( DefaultHost::new(advice_provider) }; + // TODO: Stack inputs + todo!() } From 90c3e7fc07637fd556fae844fd426334588a13b8 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Mon, 6 Nov 2023 16:49:42 -0500 Subject: [PATCH 013/166] compute_new_account_root() done --- block-producer/src/block_builder/vm.rs | 59 ++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/block-producer/src/block_builder/vm.rs b/block-producer/src/block_builder/vm.rs index fbfc5ee86..183f0d9f7 100644 --- a/block-producer/src/block_builder/vm.rs +++ b/block-producer/src/block_builder/vm.rs @@ -1,13 +1,15 @@ +use miden_air::ExecutionOptions; use miden_objects::{ + accounts::AccountId, assembly::Assembler, crypto::merkle::{MerklePath, MerkleStore, PartialMerkleTree}, Digest, Felt, FieldElement, }; use miden_stdlib::StdLibrary; -use miden_vm::{AdviceInputs, DefaultHost, MemAdviceProvider}; +use miden_vm::{execute, AdviceInputs, DefaultHost, MemAdviceProvider, StackInputs}; -/// Stack inputs: -/// [num_accounts_updated, +/// Stack inputs: +/// [num_accounts_updated, /// ACCOUNT_ROOT, /// NEW_ACCOUNT_HASH_0, account_id_0, ... , NEW_ACCOUNT_HASH_n, account_id_n] const ACCOUNT_UPDATE_ROOT_MASM: &'static str = " @@ -34,6 +36,9 @@ begin movup.4 sub.1 dup neq.0 # => [0 or 1, counter-1, ROOT_{i+1}, ...] end + + drop + # => [ROOT_{n-1}] end "; @@ -42,6 +47,7 @@ end pub fn compute_new_account_root( current_account_states: impl Iterator, account_updates: impl Iterator, + initial_account_root: Digest, ) -> Digest { let account_program = { let assembler = Assembler::default() @@ -57,9 +63,9 @@ pub fn compute_new_account_root( let advice_inputs = { let merkle_store = MerkleStore::default() - .add_merkle_paths(current_account_states.map(|(account_id, node_hash, path)| { - (u64::from(account_id), node_hash, path) - })) + .add_merkle_paths(current_account_states.map( + |(account_id, node_hash, path)| (u64::from(account_id), node_hash, path), + )) .expect("Account SMT has depth 64; all keys are valid"); AdviceInputs::default().with_merkle_store(merkle_store) @@ -70,7 +76,44 @@ pub fn compute_new_account_root( DefaultHost::new(advice_provider) }; - // TODO: Stack inputs + let stack_inputs = { + // Note: `StackInputs::new()` reverses the input vector, so we need to construct the stack + // from the bottom to the top + let mut stack_inputs = Vec::new(); + + // append all insert key/values + let mut num_accounts_updated: u32 = 0; + for (idx, (account_id, new_account_hash)) in account_updates.enumerate() { + stack_inputs.push(account_id.into()); + stack_inputs.append(new_account_hash.as_elements()); + + num_accounts_updated = idx + 1; + } + + // append initial account root + stack_inputs.append(initial_account_root.as_elements()); + + // append number of accounts updated + stack_inputs.push(num_accounts_updated.into()); + + StackInputs::new(stack_inputs) + }; + + let execution_output = + execute(&account_program, stack_inputs, host, ExecutionOptions::default()) + .expect("execution error in account update program"); + + let new_account_root = { + let stack_output = execution_output.stack_outputs().stack_truncated(4); + + stack_output + .into_iter() + .map(|num| Felt::try_from(num).expect("account update program returned invalid Felt")) + // We reverse, since a word `[a, b, c, d]` will be stored on the stack as `[d, c, b, a]` + .rev() + .try_into() + .expect("account update program didn't return a proper RpoDigest") + }; - todo!() + new_account_root } From 3441fafc6c39b3630a0ea625cdefb3dde10c7a0c Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Mon, 6 Nov 2023 16:52:40 -0500 Subject: [PATCH 014/166] rename module --- block-producer/src/block_builder/{vm.rs => account.rs} | 0 block-producer/src/block_builder/mod.rs | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename block-producer/src/block_builder/{vm.rs => account.rs} (100%) diff --git a/block-producer/src/block_builder/vm.rs b/block-producer/src/block_builder/account.rs similarity index 100% rename from block-producer/src/block_builder/vm.rs rename to block-producer/src/block_builder/account.rs diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index 6e60e306d..1b9d78e30 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -12,7 +12,7 @@ use crate::{ SharedTxBatch, }; -mod vm; +mod account; #[derive(Debug, PartialEq)] pub enum BuildBlockError { From 2fd554961679f051dcf58bdb3656675da4bef45a Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 7 Nov 2023 08:03:20 -0500 Subject: [PATCH 015/166] fix build --- block-producer/Cargo.toml | 2 +- block-producer/src/block_builder/account.rs | 33 +++++++++++++-------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/block-producer/Cargo.toml b/block-producer/Cargo.toml index 479fc21fe..bae44c2e5 100644 --- a/block-producer/Cargo.toml +++ b/block-producer/Cargo.toml @@ -10,13 +10,13 @@ edition = "2021" rust-version = "1.73" [dev-dependencies] -miden-air = { package = "miden-air", git = "https://github.com/0xPolygonMiden/miden-vm.git", branch = "next", default-features = false } miden-mock = { package = "miden-mock", git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "main", default-features = false } winterfell = "0.7" [dependencies] async-trait = "0.1" itertools = "0.11" +miden-air = { package = "miden-air", git = "https://github.com/0xPolygonMiden/miden-vm.git", branch = "next", default-features = false } miden_objects = { workspace = true } miden_stdlib = { package = "miden-stdlib", git = "https://github.com/0xPolygonMiden/miden-vm.git", branch = "next", default-features = false } miden_vm = { package = "miden-vm", git = "https://github.com/0xPolygonMiden/miden-vm.git", branch = "next", default-features = false } diff --git a/block-producer/src/block_builder/account.rs b/block-producer/src/block_builder/account.rs index 183f0d9f7..455813ae0 100644 --- a/block-producer/src/block_builder/account.rs +++ b/block-producer/src/block_builder/account.rs @@ -2,8 +2,8 @@ use miden_air::ExecutionOptions; use miden_objects::{ accounts::AccountId, assembly::Assembler, - crypto::merkle::{MerklePath, MerkleStore, PartialMerkleTree}, - Digest, Felt, FieldElement, + crypto::merkle::{MerklePath, MerkleStore}, + Digest, Felt, }; use miden_stdlib::StdLibrary; use miden_vm::{execute, AdviceInputs, DefaultHost, MemAdviceProvider, StackInputs}; @@ -60,16 +60,17 @@ pub fn compute_new_account_root( }; let host = { - let advice_inputs = { - let merkle_store = - MerkleStore::default() + let advice_inputs = + { + let mut merkle_store = MerkleStore::default(); + merkle_store .add_merkle_paths(current_account_states.map( |(account_id, node_hash, path)| (u64::from(account_id), node_hash, path), )) .expect("Account SMT has depth 64; all keys are valid"); - AdviceInputs::default().with_merkle_store(merkle_store) - }; + AdviceInputs::default().with_merkle_store(merkle_store) + }; let advice_provider = MemAdviceProvider::from(advice_inputs); @@ -82,16 +83,19 @@ pub fn compute_new_account_root( let mut stack_inputs = Vec::new(); // append all insert key/values - let mut num_accounts_updated: u32 = 0; + let mut num_accounts_updated: u64 = 0; for (idx, (account_id, new_account_hash)) in account_updates.enumerate() { + let new_account_hash: [Felt; 4] = new_account_hash.into(); stack_inputs.push(account_id.into()); - stack_inputs.append(new_account_hash.as_elements()); + stack_inputs.extend(new_account_hash); + let idx = u64::try_from(idx).expect("can't be more than 2^64 - 1 accounts"); num_accounts_updated = idx + 1; } // append initial account root - stack_inputs.append(initial_account_root.as_elements()); + let initial_account_root: [Felt; 4] = initial_account_root.into(); + stack_inputs.extend(initial_account_root); // append number of accounts updated stack_inputs.push(num_accounts_updated.into()); @@ -106,13 +110,16 @@ pub fn compute_new_account_root( let new_account_root = { let stack_output = execution_output.stack_outputs().stack_truncated(4); - stack_output + let digest_elements: [Felt; 4] = stack_output .into_iter() - .map(|num| Felt::try_from(num).expect("account update program returned invalid Felt")) + .map(|num| Felt::try_from(*num).expect("account update program returned invalid Felt")) // We reverse, since a word `[a, b, c, d]` will be stored on the stack as `[d, c, b, a]` .rev() + .collect::>() .try_into() - .expect("account update program didn't return a proper RpoDigest") + .expect("account update program didn't return a proper RpoDigest"); + + digest_elements.into() }; new_account_root From 70f5414a39277826549b419b263b80b7cdf693e7 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 7 Nov 2023 08:25:18 -0500 Subject: [PATCH 016/166] clippy --- block-producer/src/block_builder/account.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/block-producer/src/block_builder/account.rs b/block-producer/src/block_builder/account.rs index 455813ae0..f7dced6d2 100644 --- a/block-producer/src/block_builder/account.rs +++ b/block-producer/src/block_builder/account.rs @@ -12,7 +12,7 @@ use miden_vm::{execute, AdviceInputs, DefaultHost, MemAdviceProvider, StackInput /// [num_accounts_updated, /// ACCOUNT_ROOT, /// NEW_ACCOUNT_HASH_0, account_id_0, ... , NEW_ACCOUNT_HASH_n, account_id_n] -const ACCOUNT_UPDATE_ROOT_MASM: &'static str = " +const ACCOUNT_UPDATE_ROOT_MASM: &str = " use.std::collections::smt64 begin @@ -111,7 +111,7 @@ pub fn compute_new_account_root( let stack_output = execution_output.stack_outputs().stack_truncated(4); let digest_elements: [Felt; 4] = stack_output - .into_iter() + .iter() .map(|num| Felt::try_from(*num).expect("account update program returned invalid Felt")) // We reverse, since a word `[a, b, c, d]` will be stored on the stack as `[d, c, b, a]` .rev() From 98c649ce4a5648f9e0f2264a07cc6a097deb0679 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 7 Nov 2023 09:44:42 -0500 Subject: [PATCH 017/166] AccountInputRecord conversion --- block-producer/src/block_builder/mod.rs | 3 +- block-producer/src/store/mod.rs | 25 +++++++++++----- proto/Cargo.toml | 1 + proto/src/conversion.rs | 40 ++++++++++++++++++++++++- 4 files changed, 58 insertions(+), 11 deletions(-) diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index 1b9d78e30..b910a41bb 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -66,8 +66,7 @@ where block_header: prev_block_header, chain_peaks, account_states: account_states_in_store, - account_proofs, - nullifier_proofs, + nullifiers, } = self .store .get_block_inputs( diff --git a/block-producer/src/store/mod.rs b/block-producer/src/store/mod.rs index f444a14a4..ad7be55c8 100644 --- a/block-producer/src/store/mod.rs +++ b/block-producer/src/store/mod.rs @@ -3,9 +3,10 @@ use std::{collections::BTreeMap, sync::Arc}; use async_trait::async_trait; use miden_objects::{ accounts::AccountId, - crypto::merkle::{MmrPeaks, PartialMerkleTree}, + crypto::merkle::MmrPeaks, BlockHeader, Digest, }; +use miden_vm::crypto::MerklePath; use crate::{block::Block, SharedProvenTx}; @@ -41,6 +42,17 @@ pub struct TxInputs { pub nullifiers: BTreeMap, } +pub struct AccountInputRecord { + pub account_id: AccountId, + pub account_hash: Digest, + pub proof: MerklePath, +} + +pub struct NullifierInputRecord { + pub nullifier: Digest, + pub proof: MerklePath, +} + /// Information needed from the store to build a block pub struct BlockInputs { /// Previous block header @@ -49,14 +61,11 @@ pub struct BlockInputs { /// MMR peaks for the current chain state pub chain_peaks: MmrPeaks, - /// latest account state hashes for all requested account IDs - pub account_states: Vec<(AccountId, Digest)>, - - /// a partial Merkle Tree with paths to all requested accounts in the account TSMT - pub account_proofs: PartialMerkleTree, + /// The hashes of the requested accounts and their authentication paths + pub account_states: Vec, - /// a partial Merkle Tree with paths to all requested nullifiers in the account TSMT - pub nullifier_proofs: PartialMerkleTree, + /// The requested nullifiers and their authentication paths + pub nullifiers: Vec, } #[async_trait] diff --git a/proto/Cargo.toml b/proto/Cargo.toml index bd7de4148..acd303ab0 100644 --- a/proto/Cargo.toml +++ b/proto/Cargo.toml @@ -12,6 +12,7 @@ rust-version = "1.73" [dependencies] hex = { version = "0.4" } miden-crypto = { workspace = true } +miden-block-producer = { path = "../block-producer", version = "0.1"} miden_objects = { workspace = true } prost = { version = "0.12" } tonic = { version = "0.10" } diff --git a/proto/src/conversion.rs b/proto/src/conversion.rs index 9ffdc5008..8643a97fa 100644 --- a/proto/src/conversion.rs +++ b/proto/src/conversion.rs @@ -1,10 +1,11 @@ use crate::{account_id, block_header, digest, error, merkle, mmr, note, responses, tsmt}; +use miden_block_producer::store::AccountInputRecord; use miden_crypto::{ hash::rpo::RpoDigest, merkle::{MerklePath, MmrDelta, TieredSmtProof}, Felt, FieldElement, StarkField, Word, }; -use miden_objects::BlockHeader; +use miden_objects::{accounts::AccountId, BlockHeader}; impl From<[u64; 4]> for digest::Digest { fn from(value: [u64; 4]) -> Self { @@ -192,6 +193,14 @@ impl From for merkle::MerklePath { } } +impl TryFrom for MerklePath { + type Error = error::ParseError; + + fn try_from(merkle_path: merkle::MerklePath) -> Result { + merkle_path.siblings.into_iter().map(|v| v.try_into()).collect() + } +} + impl From for responses::NoteSyncRecord { fn from(value: note::Note) -> Self { Self { @@ -216,3 +225,32 @@ impl From for account_id::AccountId { account_id::AccountId { id: value } } } + +impl TryFrom for AccountId { + type Error = error::ParseError; + + fn try_from(account_id: account_id::AccountId) -> Result { + account_id.id.try_into().map_err(|_| error::ParseError::NotAValidFelt) + } +} + +impl TryFrom for AccountInputRecord { + type Error = error::ParseError; + + fn try_from(account_input_record: responses::AccountInputRecord) -> Result { + Ok(Self { + account_id: account_input_record + .account_id + .ok_or(error::ParseError::ProtobufMissingData)? + .try_into()?, + account_hash: account_input_record + .account_hash + .ok_or(error::ParseError::ProtobufMissingData)? + .try_into()?, + proof: account_input_record + .proof + .ok_or(error::ParseError::ProtobufMissingData)? + .try_into()?, + }) + } +} From c1d75400199b2889130658f2220cf19db5e9cd1e Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 7 Nov 2023 09:54:42 -0500 Subject: [PATCH 018/166] NullifierInputRecord conversion --- proto/src/conversion.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/proto/src/conversion.rs b/proto/src/conversion.rs index 8643a97fa..38aea97d2 100644 --- a/proto/src/conversion.rs +++ b/proto/src/conversion.rs @@ -1,5 +1,5 @@ use crate::{account_id, block_header, digest, error, merkle, mmr, note, responses, tsmt}; -use miden_block_producer::store::AccountInputRecord; +use miden_block_producer::store::{AccountInputRecord, NullifierInputRecord}; use miden_crypto::{ hash::rpo::RpoDigest, merkle::{MerklePath, MmrDelta, TieredSmtProof}, @@ -254,3 +254,22 @@ impl TryFrom for AccountInputRecord { }) } } + +impl TryFrom for NullifierInputRecord { + type Error = error::ParseError; + + fn try_from( + nullifier_input_record: responses::NullifierInputRecord + ) -> Result { + Ok(Self { + nullifier: nullifier_input_record + .nullifier + .ok_or(error::ParseError::ProtobufMissingData)? + .try_into()?, + proof: nullifier_input_record + .proof + .ok_or(error::ParseError::ProtobufMissingData)? + .try_into()?, + }) + } +} From cc6f0cb47b9686a11f2a17393da55b8c0974bc04 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 7 Nov 2023 10:26:17 -0500 Subject: [PATCH 019/166] BlockInputs conversion --- proto/src/conversion.rs | 43 +++++++++++++++++++++++++++++++++++++++-- proto/src/error.rs | 19 +++++++++++++++--- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/proto/src/conversion.rs b/proto/src/conversion.rs index 38aea97d2..da2a768cb 100644 --- a/proto/src/conversion.rs +++ b/proto/src/conversion.rs @@ -1,8 +1,8 @@ use crate::{account_id, block_header, digest, error, merkle, mmr, note, responses, tsmt}; -use miden_block_producer::store::{AccountInputRecord, NullifierInputRecord}; +use miden_block_producer::store::{AccountInputRecord, BlockInputs, NullifierInputRecord}; use miden_crypto::{ hash::rpo::RpoDigest, - merkle::{MerklePath, MmrDelta, TieredSmtProof}, + merkle::{MerklePath, MmrDelta, MmrPeaks, TieredSmtProof}, Felt, FieldElement, StarkField, Word, }; use miden_objects::{accounts::AccountId, BlockHeader}; @@ -273,3 +273,42 @@ impl TryFrom for NullifierInputRecord { }) } } + +impl TryFrom for BlockInputs { + type Error = error::ParseError; + + fn try_from(get_block_inputs: responses::GetBlockInputsResponse) -> Result { + let block_header: BlockHeader = get_block_inputs + .block_header + .ok_or(error::ParseError::ProtobufMissingData)? + .try_into()?; + + let chain_peaks = { + let num_leaves: u64 = block_header.block_num().into(); + MmrPeaks::new( + num_leaves.try_into().map_err(|_| error::ParseError::TooManyMmrPeaks)?, + get_block_inputs + .mmr_peaks + .into_iter() + .map(|peak| peak.try_into()) + .collect::>()?, + ) + .map_err(|x| error::ParseError::MmrPeaksError(x))? + }; + + Ok(Self { + block_header, + chain_peaks, + account_states: get_block_inputs + .account_states + .into_iter() + .map(|record| record.try_into()) + .collect::>()?, + nullifiers: get_block_inputs + .nullifiers + .into_iter() + .map(|record| record.try_into()) + .collect::>()?, + }) + } +} diff --git a/proto/src/error.rs b/proto/src/error.rs index 31c542314..20b3e7e4c 100644 --- a/proto/src/error.rs +++ b/proto/src/error.rs @@ -1,9 +1,20 @@ -#[derive(Copy, Clone, Debug, PartialEq)] +use miden_crypto::merkle::MmrError; + +#[derive(Clone, Debug, PartialEq)] pub enum ParseError { HexError(hex::FromHexError), - TooMuchData { expected: usize, got: usize }, - InsufficientData { expected: usize, got: usize }, + TooMuchData { + expected: usize, + got: usize, + }, + InsufficientData { + expected: usize, + got: usize, + }, MissingLeafKey, + MmrPeaksError(MmrError), + /// + TooManyMmrPeaks, NotAValidFelt, InvalidProof, ProtobufMissingData, @@ -34,6 +45,8 @@ impl std::fmt::Display for ParseError { write!(f, "Received TSMT proof is invalid") }, ParseError::ProtobufMissingData => write!(f, "Protobuf message missing data"), + ParseError::MmrPeaksError(err) => write!(f, "MmrPeaks error: {err}"), + ParseError::TooManyMmrPeaks => write!(f, "Number of MmrPeaks doesn't fit into memory"), } } } From 71052f3f944b59cd14d38217d9a6fa77f1ad497b Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 7 Nov 2023 10:28:17 -0500 Subject: [PATCH 020/166] add TODO --- block-producer/src/block_builder/account.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/block-producer/src/block_builder/account.rs b/block-producer/src/block_builder/account.rs index f7dced6d2..642550628 100644 --- a/block-producer/src/block_builder/account.rs +++ b/block-producer/src/block_builder/account.rs @@ -42,6 +42,7 @@ begin end "; +// TODO: COMPILE ONLY ONCE /// `current_account_states`: iterator of (account id, node hash, Merkle path) /// `account_updates`: iterator of (account id, new account hash) pub fn compute_new_account_root( From 90275ce36450553773f92903534b7bf32e3ae1d8 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 7 Nov 2023 10:45:35 -0500 Subject: [PATCH 021/166] call `compute_new_account_root()` --- block-producer/src/block_builder/mod.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index b910a41bb..86601a77e 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -12,6 +12,8 @@ use crate::{ SharedTxBatch, }; +use self::account::compute_new_account_root; + mod account; #[derive(Debug, PartialEq)] @@ -64,9 +66,9 @@ where let BlockInputs { block_header: prev_block_header, - chain_peaks, + chain_peaks: _, account_states: account_states_in_store, - nullifiers, + nullifiers: _, } = self .store .get_block_inputs( @@ -79,7 +81,13 @@ where let new_block_header = { let prev_hash = prev_block_header.prev_hash(); let chain_root = Digest::default(); - let account_root = Digest::default(); + let account_root = compute_new_account_root( + account_states_in_store + .into_iter() + .map(|record| (record.account_id, record.account_hash, record.proof)), + updated_accounts.iter().cloned(), + prev_block_header.account_root(), + ); let nullifier_root = Digest::default(); let note_root = Digest::default(); let batch_root = Digest::default(); From 07fae62a1a810179405217eb43c4ada56de49687 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 7 Nov 2023 10:45:42 -0500 Subject: [PATCH 022/166] fmt --- block-producer/src/store/mod.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/block-producer/src/store/mod.rs b/block-producer/src/store/mod.rs index ad7be55c8..1fa3119c5 100644 --- a/block-producer/src/store/mod.rs +++ b/block-producer/src/store/mod.rs @@ -1,11 +1,7 @@ use std::{collections::BTreeMap, sync::Arc}; use async_trait::async_trait; -use miden_objects::{ - accounts::AccountId, - crypto::merkle::MmrPeaks, - BlockHeader, Digest, -}; +use miden_objects::{accounts::AccountId, crypto::merkle::MmrPeaks, BlockHeader, Digest}; use miden_vm::crypto::MerklePath; use crate::{block::Block, SharedProvenTx}; From 96173034a93c47e2e7917cfdf76ce6566880987e Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 7 Nov 2023 10:53:09 -0500 Subject: [PATCH 023/166] compile account program only once --- block-producer/src/block_builder/account.rs | 120 +++++++++++--------- block-producer/src/block_builder/mod.rs | 10 +- 2 files changed, 73 insertions(+), 57 deletions(-) diff --git a/block-producer/src/block_builder/account.rs b/block-producer/src/block_builder/account.rs index 642550628..c8657aa07 100644 --- a/block-producer/src/block_builder/account.rs +++ b/block-producer/src/block_builder/account.rs @@ -6,7 +6,7 @@ use miden_objects::{ Digest, Felt, }; use miden_stdlib::StdLibrary; -use miden_vm::{execute, AdviceInputs, DefaultHost, MemAdviceProvider, StackInputs}; +use miden_vm::{execute, AdviceInputs, DefaultHost, MemAdviceProvider, Program, StackInputs}; /// Stack inputs: /// [num_accounts_updated, @@ -42,27 +42,38 @@ begin end "; -// TODO: COMPILE ONLY ONCE -/// `current_account_states`: iterator of (account id, node hash, Merkle path) -/// `account_updates`: iterator of (account id, new account hash) -pub fn compute_new_account_root( - current_account_states: impl Iterator, - account_updates: impl Iterator, - initial_account_root: Digest, -) -> Digest { - let account_program = { - let assembler = Assembler::default() - .with_library(&StdLibrary::default()) - .expect("failed to load std-lib"); - - assembler - .compile(ACCOUNT_UPDATE_ROOT_MASM) - .expect("failed to load account update program") - }; - - let host = { - let advice_inputs = - { +#[derive(Debug)] +pub struct AccountRootProgram { + program: Program, +} + +impl AccountRootProgram { + pub fn new() -> Self { + let account_program = { + let assembler = Assembler::default() + .with_library(&StdLibrary::default()) + .expect("failed to load std-lib"); + + assembler + .compile(ACCOUNT_UPDATE_ROOT_MASM) + .expect("failed to load account update program") + }; + + Self { + program: account_program, + } + } + + /// `current_account_states`: iterator of (account id, node hash, Merkle path) + /// `account_updates`: iterator of (account id, new account hash) + pub fn compute_new_account_root( + &self, + current_account_states: impl Iterator, + account_updates: impl Iterator, + initial_account_root: Digest, + ) -> Digest { + let host = { + let advice_inputs = { let mut merkle_store = MerkleStore::default(); merkle_store .add_merkle_paths(current_account_states.map( @@ -73,45 +84,45 @@ pub fn compute_new_account_root( AdviceInputs::default().with_merkle_store(merkle_store) }; - let advice_provider = MemAdviceProvider::from(advice_inputs); + let advice_provider = MemAdviceProvider::from(advice_inputs); - DefaultHost::new(advice_provider) - }; + DefaultHost::new(advice_provider) + }; - let stack_inputs = { - // Note: `StackInputs::new()` reverses the input vector, so we need to construct the stack - // from the bottom to the top - let mut stack_inputs = Vec::new(); + let stack_inputs = { + // Note: `StackInputs::new()` reverses the input vector, so we need to construct the stack + // from the bottom to the top + let mut stack_inputs = Vec::new(); - // append all insert key/values - let mut num_accounts_updated: u64 = 0; - for (idx, (account_id, new_account_hash)) in account_updates.enumerate() { - let new_account_hash: [Felt; 4] = new_account_hash.into(); - stack_inputs.push(account_id.into()); - stack_inputs.extend(new_account_hash); + // append all insert key/values + let mut num_accounts_updated: u64 = 0; + for (idx, (account_id, new_account_hash)) in account_updates.enumerate() { + let new_account_hash: [Felt; 4] = new_account_hash.into(); + stack_inputs.push(account_id.into()); + stack_inputs.extend(new_account_hash); - let idx = u64::try_from(idx).expect("can't be more than 2^64 - 1 accounts"); - num_accounts_updated = idx + 1; - } + let idx = u64::try_from(idx).expect("can't be more than 2^64 - 1 accounts"); + num_accounts_updated = idx + 1; + } - // append initial account root - let initial_account_root: [Felt; 4] = initial_account_root.into(); - stack_inputs.extend(initial_account_root); + // append initial account root + let initial_account_root: [Felt; 4] = initial_account_root.into(); + stack_inputs.extend(initial_account_root); - // append number of accounts updated - stack_inputs.push(num_accounts_updated.into()); + // append number of accounts updated + stack_inputs.push(num_accounts_updated.into()); - StackInputs::new(stack_inputs) - }; + StackInputs::new(stack_inputs) + }; - let execution_output = - execute(&account_program, stack_inputs, host, ExecutionOptions::default()) - .expect("execution error in account update program"); + let execution_output = + execute(&self.program, stack_inputs, host, ExecutionOptions::default()) + .expect("execution error in account update program"); - let new_account_root = { - let stack_output = execution_output.stack_outputs().stack_truncated(4); + let new_account_root = { + let stack_output = execution_output.stack_outputs().stack_truncated(4); - let digest_elements: [Felt; 4] = stack_output + let digest_elements: [Felt; 4] = stack_output .iter() .map(|num| Felt::try_from(*num).expect("account update program returned invalid Felt")) // We reverse, since a word `[a, b, c, d]` will be stored on the stack as `[d, c, b, a]` @@ -120,8 +131,9 @@ pub fn compute_new_account_root( .try_into() .expect("account update program didn't return a proper RpoDigest"); - digest_elements.into() - }; + digest_elements.into() + }; - new_account_root + new_account_root + } } diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index 86601a77e..5d62945f3 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -12,7 +12,7 @@ use crate::{ SharedTxBatch, }; -use self::account::compute_new_account_root; +use self::account::AccountRootProgram; mod account; @@ -37,6 +37,7 @@ pub trait BlockBuilder: Send + Sync + 'static { #[derive(Debug)] pub struct DefaultBlockBuilder { store: Arc, + account_root_program: AccountRootProgram, } impl DefaultBlockBuilder @@ -44,7 +45,10 @@ where S: Store, { pub fn new(store: Arc) -> Self { - Self { store } + Self { + store, + account_root_program: AccountRootProgram::new(), + } } } @@ -81,7 +85,7 @@ where let new_block_header = { let prev_hash = prev_block_header.prev_hash(); let chain_root = Digest::default(); - let account_root = compute_new_account_root( + let account_root = self.account_root_program.compute_new_account_root( account_states_in_store .into_iter() .map(|record| (record.account_id, record.account_hash, record.proof)), From 84c16dcbc651bc9679b75cce48773a5fad295466 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 7 Nov 2023 11:15:44 -0500 Subject: [PATCH 024/166] remove useless vars before `Vec::extend()` --- block-producer/src/block_builder/account.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/block-producer/src/block_builder/account.rs b/block-producer/src/block_builder/account.rs index c8657aa07..dd07ed866 100644 --- a/block-producer/src/block_builder/account.rs +++ b/block-producer/src/block_builder/account.rs @@ -97,7 +97,6 @@ impl AccountRootProgram { // append all insert key/values let mut num_accounts_updated: u64 = 0; for (idx, (account_id, new_account_hash)) in account_updates.enumerate() { - let new_account_hash: [Felt; 4] = new_account_hash.into(); stack_inputs.push(account_id.into()); stack_inputs.extend(new_account_hash); @@ -106,7 +105,6 @@ impl AccountRootProgram { } // append initial account root - let initial_account_root: [Felt; 4] = initial_account_root.into(); stack_inputs.extend(initial_account_root); // append number of accounts updated From dc96d691dc4826ffb7f9bab0756958c7cfe778f8 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 7 Nov 2023 11:30:45 -0500 Subject: [PATCH 025/166] tests mod --- block-producer/src/block_builder/mod.rs | 3 +++ block-producer/src/block_builder/tests.rs | 1 + 2 files changed, 4 insertions(+) create mode 100644 block-producer/src/block_builder/tests.rs diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index 5d62945f3..1ef59746b 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -16,6 +16,9 @@ use self::account::AccountRootProgram; mod account; +#[cfg(test)] +mod tests; + #[derive(Debug, PartialEq)] pub enum BuildBlockError { Dummy, diff --git a/block-producer/src/block_builder/tests.rs b/block-producer/src/block_builder/tests.rs new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/block-producer/src/block_builder/tests.rs @@ -0,0 +1 @@ + From 0f11c9d65fb8021c891ef7adb7f1decee217f5da Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 7 Nov 2023 11:44:16 -0500 Subject: [PATCH 026/166] AccountRootUpdateError --- Cargo.toml | 1 + block-producer/Cargo.toml | 1 + block-producer/src/block_builder/account.rs | 17 +++++++++++++---- block-producer/src/block_builder/mod.rs | 17 +++++++++++++---- 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 61af74618..3ac8e3513 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,3 +11,4 @@ resolver = "2" [workspace.dependencies] miden-crypto = { package = "miden-crypto", git = "https://github.com/0xPolygonMiden/crypto.git", branch = "next", features = ["std"] } miden_objects = { package = "miden-objects", git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "main", default-features = false } +thiserror = "1.0" diff --git a/block-producer/Cargo.toml b/block-producer/Cargo.toml index bae44c2e5..a13f7ca44 100644 --- a/block-producer/Cargo.toml +++ b/block-producer/Cargo.toml @@ -20,4 +20,5 @@ miden-air = { package = "miden-air", git = "https://github.com/0xPolygonMiden/mi miden_objects = { workspace = true } miden_stdlib = { package = "miden-stdlib", git = "https://github.com/0xPolygonMiden/miden-vm.git", branch = "next", default-features = false } miden_vm = { package = "miden-vm", git = "https://github.com/0xPolygonMiden/miden-vm.git", branch = "next", default-features = false } +thiserror = { workspace = true } tokio = { version = "1.29", features = ["rt-multi-thread", "macros", "sync", "time"] } diff --git a/block-producer/src/block_builder/account.rs b/block-producer/src/block_builder/account.rs index dd07ed866..0376808ad 100644 --- a/block-producer/src/block_builder/account.rs +++ b/block-producer/src/block_builder/account.rs @@ -6,7 +6,16 @@ use miden_objects::{ Digest, Felt, }; use miden_stdlib::StdLibrary; -use miden_vm::{execute, AdviceInputs, DefaultHost, MemAdviceProvider, Program, StackInputs}; +use miden_vm::{ + execute, AdviceInputs, DefaultHost, ExecutionError, MemAdviceProvider, Program, StackInputs, +}; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum AccountRootUpdateError { + #[error("account root update program execution failed")] + ProgramExecutionError(ExecutionError), +} /// Stack inputs: /// [num_accounts_updated, @@ -71,7 +80,7 @@ impl AccountRootProgram { current_account_states: impl Iterator, account_updates: impl Iterator, initial_account_root: Digest, - ) -> Digest { + ) -> Result { let host = { let advice_inputs = { let mut merkle_store = MerkleStore::default(); @@ -115,7 +124,7 @@ impl AccountRootProgram { let execution_output = execute(&self.program, stack_inputs, host, ExecutionOptions::default()) - .expect("execution error in account update program"); + .map_err(AccountRootUpdateError::ProgramExecutionError)?; let new_account_root = { let stack_output = execution_output.stack_outputs().stack_truncated(4); @@ -132,6 +141,6 @@ impl AccountRootProgram { digest_elements.into() }; - new_account_root + Ok(new_account_root) } } diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index 1ef59746b..4a437638d 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -5,6 +5,7 @@ use std::{ use async_trait::async_trait; use miden_objects::{accounts::AccountId, BlockHeader, Digest, Felt}; +use thiserror::Error; use crate::{ block::Block, @@ -12,18 +13,26 @@ use crate::{ SharedTxBatch, }; -use self::account::AccountRootProgram; - mod account; +use self::account::{AccountRootProgram, AccountRootUpdateError}; #[cfg(test)] mod tests; -#[derive(Debug, PartialEq)] +#[derive(Debug, Error)] pub enum BuildBlockError { + #[error("failed to update account root")] + AccountRootUpdateFailed(AccountRootUpdateError), + #[error("dummy")] Dummy, } +impl From for BuildBlockError { + fn from(err: AccountRootUpdateError) -> Self { + Self::AccountRootUpdateFailed(err) + } +} + #[async_trait] pub trait BlockBuilder: Send + Sync + 'static { /// Receive batches to be included in a block. An empty vector indicates that no batches were @@ -94,7 +103,7 @@ where .map(|record| (record.account_id, record.account_hash, record.proof)), updated_accounts.iter().cloned(), prev_block_header.account_root(), - ); + )?; let nullifier_root = Digest::default(); let note_root = Digest::default(); let batch_root = Digest::default(); From 7b5a462303a684cb54b6b5675d1fb989adc081bd Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 7 Nov 2023 12:06:13 -0500 Subject: [PATCH 027/166] account root update error --- block-producer/src/block_builder/account.rs | 27 +++++++++++++-------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/block-producer/src/block_builder/account.rs b/block-producer/src/block_builder/account.rs index 0376808ad..4d994303a 100644 --- a/block-producer/src/block_builder/account.rs +++ b/block-producer/src/block_builder/account.rs @@ -7,14 +7,19 @@ use miden_objects::{ }; use miden_stdlib::StdLibrary; use miden_vm::{ - execute, AdviceInputs, DefaultHost, ExecutionError, MemAdviceProvider, Program, StackInputs, + crypto::MerkleError, execute, AdviceInputs, DefaultHost, ExecutionError, MemAdviceProvider, + Program, StackInputs, }; use thiserror::Error; #[derive(Error, Debug)] pub enum AccountRootUpdateError { - #[error("account root update program execution failed")] - ProgramExecutionError(ExecutionError), + #[error("Received invalid merkle path")] + InvalidMerklePaths(MerkleError), + #[error("program execution failed")] + ProgramExecutionFailed(ExecutionError), + #[error("invalid return value on stack (not a hash)")] + InvalidRootReturned, } /// Stack inputs: @@ -88,7 +93,7 @@ impl AccountRootProgram { .add_merkle_paths(current_account_states.map( |(account_id, node_hash, path)| (u64::from(account_id), node_hash, path), )) - .expect("Account SMT has depth 64; all keys are valid"); + .map_err(AccountRootUpdateError::InvalidMerklePaths)?; AdviceInputs::default().with_merkle_store(merkle_store) }; @@ -124,19 +129,21 @@ impl AccountRootProgram { let execution_output = execute(&self.program, stack_inputs, host, ExecutionOptions::default()) - .map_err(AccountRootUpdateError::ProgramExecutionError)?; + .map_err(AccountRootUpdateError::ProgramExecutionFailed)?; let new_account_root = { let stack_output = execution_output.stack_outputs().stack_truncated(4); - let digest_elements: [Felt; 4] = stack_output + let digest_elements: Vec = stack_output .iter() - .map(|num| Felt::try_from(*num).expect("account update program returned invalid Felt")) + .map(|&num| Felt::try_from(num).map_err(|_|AccountRootUpdateError::InvalidRootReturned)) // We reverse, since a word `[a, b, c, d]` will be stored on the stack as `[d, c, b, a]` .rev() - .collect::>() - .try_into() - .expect("account update program didn't return a proper RpoDigest"); + .collect::>()?; + + let digest_elements: [Felt; 4] = digest_elements + .try_into() + .map_err(|_| AccountRootUpdateError::InvalidRootReturned)?; digest_elements.into() }; From b40dbbb02575484458df881e15babc67669c1912 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 8 Nov 2023 06:34:18 -0500 Subject: [PATCH 028/166] remove empty comment --- proto/src/error.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/proto/src/error.rs b/proto/src/error.rs index 20b3e7e4c..06bb9ea6d 100644 --- a/proto/src/error.rs +++ b/proto/src/error.rs @@ -13,7 +13,6 @@ pub enum ParseError { }, MissingLeafKey, MmrPeaksError(MmrError), - /// TooManyMmrPeaks, NotAValidFelt, InvalidProof, From 598809515b448ac60dd7b0d9a0b854829949db08 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 8 Nov 2023 06:37:59 -0500 Subject: [PATCH 029/166] add separation --- block-producer/src/block_builder/mod.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index 4a437638d..35fceb5a9 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -19,6 +19,9 @@ use self::account::{AccountRootProgram, AccountRootUpdateError}; #[cfg(test)] mod tests; +// BLOCK BUILDER +// ================================================================================================= + #[derive(Debug, Error)] pub enum BuildBlockError { #[error("failed to update account root")] From 37b501384f4f704398b358b5da4edc747219141d Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 8 Nov 2023 07:19:55 -0500 Subject: [PATCH 030/166] use OLD_ACCOUNT_ROOT --- block-producer/src/block_builder/account.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block-producer/src/block_builder/account.rs b/block-producer/src/block_builder/account.rs index 4d994303a..6d8c746ee 100644 --- a/block-producer/src/block_builder/account.rs +++ b/block-producer/src/block_builder/account.rs @@ -24,7 +24,7 @@ pub enum AccountRootUpdateError { /// Stack inputs: /// [num_accounts_updated, -/// ACCOUNT_ROOT, +/// OLD_ACCOUNT_ROOT, /// NEW_ACCOUNT_HASH_0, account_id_0, ... , NEW_ACCOUNT_HASH_n, account_id_n] const ACCOUNT_UPDATE_ROOT_MASM: &str = " use.std::collections::smt64 From 8d4ee51c66358db5f2812a4535ce1b3524d20472 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 8 Nov 2023 08:52:41 -0500 Subject: [PATCH 031/166] compute_block_header --- block-producer/src/block_builder/mod.rs | 87 ++++++++++++++----------- 1 file changed, 49 insertions(+), 38 deletions(-) diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index 35fceb5a9..5d8c196f1 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -9,7 +9,7 @@ use thiserror::Error; use crate::{ block::Block, - store::{BlockInputs, Store}, + store::{AccountInputRecord, BlockInputs, Store}, SharedTxBatch, }; @@ -65,6 +65,45 @@ where account_root_program: AccountRootProgram::new(), } } + + fn compute_block_header( + &self, + prev_block_header: &BlockHeader, + account_states: Vec, + account_updates: impl Iterator, + ) -> Result { + let prev_hash = prev_block_header.prev_hash(); + let chain_root = Digest::default(); + let account_root = self.account_root_program.compute_new_account_root( + account_states + .into_iter() + .map(|record| (record.account_id, record.account_hash, record.proof)), + account_updates, + prev_block_header.account_root(), + )?; + let nullifier_root = Digest::default(); + let note_root = Digest::default(); + let batch_root = Digest::default(); + let proof_hash = Digest::default(); + let timestamp: Felt = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("today is expected to be before 1970") + .as_millis() + .into(); + + Ok(BlockHeader::new( + prev_hash, + prev_block_header.block_num(), + chain_root, + account_root, + nullifier_root, + note_root, + batch_root, + proof_hash, + prev_block_header.version(), + timestamp, + )) + } } #[async_trait] @@ -76,7 +115,7 @@ where &self, batches: Vec, ) -> Result<(), BuildBlockError> { - let updated_accounts: Vec<(AccountId, Digest)> = + let account_updates: Vec<(AccountId, Digest)> = batches.iter().flat_map(|batch| batch.updated_accounts()).collect(); let created_notes: Vec = batches.iter().flat_map(|batch| batch.created_notes()).collect(); @@ -86,54 +125,26 @@ where let BlockInputs { block_header: prev_block_header, chain_peaks: _, - account_states: account_states_in_store, + account_states, nullifiers: _, } = self .store .get_block_inputs( - updated_accounts.iter().map(|(account_id, _)| account_id), + account_updates.iter().map(|(account_id, _)| account_id), produced_nullifiers.iter(), ) .await .unwrap(); - let new_block_header = { - let prev_hash = prev_block_header.prev_hash(); - let chain_root = Digest::default(); - let account_root = self.account_root_program.compute_new_account_root( - account_states_in_store - .into_iter() - .map(|record| (record.account_id, record.account_hash, record.proof)), - updated_accounts.iter().cloned(), - prev_block_header.account_root(), - )?; - let nullifier_root = Digest::default(); - let note_root = Digest::default(); - let batch_root = Digest::default(); - let proof_hash = Digest::default(); - let timestamp: Felt = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("today is expected to be before 1970") - .as_millis() - .into(); - - BlockHeader::new( - prev_hash, - prev_block_header.block_num(), - chain_root, - account_root, - nullifier_root, - note_root, - batch_root, - proof_hash, - prev_block_header.version(), - timestamp, - ) - }; + let new_block_header = self.compute_block_header( + &prev_block_header, + account_states, + account_updates.iter().cloned(), + )?; let block = Arc::new(Block { header: new_block_header, - updated_accounts, + updated_accounts: account_updates, created_notes, produced_nullifiers, }); From b5896658791ec0c7e6259d8ba4a8100aac075319 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 8 Nov 2023 09:26:39 -0500 Subject: [PATCH 032/166] conversion: use utilities --- proto/src/conversion.rs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/proto/src/conversion.rs b/proto/src/conversion.rs index 83fd871b2..d30694609 100644 --- a/proto/src/conversion.rs +++ b/proto/src/conversion.rs @@ -252,7 +252,9 @@ impl TryFrom for AccountId { impl TryFrom for AccountInputRecord { type Error = error::ParseError; - fn try_from(account_input_record: responses::AccountBlockInputRecord) -> Result { + fn try_from( + account_input_record: responses::AccountBlockInputRecord + ) -> Result { Ok(Self { account_id: account_input_record .account_id @@ -314,16 +316,8 @@ impl TryFrom for BlockInputs { Ok(Self { block_header, chain_peaks, - account_states: get_block_inputs - .account_states - .into_iter() - .map(|record| record.try_into()) - .collect::>()?, - nullifiers: get_block_inputs - .nullifiers - .into_iter() - .map(|record| record.try_into()) - .collect::>()?, + account_states: try_convert(get_block_inputs.account_states)?, + nullifiers: try_convert(get_block_inputs.nullifiers)?, }) } } From eea858141ac6516b037ab70434771516acdedfd9 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 8 Nov 2023 09:38:56 -0500 Subject: [PATCH 033/166] Move domain types to `miden-node-proto` --- block-producer/Cargo.toml | 1 + block-producer/src/block_builder/mod.rs | 7 ++--- block-producer/src/state_view/tests/mod.rs | 3 ++- block-producer/src/store/mod.rs | 30 ++-------------------- proto/Cargo.toml | 1 - proto/src/conversion.rs | 4 +-- proto/src/domain/mod.rs | 28 ++++++++++++++++++++ proto/src/lib.rs | 1 + 8 files changed, 38 insertions(+), 37 deletions(-) create mode 100644 proto/src/domain/mod.rs diff --git a/block-producer/Cargo.toml b/block-producer/Cargo.toml index a13f7ca44..14e14b2b7 100644 --- a/block-producer/Cargo.toml +++ b/block-producer/Cargo.toml @@ -17,6 +17,7 @@ winterfell = "0.7" async-trait = "0.1" itertools = "0.11" miden-air = { package = "miden-air", git = "https://github.com/0xPolygonMiden/miden-vm.git", branch = "next", default-features = false } +miden-node-proto = { path = "../proto", version = "0.1"} miden_objects = { workspace = true } miden_stdlib = { package = "miden-stdlib", git = "https://github.com/0xPolygonMiden/miden-vm.git", branch = "next", default-features = false } miden_vm = { package = "miden-vm", git = "https://github.com/0xPolygonMiden/miden-vm.git", branch = "next", default-features = false } diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index 5d8c196f1..bd1383e5a 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -4,14 +4,11 @@ use std::{ }; use async_trait::async_trait; +use miden_node_proto::domain::{AccountInputRecord, BlockInputs}; use miden_objects::{accounts::AccountId, BlockHeader, Digest, Felt}; use thiserror::Error; -use crate::{ - block::Block, - store::{AccountInputRecord, BlockInputs, Store}, - SharedTxBatch, -}; +use crate::{block::Block, store::Store, SharedTxBatch}; mod account; use self::account::{AccountRootProgram, AccountRootUpdateError}; diff --git a/block-producer/src/state_view/tests/mod.rs b/block-producer/src/state_view/tests/mod.rs index 91fd759e5..a2fc96d07 100644 --- a/block-producer/src/state_view/tests/mod.rs +++ b/block-producer/src/state_view/tests/mod.rs @@ -2,12 +2,13 @@ use super::*; use std::collections::BTreeMap; +use miden_node_proto::domain::BlockInputs; use miden_objects::{ accounts::get_account_seed, transaction::ConsumedNoteInfo, BlockHeader, Felt, Hasher, }; use crate::{ - store::{BlockInputs, BlockInputsError, TxInputsError}, + store::{BlockInputsError, TxInputsError}, test_utils::DummyProvenTxGenerator, }; diff --git a/block-producer/src/store/mod.rs b/block-producer/src/store/mod.rs index 1fa3119c5..703d3755e 100644 --- a/block-producer/src/store/mod.rs +++ b/block-producer/src/store/mod.rs @@ -1,8 +1,8 @@ use std::{collections::BTreeMap, sync::Arc}; use async_trait::async_trait; -use miden_objects::{accounts::AccountId, crypto::merkle::MmrPeaks, BlockHeader, Digest}; -use miden_vm::crypto::MerklePath; +use miden_node_proto::domain::BlockInputs; +use miden_objects::{accounts::AccountId, Digest}; use crate::{block::Block, SharedProvenTx}; @@ -38,32 +38,6 @@ pub struct TxInputs { pub nullifiers: BTreeMap, } -pub struct AccountInputRecord { - pub account_id: AccountId, - pub account_hash: Digest, - pub proof: MerklePath, -} - -pub struct NullifierInputRecord { - pub nullifier: Digest, - pub proof: MerklePath, -} - -/// Information needed from the store to build a block -pub struct BlockInputs { - /// Previous block header - pub block_header: BlockHeader, - - /// MMR peaks for the current chain state - pub chain_peaks: MmrPeaks, - - /// The hashes of the requested accounts and their authentication paths - pub account_states: Vec, - - /// The requested nullifiers and their authentication paths - pub nullifiers: Vec, -} - #[async_trait] pub trait Store: ApplyBlock { async fn get_tx_inputs( diff --git a/proto/Cargo.toml b/proto/Cargo.toml index acd303ab0..bd7de4148 100644 --- a/proto/Cargo.toml +++ b/proto/Cargo.toml @@ -12,7 +12,6 @@ rust-version = "1.73" [dependencies] hex = { version = "0.4" } miden-crypto = { workspace = true } -miden-block-producer = { path = "../block-producer", version = "0.1"} miden_objects = { workspace = true } prost = { version = "0.12" } tonic = { version = "0.10" } diff --git a/proto/src/conversion.rs b/proto/src/conversion.rs index d30694609..f9727b27e 100644 --- a/proto/src/conversion.rs +++ b/proto/src/conversion.rs @@ -1,5 +1,5 @@ +use crate::domain::{AccountInputRecord, BlockInputs, NullifierInputRecord}; use crate::{account_id, block_header, digest, error, merkle, mmr, note, responses, tsmt}; -use miden_block_producer::store::{AccountInputRecord, BlockInputs, NullifierInputRecord}; use miden_crypto::{ hash::rpo::RpoDigest, merkle::{MerklePath, MmrDelta, MmrPeaks, TieredSmtProof}, @@ -310,7 +310,7 @@ impl TryFrom for BlockInputs { .map(|peak| peak.try_into()) .collect::>()?, ) - .map_err(|x| error::ParseError::MmrPeaksError(x))? + .map_err(error::ParseError::MmrPeaksError)? }; Ok(Self { diff --git a/proto/src/domain/mod.rs b/proto/src/domain/mod.rs new file mode 100644 index 000000000..4f0f07b33 --- /dev/null +++ b/proto/src/domain/mod.rs @@ -0,0 +1,28 @@ +use miden_crypto::merkle::{MerklePath, MmrPeaks}; +use miden_objects::{Digest, BlockHeader, accounts::AccountId}; + +pub struct AccountInputRecord { + pub account_id: AccountId, + pub account_hash: Digest, + pub proof: MerklePath, +} + +pub struct NullifierInputRecord { + pub nullifier: Digest, + pub proof: MerklePath, +} + +/// Information needed from the store to build a block +pub struct BlockInputs { + /// Previous block header + pub block_header: BlockHeader, + + /// MMR peaks for the current chain state + pub chain_peaks: MmrPeaks, + + /// The hashes of the requested accounts and their authentication paths + pub account_states: Vec, + + /// The requested nullifiers and their authentication paths + pub nullifiers: Vec, +} diff --git a/proto/src/lib.rs b/proto/src/lib.rs index 0b52d4ac8..6a61b2d8c 100644 --- a/proto/src/lib.rs +++ b/proto/src/lib.rs @@ -1,4 +1,5 @@ pub mod conversion; +pub mod domain; pub mod error; mod generated; pub mod hex; From b9adce5daa608348784bdf05801650e98637bf2f Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 8 Nov 2023 15:37:15 -0500 Subject: [PATCH 034/166] BlockHeaderWitness --- block-producer/src/block_builder/mod.rs | 84 +++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index bd1383e5a..24cfecaa2 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -6,6 +6,7 @@ use std::{ use async_trait::async_trait; use miden_node_proto::domain::{AccountInputRecord, BlockInputs}; use miden_objects::{accounts::AccountId, BlockHeader, Digest, Felt}; +use miden_vm::{crypto::MerkleStore, AdviceInputs, StackInputs}; use thiserror::Error; use crate::{block::Block, store::Store, SharedTxBatch}; @@ -152,3 +153,86 @@ where Ok(()) } } + +// HELPERS +// ================================================================================================= + +/// Provides inputs to the `BlockKernel` so that it can generate the new header +struct BlockHeaderWitness { + account_states: Vec, + account_updates: Vec<(AccountId, Digest)>, + + prev_header: BlockHeader, +} + +impl BlockHeaderWitness { + fn new( + block_inputs: BlockInputs, + batches: Vec, + prev_header: BlockHeader, + ) -> Result { + // TODO: VALIDATE + // - initial account states in `BlockInputs` are the same as in batches + // - Block height returned for each nullifier is 0. + + + let account_updates: Vec<(AccountId, Digest)> = + batches.iter().flat_map(|batch| batch.updated_accounts()).collect(); + + Ok(Self { + account_states: block_inputs.account_states, + account_updates, + prev_header, + }) + } + + fn into_parts(self) -> Result<(AdviceInputs, StackInputs), BuildBlockError> { + let advice_inputs = { + let mut merkle_store = MerkleStore::default(); + merkle_store + .add_merkle_paths(self.account_states.into_iter().map( + |AccountInputRecord { + account_id, + account_hash, + proof, + }| (u64::from(account_id), account_hash, proof), + )) + .map_err(AccountRootUpdateError::InvalidMerklePaths)?; + + AdviceInputs::default().with_merkle_store(merkle_store) + }; + + let stack_inputs = { + // Note: `StackInputs::new()` reverses the input vector, so we need to construct the stack + // from the bottom to the top + let mut stack_inputs = Vec::new(); + + // append all insert key/values + let mut num_accounts_updated: u64 = 0; + for (idx, (account_id, new_account_hash)) in + self.account_updates.into_iter().enumerate() + { + stack_inputs.push(account_id.into()); + stack_inputs.extend(new_account_hash); + + let idx = u64::try_from(idx).expect("can't be more than 2^64 - 1 accounts"); + num_accounts_updated = idx + 1; + } + + // append initial account root + stack_inputs.extend(self.prev_header.account_root()); + + // append number of accounts updated + stack_inputs.push(num_accounts_updated.into()); + + StackInputs::new(stack_inputs) + }; + + Ok((advice_inputs, stack_inputs)) + } + + /// Note: this method will no longer be necessary when we will have a full block kernel. + fn get_previous_header(&self) -> &BlockHeader { + &self.prev_header + } +} From 8a9d23f99da1cd04831f92b6c09db1538991d044 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 8 Nov 2023 15:43:58 -0500 Subject: [PATCH 035/166] Rename to kernel terminology --- .../src/block_builder/{account.rs => kernel.rs} | 4 ++-- block-producer/src/block_builder/mod.rs | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) rename block-producer/src/block_builder/{account.rs => kernel.rs} (98%) diff --git a/block-producer/src/block_builder/account.rs b/block-producer/src/block_builder/kernel.rs similarity index 98% rename from block-producer/src/block_builder/account.rs rename to block-producer/src/block_builder/kernel.rs index 6d8c746ee..564ded415 100644 --- a/block-producer/src/block_builder/account.rs +++ b/block-producer/src/block_builder/kernel.rs @@ -57,11 +57,11 @@ end "; #[derive(Debug)] -pub struct AccountRootProgram { +pub struct BlockKernel { program: Program, } -impl AccountRootProgram { +impl BlockKernel { pub fn new() -> Self { let account_program = { let assembler = Assembler::default() diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index 24cfecaa2..537c85dec 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -11,8 +11,8 @@ use thiserror::Error; use crate::{block::Block, store::Store, SharedTxBatch}; -mod account; -use self::account::{AccountRootProgram, AccountRootUpdateError}; +mod kernel; +use self::kernel::{BlockKernel, AccountRootUpdateError}; #[cfg(test)] mod tests; @@ -50,7 +50,7 @@ pub trait BlockBuilder: Send + Sync + 'static { #[derive(Debug)] pub struct DefaultBlockBuilder { store: Arc, - account_root_program: AccountRootProgram, + account_root_program: BlockKernel, } impl DefaultBlockBuilder @@ -60,7 +60,7 @@ where pub fn new(store: Arc) -> Self { Self { store, - account_root_program: AccountRootProgram::new(), + account_root_program: BlockKernel::new(), } } From dcb3197d466da1000b9dde3f1745ed748280d439 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 8 Nov 2023 16:02:43 -0500 Subject: [PATCH 036/166] BlockKernel --- block-producer/src/block_builder/kernel.rs | 182 +++++++++++++++------ block-producer/src/block_builder/mod.rs | 158 ++---------------- 2 files changed, 142 insertions(+), 198 deletions(-) diff --git a/block-producer/src/block_builder/kernel.rs b/block-producer/src/block_builder/kernel.rs index 564ded415..d7c3f3048 100644 --- a/block-producer/src/block_builder/kernel.rs +++ b/block-producer/src/block_builder/kernel.rs @@ -1,9 +1,10 @@ +use std::time::{SystemTime, UNIX_EPOCH}; + use miden_air::ExecutionOptions; +use miden_node_proto::domain::{AccountInputRecord, BlockInputs}; use miden_objects::{ - accounts::AccountId, - assembly::Assembler, - crypto::merkle::{MerklePath, MerkleStore}, - Digest, Felt, + accounts::AccountId, assembly::Assembler, crypto::merkle::MerkleStore, BlockHeader, Digest, + Felt, }; use miden_stdlib::StdLibrary; use miden_vm::{ @@ -12,8 +13,12 @@ use miden_vm::{ }; use thiserror::Error; +use crate::SharedTxBatch; + +use super::BuildBlockError; + #[derive(Error, Debug)] -pub enum AccountRootUpdateError { +pub enum BlockKernelError { #[error("Received invalid merkle path")] InvalidMerklePaths(MerkleError), #[error("program execution failed")] @@ -22,11 +27,12 @@ pub enum AccountRootUpdateError { InvalidRootReturned, } -/// Stack inputs: -/// [num_accounts_updated, -/// OLD_ACCOUNT_ROOT, -/// NEW_ACCOUNT_HASH_0, account_id_0, ... , NEW_ACCOUNT_HASH_n, account_id_n] -const ACCOUNT_UPDATE_ROOT_MASM: &str = " +/// Note: For now, the "block kernel" only computes the account root. Eventually, it will compute +/// the entire block header. +/// +/// Stack inputs: [num_accounts_updated, OLD_ACCOUNT_ROOT, NEW_ACCOUNT_HASH_0, account_id_0, ... , +/// NEW_ACCOUNT_HASH_n, account_id_n] +const BLOCK_KERNEL_MASM: &str = " use.std::collections::smt64 begin @@ -57,7 +63,7 @@ end "; #[derive(Debug)] -pub struct BlockKernel { +pub(super) struct BlockKernel { program: Program, } @@ -69,7 +75,7 @@ impl BlockKernel { .expect("failed to load std-lib"); assembler - .compile(ACCOUNT_UPDATE_ROOT_MASM) + .compile(BLOCK_KERNEL_MASM) .expect("failed to load account update program") }; @@ -78,31 +84,122 @@ impl BlockKernel { } } + // Note: this will eventually all be done in the VM + pub fn compute_block_header( + &self, + witness: BlockHeaderWitness, + ) -> Result { + let prev_hash = witness.prev_header.prev_hash(); + let block_num = witness.prev_header.block_num(); + let version = witness.prev_header.version(); + + let chain_root = Digest::default(); + let account_root = self.compute_new_account_root(witness)?; + let nullifier_root = Digest::default(); + let note_root = Digest::default(); + let batch_root = Digest::default(); + let proof_hash = Digest::default(); + let timestamp: Felt = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("today is expected to be before 1970") + .as_millis() + .into(); + + Ok(BlockHeader::new( + prev_hash, + block_num, + chain_root, + account_root, + nullifier_root, + note_root, + batch_root, + proof_hash, + version, + timestamp, + )) + } + /// `current_account_states`: iterator of (account id, node hash, Merkle path) /// `account_updates`: iterator of (account id, new account hash) - pub fn compute_new_account_root( + fn compute_new_account_root( &self, - current_account_states: impl Iterator, - account_updates: impl Iterator, - initial_account_root: Digest, - ) -> Result { + witness: BlockHeaderWitness, + ) -> Result { + let (advice_inputs, stack_inputs) = witness.into_parts()?; let host = { - let advice_inputs = { - let mut merkle_store = MerkleStore::default(); - merkle_store - .add_merkle_paths(current_account_states.map( - |(account_id, node_hash, path)| (u64::from(account_id), node_hash, path), - )) - .map_err(AccountRootUpdateError::InvalidMerklePaths)?; - - AdviceInputs::default().with_merkle_store(merkle_store) - }; - let advice_provider = MemAdviceProvider::from(advice_inputs); DefaultHost::new(advice_provider) }; + let execution_output = + execute(&self.program, stack_inputs, host, ExecutionOptions::default()) + .map_err(BlockKernelError::ProgramExecutionFailed)?; + + let new_account_root = { + let stack_output = execution_output.stack_outputs().stack_truncated(4); + + let digest_elements: Vec = stack_output + .iter() + .map(|&num| Felt::try_from(num).map_err(|_|BlockKernelError::InvalidRootReturned)) + // We reverse, since a word `[a, b, c, d]` will be stored on the stack as `[d, c, b, a]` + .rev() + .collect::>()?; + + let digest_elements: [Felt; 4] = + digest_elements.try_into().map_err(|_| BlockKernelError::InvalidRootReturned)?; + + digest_elements.into() + }; + + Ok(new_account_root) + } +} + +/// Provides inputs to the `BlockKernel` so that it can generate the new header +pub(super) struct BlockHeaderWitness { + account_states: Vec, + account_updates: Vec<(AccountId, Digest)>, + + /// Note: this field will no longer be necessary when we have a full block kernel. + prev_header: BlockHeader, +} + +impl BlockHeaderWitness { + pub(super) fn new( + block_inputs: BlockInputs, + batches: Vec, + ) -> Result { + // TODO: VALIDATE + // - initial account states in `BlockInputs` are the same as in batches + // - Block height returned for each nullifier is 0. + + let account_updates: Vec<(AccountId, Digest)> = + batches.iter().flat_map(|batch| batch.updated_accounts()).collect(); + + Ok(Self { + account_states: block_inputs.account_states, + account_updates, + prev_header: block_inputs.block_header, + }) + } + + fn into_parts(self) -> Result<(AdviceInputs, StackInputs), BlockKernelError> { + let advice_inputs = { + let mut merkle_store = MerkleStore::default(); + merkle_store + .add_merkle_paths(self.account_states.into_iter().map( + |AccountInputRecord { + account_id, + account_hash, + proof, + }| (u64::from(account_id), account_hash, proof), + )) + .map_err(BlockKernelError::InvalidMerklePaths)?; + + AdviceInputs::default().with_merkle_store(merkle_store) + }; + let stack_inputs = { // Note: `StackInputs::new()` reverses the input vector, so we need to construct the stack // from the bottom to the top @@ -110,7 +207,9 @@ impl BlockKernel { // append all insert key/values let mut num_accounts_updated: u64 = 0; - for (idx, (account_id, new_account_hash)) in account_updates.enumerate() { + for (idx, (account_id, new_account_hash)) in + self.account_updates.into_iter().enumerate() + { stack_inputs.push(account_id.into()); stack_inputs.extend(new_account_hash); @@ -119,7 +218,7 @@ impl BlockKernel { } // append initial account root - stack_inputs.extend(initial_account_root); + stack_inputs.extend(self.prev_header.account_root()); // append number of accounts updated stack_inputs.push(num_accounts_updated.into()); @@ -127,27 +226,6 @@ impl BlockKernel { StackInputs::new(stack_inputs) }; - let execution_output = - execute(&self.program, stack_inputs, host, ExecutionOptions::default()) - .map_err(AccountRootUpdateError::ProgramExecutionFailed)?; - - let new_account_root = { - let stack_output = execution_output.stack_outputs().stack_truncated(4); - - let digest_elements: Vec = stack_output - .iter() - .map(|&num| Felt::try_from(num).map_err(|_|AccountRootUpdateError::InvalidRootReturned)) - // We reverse, since a word `[a, b, c, d]` will be stored on the stack as `[d, c, b, a]` - .rev() - .collect::>()?; - - let digest_elements: [Felt; 4] = digest_elements - .try_into() - .map_err(|_| AccountRootUpdateError::InvalidRootReturned)?; - - digest_elements.into() - }; - - Ok(new_account_root) + Ok((advice_inputs, stack_inputs)) } } diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index 537c85dec..8dc766bae 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -1,18 +1,13 @@ -use std::{ - sync::Arc, - time::{SystemTime, UNIX_EPOCH}, -}; +use std::sync::Arc; use async_trait::async_trait; -use miden_node_proto::domain::{AccountInputRecord, BlockInputs}; -use miden_objects::{accounts::AccountId, BlockHeader, Digest, Felt}; -use miden_vm::{crypto::MerkleStore, AdviceInputs, StackInputs}; +use miden_objects::{accounts::AccountId, Digest}; use thiserror::Error; use crate::{block::Block, store::Store, SharedTxBatch}; mod kernel; -use self::kernel::{BlockKernel, AccountRootUpdateError}; +use self::kernel::{BlockKernelError, BlockHeaderWitness, BlockKernel}; #[cfg(test)] mod tests; @@ -23,13 +18,13 @@ mod tests; #[derive(Debug, Error)] pub enum BuildBlockError { #[error("failed to update account root")] - AccountRootUpdateFailed(AccountRootUpdateError), + AccountRootUpdateFailed(BlockKernelError), #[error("dummy")] Dummy, } -impl From for BuildBlockError { - fn from(err: AccountRootUpdateError) -> Self { +impl From for BuildBlockError { + fn from(err: BlockKernelError) -> Self { Self::AccountRootUpdateFailed(err) } } @@ -50,7 +45,7 @@ pub trait BlockBuilder: Send + Sync + 'static { #[derive(Debug)] pub struct DefaultBlockBuilder { store: Arc, - account_root_program: BlockKernel, + block_kernel: BlockKernel, } impl DefaultBlockBuilder @@ -60,48 +55,9 @@ where pub fn new(store: Arc) -> Self { Self { store, - account_root_program: BlockKernel::new(), + block_kernel: BlockKernel::new(), } } - - fn compute_block_header( - &self, - prev_block_header: &BlockHeader, - account_states: Vec, - account_updates: impl Iterator, - ) -> Result { - let prev_hash = prev_block_header.prev_hash(); - let chain_root = Digest::default(); - let account_root = self.account_root_program.compute_new_account_root( - account_states - .into_iter() - .map(|record| (record.account_id, record.account_hash, record.proof)), - account_updates, - prev_block_header.account_root(), - )?; - let nullifier_root = Digest::default(); - let note_root = Digest::default(); - let batch_root = Digest::default(); - let proof_hash = Digest::default(); - let timestamp: Felt = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("today is expected to be before 1970") - .as_millis() - .into(); - - Ok(BlockHeader::new( - prev_hash, - prev_block_header.block_num(), - chain_root, - account_root, - nullifier_root, - note_root, - batch_root, - proof_hash, - prev_block_header.version(), - timestamp, - )) - } } #[async_trait] @@ -120,12 +76,7 @@ where let produced_nullifiers: Vec = batches.iter().flat_map(|batch| batch.produced_nullifiers()).collect(); - let BlockInputs { - block_header: prev_block_header, - chain_peaks: _, - account_states, - nullifiers: _, - } = self + let block_inputs = self .store .get_block_inputs( account_updates.iter().map(|(account_id, _)| account_id), @@ -134,11 +85,9 @@ where .await .unwrap(); - let new_block_header = self.compute_block_header( - &prev_block_header, - account_states, - account_updates.iter().cloned(), - )?; + let block_header_witness = BlockHeaderWitness::new(block_inputs, batches)?; + + let new_block_header = self.block_kernel.compute_block_header(block_header_witness)?; let block = Arc::new(Block { header: new_block_header, @@ -153,86 +102,3 @@ where Ok(()) } } - -// HELPERS -// ================================================================================================= - -/// Provides inputs to the `BlockKernel` so that it can generate the new header -struct BlockHeaderWitness { - account_states: Vec, - account_updates: Vec<(AccountId, Digest)>, - - prev_header: BlockHeader, -} - -impl BlockHeaderWitness { - fn new( - block_inputs: BlockInputs, - batches: Vec, - prev_header: BlockHeader, - ) -> Result { - // TODO: VALIDATE - // - initial account states in `BlockInputs` are the same as in batches - // - Block height returned for each nullifier is 0. - - - let account_updates: Vec<(AccountId, Digest)> = - batches.iter().flat_map(|batch| batch.updated_accounts()).collect(); - - Ok(Self { - account_states: block_inputs.account_states, - account_updates, - prev_header, - }) - } - - fn into_parts(self) -> Result<(AdviceInputs, StackInputs), BuildBlockError> { - let advice_inputs = { - let mut merkle_store = MerkleStore::default(); - merkle_store - .add_merkle_paths(self.account_states.into_iter().map( - |AccountInputRecord { - account_id, - account_hash, - proof, - }| (u64::from(account_id), account_hash, proof), - )) - .map_err(AccountRootUpdateError::InvalidMerklePaths)?; - - AdviceInputs::default().with_merkle_store(merkle_store) - }; - - let stack_inputs = { - // Note: `StackInputs::new()` reverses the input vector, so we need to construct the stack - // from the bottom to the top - let mut stack_inputs = Vec::new(); - - // append all insert key/values - let mut num_accounts_updated: u64 = 0; - for (idx, (account_id, new_account_hash)) in - self.account_updates.into_iter().enumerate() - { - stack_inputs.push(account_id.into()); - stack_inputs.extend(new_account_hash); - - let idx = u64::try_from(idx).expect("can't be more than 2^64 - 1 accounts"); - num_accounts_updated = idx + 1; - } - - // append initial account root - stack_inputs.extend(self.prev_header.account_root()); - - // append number of accounts updated - stack_inputs.push(num_accounts_updated.into()); - - StackInputs::new(stack_inputs) - }; - - Ok((advice_inputs, stack_inputs)) - } - - /// Note: this method will no longer be necessary when we will have a full block kernel. - fn get_previous_header(&self) -> &BlockHeader { - &self.prev_header - } -} From 3669e59484a1941df8da5872dc8540566ff73d9e Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 8 Nov 2023 16:06:04 -0500 Subject: [PATCH 037/166] BlockHeaderWitness: validate_inputs --- block-producer/src/block_builder/kernel.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/block-producer/src/block_builder/kernel.rs b/block-producer/src/block_builder/kernel.rs index d7c3f3048..1ca386b76 100644 --- a/block-producer/src/block_builder/kernel.rs +++ b/block-producer/src/block_builder/kernel.rs @@ -170,9 +170,7 @@ impl BlockHeaderWitness { block_inputs: BlockInputs, batches: Vec, ) -> Result { - // TODO: VALIDATE - // - initial account states in `BlockInputs` are the same as in batches - // - Block height returned for each nullifier is 0. + Self::validate_inputs(&block_inputs, &batches)?; let account_updates: Vec<(AccountId, Digest)> = batches.iter().flat_map(|batch| batch.updated_accounts()).collect(); @@ -184,6 +182,16 @@ impl BlockHeaderWitness { }) } + fn validate_inputs( + block_inputs: &BlockInputs, + batches: &[SharedTxBatch], + ) -> Result { + // TODO: VALIDATE + // - initial account states in `BlockInputs` are the same as in batches + // - Block height returned for each nullifier is 0. + todo!() + } + fn into_parts(self) -> Result<(AdviceInputs, StackInputs), BlockKernelError> { let advice_inputs = { let mut merkle_store = MerkleStore::default(); From 9bb256b4a1f1b8cc4159074be1e780b799721f64 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 9 Nov 2023 06:51:20 -0500 Subject: [PATCH 038/166] validate_account_states --- block-producer/src/batch_builder/mod.rs | 6 ++ block-producer/src/block_builder/kernel.rs | 67 ++++++++++++++++++++-- block-producer/src/block_builder/mod.rs | 6 +- 3 files changed, 74 insertions(+), 5 deletions(-) diff --git a/block-producer/src/batch_builder/mod.rs b/block-producer/src/batch_builder/mod.rs index 998a0a95b..8d7eeda68 100644 --- a/block-producer/src/batch_builder/mod.rs +++ b/block-producer/src/batch_builder/mod.rs @@ -26,6 +26,12 @@ impl TransactionBatch { Self { txs } } + /// Returns an iterator over account ids that were modified in the transaction batch, and their + /// corresponding initial hash + pub fn account_initial_states(&self) -> impl Iterator + '_ { + self.txs.iter().map(|tx| (tx.account_id(), tx.initial_account_hash())) + } + /// Returns an iterator over account ids that were modified in the transaction batch, and their /// corresponding new hash pub fn updated_accounts(&self) -> impl Iterator + '_ { diff --git a/block-producer/src/block_builder/kernel.rs b/block-producer/src/block_builder/kernel.rs index 1ca386b76..190ffef74 100644 --- a/block-producer/src/block_builder/kernel.rs +++ b/block-producer/src/block_builder/kernel.rs @@ -1,4 +1,7 @@ -use std::time::{SystemTime, UNIX_EPOCH}; +use std::{ + collections::{BTreeMap, BTreeSet}, + time::{SystemTime, UNIX_EPOCH}, +}; use miden_air::ExecutionOptions; use miden_node_proto::domain::{AccountInputRecord, BlockInputs}; @@ -185,11 +188,67 @@ impl BlockHeaderWitness { fn validate_inputs( block_inputs: &BlockInputs, batches: &[SharedTxBatch], - ) -> Result { + ) -> Result<(), BuildBlockError> { // TODO: VALIDATE - // - initial account states in `BlockInputs` are the same as in batches // - Block height returned for each nullifier is 0. - todo!() + + Self::validate_account_states(block_inputs, batches)?; + + Ok(()) + } + + /// Validate that initial account states coming from the batches are the same as the account + /// states returned from the store + fn validate_account_states( + block_inputs: &BlockInputs, + batches: &[SharedTxBatch], + ) -> Result<(), BuildBlockError> { + let batches_initial_states: BTreeMap = + batches.iter().flat_map(|batch| batch.account_initial_states()).collect(); + + let accounts_in_batches: BTreeSet = + batches_initial_states.keys().cloned().collect(); + let accounts_in_store: BTreeSet = block_inputs + .account_states + .iter() + .map(|record| &record.account_id) + .cloned() + .collect(); + + if accounts_in_batches == accounts_in_store { + let accounts_with_different_hashes: Vec = block_inputs + .account_states + .iter() + .filter_map(|record| { + let hash_in_store = record.account_hash; + let hash_in_batches = batches_initial_states + .get(&record.account_id) + .expect("we already verified that account id is contained in batches"); + + if hash_in_store == *hash_in_batches { + None + } else { + Some(record.account_id) + } + }) + .collect(); + + if accounts_with_different_hashes.is_empty() { + Ok(()) + } else { + Err(BuildBlockError::InconsistentAccountStates(accounts_with_different_hashes)) + } + } else { + // The batches and store don't modify the same set of accounts + let union: BTreeSet = + accounts_in_batches.union(&accounts_in_store).cloned().collect(); + let intersection: BTreeSet = + accounts_in_batches.intersection(&accounts_in_store).cloned().collect(); + + let difference: Vec = union.difference(&intersection).cloned().collect(); + + Err(BuildBlockError::InconsistentAccountIds(difference)) + } } fn into_parts(self) -> Result<(AdviceInputs, StackInputs), BlockKernelError> { diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index 8dc766bae..bf7ac9ded 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -7,7 +7,7 @@ use thiserror::Error; use crate::{block::Block, store::Store, SharedTxBatch}; mod kernel; -use self::kernel::{BlockKernelError, BlockHeaderWitness, BlockKernel}; +use self::kernel::{BlockHeaderWitness, BlockKernel, BlockKernelError}; #[cfg(test)] mod tests; @@ -19,6 +19,10 @@ mod tests; pub enum BuildBlockError { #[error("failed to update account root")] AccountRootUpdateFailed(BlockKernelError), + #[error("transaction batches and store don't modify the same account IDs. Offending accounts: {0:?}")] + InconsistentAccountIds(Vec), + #[error("transaction batches and store contain different hashes for some accounts. Offending accounts: {0:?}")] + InconsistentAccountStates(Vec), #[error("dummy")] Dummy, } From 4733d4e7e134d76870bd77a52df8bfcb276547a1 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 9 Nov 2023 07:05:04 -0500 Subject: [PATCH 039/166] BlockHeaderWitness -> BlockWitness --- block-producer/src/block_builder/kernel.rs | 8 ++++---- block-producer/src/block_builder/mod.rs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/block-producer/src/block_builder/kernel.rs b/block-producer/src/block_builder/kernel.rs index 190ffef74..c51363de9 100644 --- a/block-producer/src/block_builder/kernel.rs +++ b/block-producer/src/block_builder/kernel.rs @@ -90,7 +90,7 @@ impl BlockKernel { // Note: this will eventually all be done in the VM pub fn compute_block_header( &self, - witness: BlockHeaderWitness, + witness: BlockWitness, ) -> Result { let prev_hash = witness.prev_header.prev_hash(); let block_num = witness.prev_header.block_num(); @@ -126,7 +126,7 @@ impl BlockKernel { /// `account_updates`: iterator of (account id, new account hash) fn compute_new_account_root( &self, - witness: BlockHeaderWitness, + witness: BlockWitness, ) -> Result { let (advice_inputs, stack_inputs) = witness.into_parts()?; let host = { @@ -160,7 +160,7 @@ impl BlockKernel { } /// Provides inputs to the `BlockKernel` so that it can generate the new header -pub(super) struct BlockHeaderWitness { +pub(super) struct BlockWitness { account_states: Vec, account_updates: Vec<(AccountId, Digest)>, @@ -168,7 +168,7 @@ pub(super) struct BlockHeaderWitness { prev_header: BlockHeader, } -impl BlockHeaderWitness { +impl BlockWitness { pub(super) fn new( block_inputs: BlockInputs, batches: Vec, diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index bf7ac9ded..844371877 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -7,7 +7,7 @@ use thiserror::Error; use crate::{block::Block, store::Store, SharedTxBatch}; mod kernel; -use self::kernel::{BlockHeaderWitness, BlockKernel, BlockKernelError}; +use self::kernel::{BlockWitness, BlockKernel, BlockKernelError}; #[cfg(test)] mod tests; @@ -89,7 +89,7 @@ where .await .unwrap(); - let block_header_witness = BlockHeaderWitness::new(block_inputs, batches)?; + let block_header_witness = BlockWitness::new(block_inputs, batches)?; let new_block_header = self.block_kernel.compute_block_header(block_header_witness)?; From c45f9be92cf65887550d4521852885333094b1f4 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 9 Nov 2023 07:06:24 -0500 Subject: [PATCH 040/166] fmt --- block-producer/src/block_builder/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index 844371877..3afd9fc64 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -7,7 +7,7 @@ use thiserror::Error; use crate::{block::Block, store::Store, SharedTxBatch}; mod kernel; -use self::kernel::{BlockWitness, BlockKernel, BlockKernelError}; +use self::kernel::{BlockKernel, BlockKernelError, BlockWitness}; #[cfg(test)] mod tests; From 1464728c693dd688e367a6fc03a3c93380679165 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 9 Nov 2023 07:06:44 -0500 Subject: [PATCH 041/166] Remove comment --- block-producer/src/block_builder/kernel.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/block-producer/src/block_builder/kernel.rs b/block-producer/src/block_builder/kernel.rs index c51363de9..2bb650958 100644 --- a/block-producer/src/block_builder/kernel.rs +++ b/block-producer/src/block_builder/kernel.rs @@ -163,8 +163,6 @@ impl BlockKernel { pub(super) struct BlockWitness { account_states: Vec, account_updates: Vec<(AccountId, Digest)>, - - /// Note: this field will no longer be necessary when we have a full block kernel. prev_header: BlockHeader, } From 7719b254327b22c7e46d4faa68b617a250feed20 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 9 Nov 2023 07:08:41 -0500 Subject: [PATCH 042/166] comment --- block-producer/src/block_builder/kernel.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block-producer/src/block_builder/kernel.rs b/block-producer/src/block_builder/kernel.rs index 2bb650958..eb109315f 100644 --- a/block-producer/src/block_builder/kernel.rs +++ b/block-producer/src/block_builder/kernel.rs @@ -187,7 +187,7 @@ impl BlockWitness { block_inputs: &BlockInputs, batches: &[SharedTxBatch], ) -> Result<(), BuildBlockError> { - // TODO: VALIDATE + // TODO: // - Block height returned for each nullifier is 0. Self::validate_account_states(block_inputs, batches)?; From 9ae731985c1e0f80477228f1ea0316023756428a Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 9 Nov 2023 08:04:58 -0500 Subject: [PATCH 043/166] errors module --- block-producer/src/batch_builder/tests/mod.rs | 4 ++- block-producer/src/block_builder/errors.rs | 31 +++++++++++++++++++ block-producer/src/block_builder/kernel.rs | 18 ++--------- block-producer/src/block_builder/mod.rs | 26 ++++------------ 4 files changed, 42 insertions(+), 37 deletions(-) create mode 100644 block-producer/src/block_builder/errors.rs diff --git a/block-producer/src/batch_builder/tests/mod.rs b/block-producer/src/batch_builder/tests/mod.rs index 2064a2369..826404a3b 100644 --- a/block-producer/src/batch_builder/tests/mod.rs +++ b/block-producer/src/batch_builder/tests/mod.rs @@ -1,5 +1,7 @@ use super::*; -use crate::{block_builder::BuildBlockError, test_utils::DummyProvenTxGenerator, SharedTxBatch}; +use crate::{ + block_builder::errors::BuildBlockError, test_utils::DummyProvenTxGenerator, SharedTxBatch, +}; // STRUCTS // ================================================================================================ diff --git a/block-producer/src/block_builder/errors.rs b/block-producer/src/block_builder/errors.rs new file mode 100644 index 000000000..50af4c22e --- /dev/null +++ b/block-producer/src/block_builder/errors.rs @@ -0,0 +1,31 @@ +use miden_objects::accounts::AccountId; +use miden_vm::{crypto::MerkleError, ExecutionError}; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum BuildBlockError { + #[error("failed to update account root")] + AccountRootUpdateFailed(BlockKernelError), + #[error("transaction batches and store don't modify the same account IDs. Offending accounts: {0:?}")] + InconsistentAccountIds(Vec), + #[error("transaction batches and store contain different hashes for some accounts. Offending accounts: {0:?}")] + InconsistentAccountStates(Vec), + #[error("dummy")] + Dummy, +} + +impl From for BuildBlockError { + fn from(err: BlockKernelError) -> Self { + Self::AccountRootUpdateFailed(err) + } +} + +#[derive(Error, Debug)] +pub enum BlockKernelError { + #[error("Received invalid merkle path")] + InvalidMerklePaths(MerkleError), + #[error("program execution failed")] + ProgramExecutionFailed(ExecutionError), + #[error("invalid return value on stack (not a hash)")] + InvalidRootReturned, +} diff --git a/block-producer/src/block_builder/kernel.rs b/block-producer/src/block_builder/kernel.rs index eb109315f..197dd7f9c 100644 --- a/block-producer/src/block_builder/kernel.rs +++ b/block-producer/src/block_builder/kernel.rs @@ -10,25 +10,11 @@ use miden_objects::{ Felt, }; use miden_stdlib::StdLibrary; -use miden_vm::{ - crypto::MerkleError, execute, AdviceInputs, DefaultHost, ExecutionError, MemAdviceProvider, - Program, StackInputs, -}; -use thiserror::Error; +use miden_vm::{execute, AdviceInputs, DefaultHost, MemAdviceProvider, Program, StackInputs}; use crate::SharedTxBatch; -use super::BuildBlockError; - -#[derive(Error, Debug)] -pub enum BlockKernelError { - #[error("Received invalid merkle path")] - InvalidMerklePaths(MerkleError), - #[error("program execution failed")] - ProgramExecutionFailed(ExecutionError), - #[error("invalid return value on stack (not a hash)")] - InvalidRootReturned, -} +use super::{errors::BlockKernelError, BuildBlockError}; /// Note: For now, the "block kernel" only computes the account root. Eventually, it will compute /// the entire block header. diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index 3afd9fc64..baf89654a 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -2,12 +2,16 @@ use std::sync::Arc; use async_trait::async_trait; use miden_objects::{accounts::AccountId, Digest}; -use thiserror::Error; use crate::{block::Block, store::Store, SharedTxBatch}; +pub mod errors; + mod kernel; -use self::kernel::{BlockKernel, BlockKernelError, BlockWitness}; +use self::{ + errors::BuildBlockError, + kernel::{BlockKernel, BlockWitness}, +}; #[cfg(test)] mod tests; @@ -15,24 +19,6 @@ mod tests; // BLOCK BUILDER // ================================================================================================= -#[derive(Debug, Error)] -pub enum BuildBlockError { - #[error("failed to update account root")] - AccountRootUpdateFailed(BlockKernelError), - #[error("transaction batches and store don't modify the same account IDs. Offending accounts: {0:?}")] - InconsistentAccountIds(Vec), - #[error("transaction batches and store contain different hashes for some accounts. Offending accounts: {0:?}")] - InconsistentAccountStates(Vec), - #[error("dummy")] - Dummy, -} - -impl From for BuildBlockError { - fn from(err: BlockKernelError) -> Self { - Self::AccountRootUpdateFailed(err) - } -} - #[async_trait] pub trait BlockBuilder: Send + Sync + 'static { /// Receive batches to be included in a block. An empty vector indicates that no batches were From c3c20e2ff153994eefa737b408e9f1f6e53ad285 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 9 Nov 2023 08:07:45 -0500 Subject: [PATCH 044/166] change names --- block-producer/src/block_builder/kernel.rs | 14 +++++++------- block-producer/src/block_builder/mod.rs | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/block-producer/src/block_builder/kernel.rs b/block-producer/src/block_builder/kernel.rs index 197dd7f9c..8451324e3 100644 --- a/block-producer/src/block_builder/kernel.rs +++ b/block-producer/src/block_builder/kernel.rs @@ -52,11 +52,11 @@ end "; #[derive(Debug)] -pub(super) struct BlockKernel { - program: Program, +pub(super) struct BlockProver { + kernel: Program, } -impl BlockKernel { +impl BlockProver { pub fn new() -> Self { let account_program = { let assembler = Assembler::default() @@ -69,12 +69,12 @@ impl BlockKernel { }; Self { - program: account_program, + kernel: account_program, } } - // Note: this will eventually all be done in the VM - pub fn compute_block_header( + // Note: this will eventually all be done in the VM, and also return an `ExecutionProof` + pub fn prove( &self, witness: BlockWitness, ) -> Result { @@ -122,7 +122,7 @@ impl BlockKernel { }; let execution_output = - execute(&self.program, stack_inputs, host, ExecutionOptions::default()) + execute(&self.kernel, stack_inputs, host, ExecutionOptions::default()) .map_err(BlockKernelError::ProgramExecutionFailed)?; let new_account_root = { diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index baf89654a..a507de8f9 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -10,7 +10,7 @@ pub mod errors; mod kernel; use self::{ errors::BuildBlockError, - kernel::{BlockKernel, BlockWitness}, + kernel::{BlockProver, BlockWitness}, }; #[cfg(test)] @@ -35,7 +35,7 @@ pub trait BlockBuilder: Send + Sync + 'static { #[derive(Debug)] pub struct DefaultBlockBuilder { store: Arc, - block_kernel: BlockKernel, + block_kernel: BlockProver, } impl DefaultBlockBuilder @@ -45,7 +45,7 @@ where pub fn new(store: Arc) -> Self { Self { store, - block_kernel: BlockKernel::new(), + block_kernel: BlockProver::new(), } } } @@ -77,7 +77,7 @@ where let block_header_witness = BlockWitness::new(block_inputs, batches)?; - let new_block_header = self.block_kernel.compute_block_header(block_header_witness)?; + let new_block_header = self.block_kernel.prove(block_header_witness)?; let block = Arc::new(Block { header: new_block_header, From dcf644c5a9f41b88081edeebb1017f3b15fe6350 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 9 Nov 2023 08:15:01 -0500 Subject: [PATCH 045/166] rename module: kernel -> prover --- block-producer/src/block_builder/mod.rs | 4 ++-- block-producer/src/block_builder/{kernel.rs => prover.rs} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename block-producer/src/block_builder/{kernel.rs => prover.rs} (100%) diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index a507de8f9..acadf19fe 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -7,10 +7,10 @@ use crate::{block::Block, store::Store, SharedTxBatch}; pub mod errors; -mod kernel; +mod prover; use self::{ errors::BuildBlockError, - kernel::{BlockProver, BlockWitness}, + prover::{BlockProver, BlockWitness}, }; #[cfg(test)] diff --git a/block-producer/src/block_builder/kernel.rs b/block-producer/src/block_builder/prover.rs similarity index 100% rename from block-producer/src/block_builder/kernel.rs rename to block-producer/src/block_builder/prover.rs From f50688b127f13fae4c28401df551a166acede9a0 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 9 Nov 2023 08:51:41 -0500 Subject: [PATCH 046/166] Use `AccountUpdate` in `BlockWitness` --- block-producer/src/block_builder/prover.rs | 93 +++++++++++++++------- 1 file changed, 66 insertions(+), 27 deletions(-) diff --git a/block-producer/src/block_builder/prover.rs b/block-producer/src/block_builder/prover.rs index 8451324e3..f74000145 100644 --- a/block-producer/src/block_builder/prover.rs +++ b/block-producer/src/block_builder/prover.rs @@ -4,13 +4,15 @@ use std::{ }; use miden_air::ExecutionOptions; -use miden_node_proto::domain::{AccountInputRecord, BlockInputs}; +use miden_node_proto::domain::BlockInputs; use miden_objects::{ accounts::AccountId, assembly::Assembler, crypto::merkle::MerkleStore, BlockHeader, Digest, Felt, }; use miden_stdlib::StdLibrary; -use miden_vm::{execute, AdviceInputs, DefaultHost, MemAdviceProvider, Program, StackInputs}; +use miden_vm::{ + crypto::MerklePath, execute, AdviceInputs, DefaultHost, MemAdviceProvider, Program, StackInputs, +}; use crate::SharedTxBatch; @@ -147,8 +149,9 @@ impl BlockProver { /// Provides inputs to the `BlockKernel` so that it can generate the new header pub(super) struct BlockWitness { - account_states: Vec, - account_updates: Vec<(AccountId, Digest)>, + updated_accounts: BTreeMap, + // account_states: Vec, + // account_updates: Vec<(AccountId, Digest)>, prev_header: BlockHeader, } @@ -159,12 +162,41 @@ impl BlockWitness { ) -> Result { Self::validate_inputs(&block_inputs, &batches)?; - let account_updates: Vec<(AccountId, Digest)> = - batches.iter().flat_map(|batch| batch.updated_accounts()).collect(); + let updated_accounts = { + let mut account_initial_states: BTreeMap = + batches.iter().flat_map(|batch| batch.account_initial_states()).collect(); + + let mut account_merkle_proofs: BTreeMap = block_inputs + .account_states + .into_iter() + .map(|record| (record.account_id, record.proof)) + .collect(); + + batches + .iter() + .flat_map(|batch| batch.updated_accounts()) + .map(|(account_id, final_state_hash)| { + let initial_state_hash = account_initial_states + .remove(&account_id) + .expect("already validated that key exists"); + let proof = account_merkle_proofs + .remove(&account_id) + .expect("already validated that key exists"); + + ( + account_id, + AccountUpdate { + initial_state_hash, + final_state_hash, + proof, + }, + ) + }) + .collect() + }; Ok(Self { - account_states: block_inputs.account_states, - account_updates, + updated_accounts, prev_header: block_inputs.block_header, }) } @@ -236,21 +268,6 @@ impl BlockWitness { } fn into_parts(self) -> Result<(AdviceInputs, StackInputs), BlockKernelError> { - let advice_inputs = { - let mut merkle_store = MerkleStore::default(); - merkle_store - .add_merkle_paths(self.account_states.into_iter().map( - |AccountInputRecord { - account_id, - account_hash, - proof, - }| (u64::from(account_id), account_hash, proof), - )) - .map_err(BlockKernelError::InvalidMerklePaths)?; - - AdviceInputs::default().with_merkle_store(merkle_store) - }; - let stack_inputs = { // Note: `StackInputs::new()` reverses the input vector, so we need to construct the stack // from the bottom to the top @@ -258,11 +275,9 @@ impl BlockWitness { // append all insert key/values let mut num_accounts_updated: u64 = 0; - for (idx, (account_id, new_account_hash)) in - self.account_updates.into_iter().enumerate() - { + for (idx, (&account_id, account_update)) in self.updated_accounts.iter().enumerate() { stack_inputs.push(account_id.into()); - stack_inputs.extend(new_account_hash); + stack_inputs.extend(account_update.final_state_hash); let idx = u64::try_from(idx).expect("can't be more than 2^64 - 1 accounts"); num_accounts_updated = idx + 1; @@ -277,6 +292,30 @@ impl BlockWitness { StackInputs::new(stack_inputs) }; + let advice_inputs = { + let mut merkle_store = MerkleStore::default(); + merkle_store + .add_merkle_paths(self.updated_accounts.into_iter().map( + |( + account_id, + AccountUpdate { + initial_state_hash, + final_state_hash: _, + proof, + }, + )| { (u64::from(account_id), initial_state_hash, proof) }, + )) + .map_err(BlockKernelError::InvalidMerklePaths)?; + + AdviceInputs::default().with_merkle_store(merkle_store) + }; + Ok((advice_inputs, stack_inputs)) } } + +pub(super) struct AccountUpdate { + pub initial_state_hash: Digest, + pub final_state_hash: Digest, + pub proof: MerklePath, +} From 8ae7d8593674a649ea6859d372e74ad5775b024b Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 9 Nov 2023 09:06:11 -0500 Subject: [PATCH 047/166] rename error --- block-producer/src/block_builder/errors.rs | 8 ++++---- block-producer/src/block_builder/prover.rs | 16 ++++++++-------- block-producer/src/block_builder/tests.rs | 3 ++- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/block-producer/src/block_builder/errors.rs b/block-producer/src/block_builder/errors.rs index 50af4c22e..592203ff1 100644 --- a/block-producer/src/block_builder/errors.rs +++ b/block-producer/src/block_builder/errors.rs @@ -5,7 +5,7 @@ use thiserror::Error; #[derive(Debug, Error)] pub enum BuildBlockError { #[error("failed to update account root")] - AccountRootUpdateFailed(BlockKernelError), + AccountRootUpdateFailed(BlockProverError), #[error("transaction batches and store don't modify the same account IDs. Offending accounts: {0:?}")] InconsistentAccountIds(Vec), #[error("transaction batches and store contain different hashes for some accounts. Offending accounts: {0:?}")] @@ -14,14 +14,14 @@ pub enum BuildBlockError { Dummy, } -impl From for BuildBlockError { - fn from(err: BlockKernelError) -> Self { +impl From for BuildBlockError { + fn from(err: BlockProverError) -> Self { Self::AccountRootUpdateFailed(err) } } #[derive(Error, Debug)] -pub enum BlockKernelError { +pub enum BlockProverError { #[error("Received invalid merkle path")] InvalidMerklePaths(MerkleError), #[error("program execution failed")] diff --git a/block-producer/src/block_builder/prover.rs b/block-producer/src/block_builder/prover.rs index f74000145..9db7d0cac 100644 --- a/block-producer/src/block_builder/prover.rs +++ b/block-producer/src/block_builder/prover.rs @@ -16,7 +16,7 @@ use miden_vm::{ use crate::SharedTxBatch; -use super::{errors::BlockKernelError, BuildBlockError}; +use super::{errors::BlockProverError, BuildBlockError}; /// Note: For now, the "block kernel" only computes the account root. Eventually, it will compute /// the entire block header. @@ -115,7 +115,7 @@ impl BlockProver { fn compute_new_account_root( &self, witness: BlockWitness, - ) -> Result { + ) -> Result { let (advice_inputs, stack_inputs) = witness.into_parts()?; let host = { let advice_provider = MemAdviceProvider::from(advice_inputs); @@ -125,20 +125,20 @@ impl BlockProver { let execution_output = execute(&self.kernel, stack_inputs, host, ExecutionOptions::default()) - .map_err(BlockKernelError::ProgramExecutionFailed)?; + .map_err(BlockProverError::ProgramExecutionFailed)?; let new_account_root = { let stack_output = execution_output.stack_outputs().stack_truncated(4); let digest_elements: Vec = stack_output .iter() - .map(|&num| Felt::try_from(num).map_err(|_|BlockKernelError::InvalidRootReturned)) + .map(|&num| Felt::try_from(num).map_err(|_|BlockProverError::InvalidRootReturned)) // We reverse, since a word `[a, b, c, d]` will be stored on the stack as `[d, c, b, a]` .rev() - .collect::>()?; + .collect::>()?; let digest_elements: [Felt; 4] = - digest_elements.try_into().map_err(|_| BlockKernelError::InvalidRootReturned)?; + digest_elements.try_into().map_err(|_| BlockProverError::InvalidRootReturned)?; digest_elements.into() }; @@ -267,7 +267,7 @@ impl BlockWitness { } } - fn into_parts(self) -> Result<(AdviceInputs, StackInputs), BlockKernelError> { + fn into_parts(self) -> Result<(AdviceInputs, StackInputs), BlockProverError> { let stack_inputs = { // Note: `StackInputs::new()` reverses the input vector, so we need to construct the stack // from the bottom to the top @@ -305,7 +305,7 @@ impl BlockWitness { }, )| { (u64::from(account_id), initial_state_hash, proof) }, )) - .map_err(BlockKernelError::InvalidMerklePaths)?; + .map_err(BlockProverError::InvalidMerklePaths)?; AdviceInputs::default().with_merkle_store(merkle_store) }; diff --git a/block-producer/src/block_builder/tests.rs b/block-producer/src/block_builder/tests.rs index 8b1378917..7a6ca9b28 100644 --- a/block-producer/src/block_builder/tests.rs +++ b/block-producer/src/block_builder/tests.rs @@ -1 +1,2 @@ - +// tests +// 1. From 8af7e944641fee34e1a8bec9e31137b1d0ba1e90 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 9 Nov 2023 09:23:42 -0500 Subject: [PATCH 048/166] tests comment --- block-producer/src/block_builder/tests.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/block-producer/src/block_builder/tests.rs b/block-producer/src/block_builder/tests.rs index 7a6ca9b28..8539aa10d 100644 --- a/block-producer/src/block_builder/tests.rs +++ b/block-producer/src/block_builder/tests.rs @@ -1,2 +1,8 @@ -// tests -// 1. +// prover tests +// 1. account validation works +// 2. `BlockProver::compute_account_root()` works +// + make the updates outside the VM, and compare root + +// block builder tests (higher level) +// 1. `apply_block()` is called +// 2. if `apply_block()` fails, you fail too From 33efedd65884104b1e1c3684ee932a5d12c5a18a Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 9 Nov 2023 09:25:15 -0500 Subject: [PATCH 049/166] prover-only test file --- .../src/block_builder/{prover.rs => prover/mod.rs} | 3 +++ block-producer/src/block_builder/prover/tests.rs | 4 ++++ block-producer/src/block_builder/tests.rs | 5 ----- 3 files changed, 7 insertions(+), 5 deletions(-) rename block-producer/src/block_builder/{prover.rs => prover/mod.rs} (99%) create mode 100644 block-producer/src/block_builder/prover/tests.rs diff --git a/block-producer/src/block_builder/prover.rs b/block-producer/src/block_builder/prover/mod.rs similarity index 99% rename from block-producer/src/block_builder/prover.rs rename to block-producer/src/block_builder/prover/mod.rs index 9db7d0cac..20961837b 100644 --- a/block-producer/src/block_builder/prover.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -18,6 +18,9 @@ use crate::SharedTxBatch; use super::{errors::BlockProverError, BuildBlockError}; +#[cfg(test)] +mod tests; + /// Note: For now, the "block kernel" only computes the account root. Eventually, it will compute /// the entire block header. /// diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs new file mode 100644 index 000000000..cd061d8b6 --- /dev/null +++ b/block-producer/src/block_builder/prover/tests.rs @@ -0,0 +1,4 @@ +// prover tests +// 1. account validation works +// 2. `BlockProver::compute_account_root()` works +// + make the updates outside the VM, and compare root diff --git a/block-producer/src/block_builder/tests.rs b/block-producer/src/block_builder/tests.rs index 8539aa10d..cf16b751b 100644 --- a/block-producer/src/block_builder/tests.rs +++ b/block-producer/src/block_builder/tests.rs @@ -1,8 +1,3 @@ -// prover tests -// 1. account validation works -// 2. `BlockProver::compute_account_root()` works -// + make the updates outside the VM, and compare root - // block builder tests (higher level) // 1. `apply_block()` is called // 2. if `apply_block()` fails, you fail too From 50246bcf6b6ed5bcef17036dc0c05fda84643435 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 9 Nov 2023 09:30:02 -0500 Subject: [PATCH 050/166] derive Clone,Debug for `BlockInputs` --- proto/src/domain/mod.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/proto/src/domain/mod.rs b/proto/src/domain/mod.rs index 4f0f07b33..93bd16e89 100644 --- a/proto/src/domain/mod.rs +++ b/proto/src/domain/mod.rs @@ -1,18 +1,21 @@ use miden_crypto::merkle::{MerklePath, MmrPeaks}; use miden_objects::{Digest, BlockHeader, accounts::AccountId}; +#[derive(Clone, Debug)] pub struct AccountInputRecord { pub account_id: AccountId, pub account_hash: Digest, pub proof: MerklePath, } +#[derive(Clone, Debug)] pub struct NullifierInputRecord { pub nullifier: Digest, pub proof: MerklePath, } /// Information needed from the store to build a block +#[derive(Clone, Debug)] pub struct BlockInputs { /// Previous block header pub block_header: BlockHeader, From 50a6e2ed37d52596f5d84e06ede200d67070f505 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 9 Nov 2023 10:16:37 -0500 Subject: [PATCH 051/166] test_block_witness_validation_inconsistent_account_ids --- .../src/block_builder/prover/tests.rs | 99 +++++++++++++++++++ proto/src/domain/mod.rs | 2 +- 2 files changed, 100 insertions(+), 1 deletion(-) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index cd061d8b6..88d0a4695 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -2,3 +2,102 @@ // 1. account validation works // 2. `BlockProver::compute_account_root()` works // + make the updates outside the VM, and compare root + +use std::sync::Arc; + +use miden_air::FieldElement; +use miden_node_proto::domain::AccountInputRecord; +use miden_objects::crypto::merkle::MmrPeaks; + +use crate::{batch_builder::TransactionBatch, test_utils::DummyProvenTxGenerator}; + +use super::*; + +/// Tests that `BlockWitness` constructor fails if the store and transaction batches contain a +/// different set of account ids. +/// +/// The store will contain accounts 1 & 2, while the transaction batches will contain 2 & 3. +#[test] +fn test_block_witness_validation_inconsistent_account_ids() { + let tx_gen = DummyProvenTxGenerator::new(); + let account_id_1 = unsafe { AccountId::new_unchecked(Felt::ZERO) }; + let account_id_2 = unsafe { AccountId::new_unchecked(Felt::ONE) }; + let account_id_3 = unsafe { AccountId::new_unchecked(Felt::new(42)) }; + + let block_inputs_from_store: BlockInputs = { + // dummy values + let block_header = BlockHeader::new( + Digest::default(), + Felt::ZERO, + Digest::default(), + Digest::default(), + Digest::default(), + Digest::default(), + Digest::default(), + Digest::default(), + Felt::ZERO, + Felt::ZERO, + ); + let chain_peaks = MmrPeaks::new(0, Vec::new()).unwrap(); + + let account_states = vec![ + AccountInputRecord { + account_id: account_id_1, + account_hash: Digest::default(), + proof: MerklePath::default(), + }, + AccountInputRecord { + account_id: account_id_2, + account_hash: Digest::default(), + proof: MerklePath::default(), + }, + ]; + + BlockInputs { + block_header, + chain_peaks, + account_states, + nullifiers: Vec::new(), + } + }; + + let batches: Vec = { + let batch_1 = { + let tx = Arc::new(tx_gen.dummy_proven_tx_with_params( + account_id_2, + Digest::default(), + Digest::default(), + Vec::new(), + )); + + Arc::new(TransactionBatch::new(vec![tx])) + }; + + let batch_2 = { + let tx = Arc::new(tx_gen.dummy_proven_tx_with_params( + account_id_3, + Digest::default(), + Digest::default(), + Vec::new(), + )); + + Arc::new(TransactionBatch::new(vec![tx])) + }; + + vec![batch_1, batch_2] + }; + + let block_witness_result = BlockWitness::new(block_inputs_from_store, batches); + + assert!(matches!(block_witness_result, Err(BuildBlockError::InconsistentAccountIds(_)))); + + match block_witness_result { + Ok(_) => panic!("incorrect result"), + Err(err) => match err { + BuildBlockError::InconsistentAccountIds(ids) => { + assert_eq!(ids, vec![account_id_1, account_id_3]) + }, + _ => panic!("Incorrect error"), + }, + } +} diff --git a/proto/src/domain/mod.rs b/proto/src/domain/mod.rs index 93bd16e89..6e4b3a5e4 100644 --- a/proto/src/domain/mod.rs +++ b/proto/src/domain/mod.rs @@ -1,5 +1,5 @@ use miden_crypto::merkle::{MerklePath, MmrPeaks}; -use miden_objects::{Digest, BlockHeader, accounts::AccountId}; +use miden_objects::{accounts::AccountId, BlockHeader, Digest}; #[derive(Clone, Debug)] pub struct AccountInputRecord { From 0e99f9500506c11a90431d3b2ab8649790976736 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 9 Nov 2023 15:35:49 -0500 Subject: [PATCH 052/166] use error equality in tests --- block-producer/src/block_builder/errors.rs | 4 +-- .../src/block_builder/prover/mod.rs | 2 ++ .../src/block_builder/prover/tests.rs | 30 ++++--------------- 3 files changed, 10 insertions(+), 26 deletions(-) diff --git a/block-producer/src/block_builder/errors.rs b/block-producer/src/block_builder/errors.rs index 592203ff1..648bdca49 100644 --- a/block-producer/src/block_builder/errors.rs +++ b/block-producer/src/block_builder/errors.rs @@ -2,7 +2,7 @@ use miden_objects::accounts::AccountId; use miden_vm::{crypto::MerkleError, ExecutionError}; use thiserror::Error; -#[derive(Debug, Error)] +#[derive(Debug, Error, PartialEq, Eq)] pub enum BuildBlockError { #[error("failed to update account root")] AccountRootUpdateFailed(BlockProverError), @@ -20,7 +20,7 @@ impl From for BuildBlockError { } } -#[derive(Error, Debug)] +#[derive(Error, Debug, PartialEq, Eq)] pub enum BlockProverError { #[error("Received invalid merkle path")] InvalidMerklePaths(MerkleError), diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 20961837b..16ba640d2 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -151,6 +151,7 @@ impl BlockProver { } /// Provides inputs to the `BlockKernel` so that it can generate the new header +#[derive(Debug, PartialEq, Eq)] pub(super) struct BlockWitness { updated_accounts: BTreeMap, // account_states: Vec, @@ -317,6 +318,7 @@ impl BlockWitness { } } +#[derive(Debug, PartialEq, Eq)] pub(super) struct AccountUpdate { pub initial_state_hash: Digest, pub final_state_hash: Digest, diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index 88d0a4695..ab46e1e7f 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use miden_air::FieldElement; +use miden_mock::mock::block::mock_block_header; use miden_node_proto::domain::AccountInputRecord; use miden_objects::crypto::merkle::MmrPeaks; @@ -25,19 +26,7 @@ fn test_block_witness_validation_inconsistent_account_ids() { let account_id_3 = unsafe { AccountId::new_unchecked(Felt::new(42)) }; let block_inputs_from_store: BlockInputs = { - // dummy values - let block_header = BlockHeader::new( - Digest::default(), - Felt::ZERO, - Digest::default(), - Digest::default(), - Digest::default(), - Digest::default(), - Digest::default(), - Digest::default(), - Felt::ZERO, - Felt::ZERO, - ); + let block_header = mock_block_header(Felt::ZERO, None, None, &[]); let chain_peaks = MmrPeaks::new(0, Vec::new()).unwrap(); let account_states = vec![ @@ -89,15 +78,8 @@ fn test_block_witness_validation_inconsistent_account_ids() { let block_witness_result = BlockWitness::new(block_inputs_from_store, batches); - assert!(matches!(block_witness_result, Err(BuildBlockError::InconsistentAccountIds(_)))); - - match block_witness_result { - Ok(_) => panic!("incorrect result"), - Err(err) => match err { - BuildBlockError::InconsistentAccountIds(ids) => { - assert_eq!(ids, vec![account_id_1, account_id_3]) - }, - _ => panic!("Incorrect error"), - }, - } + assert_eq!( + block_witness_result, + Err(BuildBlockError::InconsistentAccountIds(vec![account_id_1, account_id_3])) + ); } From 37b2c97713843cc6f5650591f4f84d2b9c1d25c3 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 9 Nov 2023 15:41:59 -0500 Subject: [PATCH 053/166] test_block_witness_validation_inconsistent_account_hashes() --- .../src/block_builder/prover/tests.rs | 77 ++++++++++++++++++- 1 file changed, 75 insertions(+), 2 deletions(-) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index ab46e1e7f..efc22eb08 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -1,6 +1,5 @@ // prover tests -// 1. account validation works -// 2. `BlockProver::compute_account_root()` works +// + `BlockProver::compute_account_root()` works // + make the updates outside the VM, and compare root use std::sync::Arc; @@ -83,3 +82,77 @@ fn test_block_witness_validation_inconsistent_account_ids() { Err(BuildBlockError::InconsistentAccountIds(vec![account_id_1, account_id_3])) ); } + +/// Tests that `BlockWitness` constructor fails if the store and transaction batches contain a +/// different at least 1 account who's state hash is different. +/// +/// Only account 1 will have a different state hash +#[test] +fn test_block_witness_validation_inconsistent_account_hashes() { + let tx_gen = DummyProvenTxGenerator::new(); + let account_id_1 = unsafe { AccountId::new_unchecked(Felt::ZERO) }; + let account_id_2 = unsafe { AccountId::new_unchecked(Felt::ONE) }; + + let account_1_hash_store = + Digest::new([Felt::from(1u64), Felt::from(2u64), Felt::from(3u64), Felt::from(4u64)]); + let account_1_hash_batches = + Digest::new([Felt::from(4u64), Felt::from(3u64), Felt::from(2u64), Felt::from(1u64)]); + + let block_inputs_from_store: BlockInputs = { + let block_header = mock_block_header(Felt::ZERO, None, None, &[]); + let chain_peaks = MmrPeaks::new(0, Vec::new()).unwrap(); + + let account_states = vec![ + AccountInputRecord { + account_id: account_id_1, + account_hash: account_1_hash_store, + proof: MerklePath::default(), + }, + AccountInputRecord { + account_id: account_id_2, + account_hash: Digest::default(), + proof: MerklePath::default(), + }, + ]; + + BlockInputs { + block_header, + chain_peaks, + account_states, + nullifiers: Vec::new(), + } + }; + + let batches: Vec = { + let batch_1 = { + let tx = Arc::new(tx_gen.dummy_proven_tx_with_params( + account_id_1, + account_1_hash_batches, + Digest::default(), + Vec::new(), + )); + + Arc::new(TransactionBatch::new(vec![tx])) + }; + + let batch_2 = { + let tx = Arc::new(tx_gen.dummy_proven_tx_with_params( + account_id_2, + Digest::default(), + Digest::default(), + Vec::new(), + )); + + Arc::new(TransactionBatch::new(vec![tx])) + }; + + vec![batch_1, batch_2] + }; + + let block_witness_result = BlockWitness::new(block_inputs_from_store, batches); + + assert_eq!( + block_witness_result, + Err(BuildBlockError::InconsistentAccountStates(vec![account_id_1])) + ); +} From 9053a69c1c4f46eb9c742210e09d10065946ab0c Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 9 Nov 2023 16:33:10 -0500 Subject: [PATCH 054/166] test_compute_account_root_success --- .../src/block_builder/prover/mod.rs | 2 +- .../src/block_builder/prover/tests.rs | 113 +++++++++++++++++- 2 files changed, 110 insertions(+), 5 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 16ba640d2..1f9693b0f 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -43,7 +43,7 @@ begin # => [NEW_ACCOUNT_HASH_i, account_id_i, ROOT_i, counter, ...] # set new value in SMT - smt64.set dropw + exec.smt64::set dropw # => [ROOT_{i+1}, counter, ...] # loop counter diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index efc22eb08..1a46a3dd9 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -1,13 +1,10 @@ -// prover tests -// + `BlockProver::compute_account_root()` works -// + make the updates outside the VM, and compare root - use std::sync::Arc; use miden_air::FieldElement; use miden_mock::mock::block::mock_block_header; use miden_node_proto::domain::AccountInputRecord; use miden_objects::crypto::merkle::MmrPeaks; +use miden_vm::crypto::SimpleSmt; use crate::{batch_builder::TransactionBatch, test_utils::DummyProvenTxGenerator}; @@ -156,3 +153,111 @@ fn test_block_witness_validation_inconsistent_account_hashes() { Err(BuildBlockError::InconsistentAccountStates(vec![account_id_1])) ); } + +/// Tests that the `BlockProver` computes the proper account root. +/// +/// We assume an initial store with 5 accounts, and all will be updated. +#[test] +fn test_compute_account_root_success() { + let tx_gen = DummyProvenTxGenerator::new(); + + // ACCOUNT STATES + // --------------------------------------------------------------------------------------------- + let account_ids = vec![ + unsafe { AccountId::new_unchecked(Felt::from(0b0000_0000_0000_0000u64)) }, + unsafe { AccountId::new_unchecked(Felt::from(0b1111_0000_0000_0000u64)) }, + unsafe { AccountId::new_unchecked(Felt::from(0b1111_1111_0000_0000u64)) }, + unsafe { AccountId::new_unchecked(Felt::from(0b1111_1111_1111_0000u64)) }, + unsafe { AccountId::new_unchecked(Felt::from(0b1111_1111_1111_1111u64)) }, + ]; + + let account_initial_states = vec![ + [Felt::from(1u64), Felt::from(1u64), Felt::from(1u64), Felt::from(1u64)], + [Felt::from(2u64), Felt::from(2u64), Felt::from(2u64), Felt::from(2u64)], + [Felt::from(3u64), Felt::from(3u64), Felt::from(3u64), Felt::from(3u64)], + [Felt::from(4u64), Felt::from(4u64), Felt::from(4u64), Felt::from(4u64)], + [Felt::from(5u64), Felt::from(5u64), Felt::from(5u64), Felt::from(5u64)], + ]; + + let account_final_states = vec![ + [Felt::from(2u64), Felt::from(2u64), Felt::from(2u64), Felt::from(2u64)], + [Felt::from(3u64), Felt::from(3u64), Felt::from(3u64), Felt::from(3u64)], + [Felt::from(4u64), Felt::from(4u64), Felt::from(4u64), Felt::from(4u64)], + [Felt::from(5u64), Felt::from(5u64), Felt::from(5u64), Felt::from(5u64)], + [Felt::from(1u64), Felt::from(1u64), Felt::from(1u64), Felt::from(1u64)], + ]; + + // Store SMT + // --------------------------------------------------------------------------------------------- + + // store SMT is initialized with all the accounts and their initial state + let mut store_smt = SimpleSmt::with_leaves( + 64, + account_ids + .iter() + .zip(account_initial_states.iter()) + .map(|(&account_id, &account_hash)| (account_id.into(), account_hash)), + ) + .unwrap(); + + // Block prover + // --------------------------------------------------------------------------------------------- + + // Block inputs is initialized with all the accounts and their initial state + let block_inputs_from_store: BlockInputs = { + let block_header = mock_block_header(Felt::ZERO, None, None, &[]); + let chain_peaks = MmrPeaks::new(0, Vec::new()).unwrap(); + + let account_states = account_ids + .iter() + .zip(account_initial_states.iter()) + .map(|(&account_id, account_hash)| AccountInputRecord { + account_id, + account_hash: Digest::from(account_hash), + proof: store_smt.get_leaf_path(account_id.into()).unwrap(), + }) + .collect(); + + BlockInputs { + block_header, + chain_peaks, + account_states, + nullifiers: Vec::new(), + } + }; + + let batches: Vec = { + let txs: Vec<_> = account_ids + .iter() + .enumerate() + .map(|(idx, &account_id)| { + Arc::new(tx_gen.dummy_proven_tx_with_params( + account_id, + account_initial_states[idx].into(), + account_final_states[idx].into(), + Vec::new(), + )) + }) + .collect(); + + let batch_1 = Arc::new(TransactionBatch::new(txs[..2].to_vec())); + let batch_2 = Arc::new(TransactionBatch::new(txs[2..].to_vec())); + + vec![batch_1, batch_2] + }; + + let block_witness = BlockWitness::new(block_inputs_from_store, batches).unwrap(); + + let block_prover = BlockProver::new(); + let block_header = block_prover.prove(block_witness).unwrap(); + + // Update SMT by hand to get new root + // --------------------------------------------------------------------------------------------- + for (idx, &account_id) in account_ids.iter().enumerate() { + store_smt.update_leaf(account_id.into(), account_final_states[idx]).unwrap(); + } + + // Compare roots + // --------------------------------------------------------------------------------------------- + assert_eq!(block_header.account_root(), store_smt.root()); +} From 8ca84b787d71e3cefbfa884e748f105ef844a62b Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 9 Nov 2023 16:39:48 -0500 Subject: [PATCH 055/166] comments --- block-producer/src/block_builder/prover/tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index 1a46a3dd9..c03b9c043 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -161,7 +161,7 @@ fn test_block_witness_validation_inconsistent_account_hashes() { fn test_compute_account_root_success() { let tx_gen = DummyProvenTxGenerator::new(); - // ACCOUNT STATES + // Set up account states // --------------------------------------------------------------------------------------------- let account_ids = vec![ unsafe { AccountId::new_unchecked(Felt::from(0b0000_0000_0000_0000u64)) }, @@ -187,7 +187,7 @@ fn test_compute_account_root_success() { [Felt::from(1u64), Felt::from(1u64), Felt::from(1u64), Felt::from(1u64)], ]; - // Store SMT + // Set up store's account SMT // --------------------------------------------------------------------------------------------- // store SMT is initialized with all the accounts and their initial state From bce42f9831e93ce1c73259cd3f7bc0bc201ab19b Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 10 Nov 2023 06:02:09 -0500 Subject: [PATCH 056/166] error: use #[from] --- block-producer/src/block_builder/errors.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/block-producer/src/block_builder/errors.rs b/block-producer/src/block_builder/errors.rs index 648bdca49..13b0350b2 100644 --- a/block-producer/src/block_builder/errors.rs +++ b/block-producer/src/block_builder/errors.rs @@ -5,7 +5,7 @@ use thiserror::Error; #[derive(Debug, Error, PartialEq, Eq)] pub enum BuildBlockError { #[error("failed to update account root")] - AccountRootUpdateFailed(BlockProverError), + AccountRootUpdateFailed(#[from] BlockProverError), #[error("transaction batches and store don't modify the same account IDs. Offending accounts: {0:?}")] InconsistentAccountIds(Vec), #[error("transaction batches and store contain different hashes for some accounts. Offending accounts: {0:?}")] @@ -14,12 +14,6 @@ pub enum BuildBlockError { Dummy, } -impl From for BuildBlockError { - fn from(err: BlockProverError) -> Self { - Self::AccountRootUpdateFailed(err) - } -} - #[derive(Error, Debug, PartialEq, Eq)] pub enum BlockProverError { #[error("Received invalid merkle path")] From f87f4d72b1e5a7cc5c0b66ab39c996a6c18e61a9 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 10 Nov 2023 06:06:28 -0500 Subject: [PATCH 057/166] GetBlockInputs error --- block-producer/src/block_builder/errors.rs | 4 ++++ block-producer/src/block_builder/mod.rs | 3 +-- block-producer/src/store/mod.rs | 4 +++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/block-producer/src/block_builder/errors.rs b/block-producer/src/block_builder/errors.rs index 13b0350b2..2e1f91501 100644 --- a/block-producer/src/block_builder/errors.rs +++ b/block-producer/src/block_builder/errors.rs @@ -2,10 +2,14 @@ use miden_objects::accounts::AccountId; use miden_vm::{crypto::MerkleError, ExecutionError}; use thiserror::Error; +use crate::store::BlockInputsError; + #[derive(Debug, Error, PartialEq, Eq)] pub enum BuildBlockError { #[error("failed to update account root")] AccountRootUpdateFailed(#[from] BlockProverError), + #[error("failed to get block inputs from store: {0}")] + GetBlockInputsFailed(#[from] BlockInputsError), #[error("transaction batches and store don't modify the same account IDs. Offending accounts: {0:?}")] InconsistentAccountIds(Vec), #[error("transaction batches and store contain different hashes for some accounts. Offending accounts: {0:?}")] diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index acadf19fe..d08b5ba0d 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -72,8 +72,7 @@ where account_updates.iter().map(|(account_id, _)| account_id), produced_nullifiers.iter(), ) - .await - .unwrap(); + .await?; let block_header_witness = BlockWitness::new(block_inputs, batches)?; diff --git a/block-producer/src/store/mod.rs b/block-producer/src/store/mod.rs index 703d3755e..c36371a7e 100644 --- a/block-producer/src/store/mod.rs +++ b/block-producer/src/store/mod.rs @@ -3,6 +3,7 @@ use std::{collections::BTreeMap, sync::Arc}; use async_trait::async_trait; use miden_node_proto::domain::BlockInputs; use miden_objects::{accounts::AccountId, Digest}; +use thiserror::Error; use crate::{block::Block, SharedProvenTx}; @@ -11,8 +12,9 @@ pub enum TxInputsError { Dummy, } -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Eq, Error)] pub enum BlockInputsError { + #[error("dummy")] Dummy, } From 8c2888a81ce29c306db04268e34f885e2679dfc2 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 10 Nov 2023 06:10:07 -0500 Subject: [PATCH 058/166] ApplyBlockError --- block-producer/src/block_builder/errors.rs | 6 ++++-- block-producer/src/block_builder/mod.rs | 3 +-- block-producer/src/store/mod.rs | 3 ++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/block-producer/src/block_builder/errors.rs b/block-producer/src/block_builder/errors.rs index 2e1f91501..37ef2b11d 100644 --- a/block-producer/src/block_builder/errors.rs +++ b/block-producer/src/block_builder/errors.rs @@ -2,12 +2,14 @@ use miden_objects::accounts::AccountId; use miden_vm::{crypto::MerkleError, ExecutionError}; use thiserror::Error; -use crate::store::BlockInputsError; +use crate::store::{BlockInputsError, ApplyBlockError}; #[derive(Debug, Error, PartialEq, Eq)] pub enum BuildBlockError { - #[error("failed to update account root")] + #[error("failed to update account root: {0}")] AccountRootUpdateFailed(#[from] BlockProverError), + #[error("failed to apply block: {0}")] + ApplyBlockFailed(#[from] ApplyBlockError), #[error("failed to get block inputs from store: {0}")] GetBlockInputsFailed(#[from] BlockInputsError), #[error("transaction batches and store don't modify the same account IDs. Offending accounts: {0:?}")] diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index d08b5ba0d..1c47876cd 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -85,8 +85,7 @@ where produced_nullifiers, }); - // TODO: properly handle - self.store.apply_block(block.clone()).await.expect("apply block failed"); + self.store.apply_block(block.clone()).await?; Ok(()) } diff --git a/block-producer/src/store/mod.rs b/block-producer/src/store/mod.rs index c36371a7e..1472a87f1 100644 --- a/block-producer/src/store/mod.rs +++ b/block-producer/src/store/mod.rs @@ -18,8 +18,9 @@ pub enum BlockInputsError { Dummy, } -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Eq, Error)] pub enum ApplyBlockError { + #[error("dummy")] Dummy, } From 2996277a4ce0212ff3889f400b87247a71a70040 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 10 Nov 2023 06:21:03 -0500 Subject: [PATCH 059/166] test_utils: proven_tx module --- block-producer/src/block_builder/errors.rs | 2 +- block-producer/src/test_utils/mod.rs | 213 +-------------------- block-producer/src/test_utils/proven_tx.rs | 211 ++++++++++++++++++++ 3 files changed, 214 insertions(+), 212 deletions(-) create mode 100644 block-producer/src/test_utils/proven_tx.rs diff --git a/block-producer/src/block_builder/errors.rs b/block-producer/src/block_builder/errors.rs index 37ef2b11d..0ceeb7f27 100644 --- a/block-producer/src/block_builder/errors.rs +++ b/block-producer/src/block_builder/errors.rs @@ -2,7 +2,7 @@ use miden_objects::accounts::AccountId; use miden_vm::{crypto::MerkleError, ExecutionError}; use thiserror::Error; -use crate::store::{BlockInputsError, ApplyBlockError}; +use crate::store::{ApplyBlockError, BlockInputsError}; #[derive(Debug, Error, PartialEq, Eq)] pub enum BuildBlockError { diff --git a/block-producer/src/test_utils/mod.rs b/block-producer/src/test_utils/mod.rs index 466e5e4de..26b85a2c8 100644 --- a/block-producer/src/test_utils/mod.rs +++ b/block-producer/src/test_utils/mod.rs @@ -1,211 +1,2 @@ -use miden_air::{ExecutionProof, HashFunction}; -use miden_mock::constants::ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_ON_CHAIN; -use miden_objects::{ - accounts::AccountId, - transaction::{ConsumedNoteInfo, ProvenTransaction}, - Digest, -}; -///! FibSmall taken from the `fib_small` example in `winterfell` -use winterfell::{ - crypto::{hashers::Blake3_192, DefaultRandomCoin}, - math::fields::f64::BaseElement, - math::FieldElement, - matrix::ColMatrix, - Air, AirContext, Assertion, AuxTraceRandElements, ConstraintCompositionCoefficients, - DefaultConstraintEvaluator, DefaultTraceLde, EvaluationFrame, FieldExtension, ProofOptions, - Prover, StarkDomain, StarkProof, Trace, TraceInfo, TracePolyTable, TraceTable, - TransitionConstraintDegree, -}; - -/// We need to generate a new `ProvenTransaction` every time because it doesn't -/// derive `Clone`. Doing it this way allows us to compute the `StarkProof` -/// once, and clone it for each new `ProvenTransaction`. -#[derive(Clone)] -pub struct DummyProvenTxGenerator { - stark_proof: StarkProof, -} - -impl DummyProvenTxGenerator { - pub fn new() -> Self { - let prover = DummyProver::new(); - let stark_proof = prover.prove(prover.build_trace(16)).unwrap(); - Self { stark_proof } - } - - pub fn dummy_proven_tx(&self) -> ProvenTransaction { - ProvenTransaction::new( - AccountId::try_from(ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_ON_CHAIN).unwrap(), - Digest::default(), - Digest::default(), - Vec::new(), - Vec::new(), - None, - Digest::default(), - ExecutionProof::new(self.stark_proof.clone(), HashFunction::Blake3_192), - ) - } - - pub fn dummy_proven_tx_with_params( - &self, - account_id: AccountId, - initial_account_hash: Digest, - final_account_hash: Digest, - consumed_notes: Vec, - ) -> ProvenTransaction { - ProvenTransaction::new( - account_id, - initial_account_hash, - final_account_hash, - consumed_notes, - Vec::new(), - None, - Digest::default(), - ExecutionProof::new(self.stark_proof.clone(), HashFunction::Blake3_192), - ) - } -} - -const TRACE_WIDTH: usize = 2; - -pub fn are_equal( - a: E, - b: E, -) -> E { - a - b -} - -pub struct FibSmall { - context: AirContext, - result: BaseElement, -} - -impl Air for FibSmall { - type BaseField = BaseElement; - type PublicInputs = BaseElement; - - // CONSTRUCTOR - // -------------------------------------------------------------------------------------------- - fn new( - trace_info: TraceInfo, - pub_inputs: Self::BaseField, - options: ProofOptions, - ) -> Self { - let degrees = vec![TransitionConstraintDegree::new(1), TransitionConstraintDegree::new(1)]; - assert_eq!(TRACE_WIDTH, trace_info.width()); - FibSmall { - context: AirContext::new(trace_info, degrees, 3, options), - result: pub_inputs, - } - } - - fn context(&self) -> &AirContext { - &self.context - } - - fn evaluate_transition>( - &self, - frame: &EvaluationFrame, - _periodic_values: &[E], - result: &mut [E], - ) { - let current = frame.current(); - let next = frame.next(); - // expected state width is 2 field elements - debug_assert_eq!(TRACE_WIDTH, current.len()); - debug_assert_eq!(TRACE_WIDTH, next.len()); - - // constraints of Fibonacci sequence (2 terms per step): - // s_{0, i+1} = s_{0, i} + s_{1, i} - // s_{1, i+1} = s_{1, i} + s_{0, i+1} - result[0] = are_equal(next[0], current[0] + current[1]); - result[1] = are_equal(next[1], current[1] + next[0]); - } - - fn get_assertions(&self) -> Vec> { - // a valid Fibonacci sequence should start with two ones and terminate with - // the expected result - let last_step = self.trace_length() - 1; - vec![ - Assertion::single(0, 0, Self::BaseField::ONE), - Assertion::single(1, 0, Self::BaseField::ONE), - Assertion::single(1, last_step, self.result), - ] - } -} - -pub struct DummyProver { - options: ProofOptions, -} - -impl DummyProver { - pub fn new() -> Self { - Self { - options: ProofOptions::new(1, 2, 1, FieldExtension::None, 2, 127), - } - } - - /// Builds an execution trace for computing a Fibonacci sequence of the specified length such - /// that each row advances the sequence by 2 terms. - pub fn build_trace( - &self, - sequence_length: usize, - ) -> TraceTable { - assert!(sequence_length.is_power_of_two(), "sequence length must be a power of 2"); - - let mut trace = TraceTable::new(TRACE_WIDTH, sequence_length / 2); - trace.fill( - |state| { - state[0] = BaseElement::ONE; - state[1] = BaseElement::ONE; - }, - |_, state| { - state[0] += state[1]; - state[1] += state[0]; - }, - ); - - trace - } -} - -impl Prover for DummyProver { - type BaseField = BaseElement; - type Air = FibSmall; - type Trace = TraceTable; - type HashFn = Blake3_192; - type RandomCoin = DefaultRandomCoin; - type TraceLde> = - DefaultTraceLde>; - type ConstraintEvaluator<'a, E: FieldElement> = - DefaultConstraintEvaluator<'a, FibSmall, E>; - - fn get_pub_inputs( - &self, - trace: &Self::Trace, - ) -> BaseElement { - let last_step = trace.length() - 1; - trace.get(1, last_step) - } - - fn options(&self) -> &ProofOptions { - &self.options - } - - fn new_trace_lde>( - &self, - trace_info: &TraceInfo, - main_trace: &ColMatrix, - domain: &StarkDomain, - ) -> (Self::TraceLde, TracePolyTable) { - DefaultTraceLde::new(trace_info, main_trace, domain) - } - - fn new_evaluator<'a, E: FieldElement>( - &self, - air: &'a FibSmall, - aux_rand_elements: AuxTraceRandElements, - composition_coefficients: ConstraintCompositionCoefficients, - ) -> Self::ConstraintEvaluator<'a, E> { - DefaultConstraintEvaluator::new(air, aux_rand_elements, composition_coefficients) - } -} +mod proven_tx; +pub use proven_tx::DummyProvenTxGenerator; diff --git a/block-producer/src/test_utils/proven_tx.rs b/block-producer/src/test_utils/proven_tx.rs new file mode 100644 index 000000000..466e5e4de --- /dev/null +++ b/block-producer/src/test_utils/proven_tx.rs @@ -0,0 +1,211 @@ +use miden_air::{ExecutionProof, HashFunction}; +use miden_mock::constants::ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_ON_CHAIN; +use miden_objects::{ + accounts::AccountId, + transaction::{ConsumedNoteInfo, ProvenTransaction}, + Digest, +}; +///! FibSmall taken from the `fib_small` example in `winterfell` +use winterfell::{ + crypto::{hashers::Blake3_192, DefaultRandomCoin}, + math::fields::f64::BaseElement, + math::FieldElement, + matrix::ColMatrix, + Air, AirContext, Assertion, AuxTraceRandElements, ConstraintCompositionCoefficients, + DefaultConstraintEvaluator, DefaultTraceLde, EvaluationFrame, FieldExtension, ProofOptions, + Prover, StarkDomain, StarkProof, Trace, TraceInfo, TracePolyTable, TraceTable, + TransitionConstraintDegree, +}; + +/// We need to generate a new `ProvenTransaction` every time because it doesn't +/// derive `Clone`. Doing it this way allows us to compute the `StarkProof` +/// once, and clone it for each new `ProvenTransaction`. +#[derive(Clone)] +pub struct DummyProvenTxGenerator { + stark_proof: StarkProof, +} + +impl DummyProvenTxGenerator { + pub fn new() -> Self { + let prover = DummyProver::new(); + let stark_proof = prover.prove(prover.build_trace(16)).unwrap(); + Self { stark_proof } + } + + pub fn dummy_proven_tx(&self) -> ProvenTransaction { + ProvenTransaction::new( + AccountId::try_from(ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_ON_CHAIN).unwrap(), + Digest::default(), + Digest::default(), + Vec::new(), + Vec::new(), + None, + Digest::default(), + ExecutionProof::new(self.stark_proof.clone(), HashFunction::Blake3_192), + ) + } + + pub fn dummy_proven_tx_with_params( + &self, + account_id: AccountId, + initial_account_hash: Digest, + final_account_hash: Digest, + consumed_notes: Vec, + ) -> ProvenTransaction { + ProvenTransaction::new( + account_id, + initial_account_hash, + final_account_hash, + consumed_notes, + Vec::new(), + None, + Digest::default(), + ExecutionProof::new(self.stark_proof.clone(), HashFunction::Blake3_192), + ) + } +} + +const TRACE_WIDTH: usize = 2; + +pub fn are_equal( + a: E, + b: E, +) -> E { + a - b +} + +pub struct FibSmall { + context: AirContext, + result: BaseElement, +} + +impl Air for FibSmall { + type BaseField = BaseElement; + type PublicInputs = BaseElement; + + // CONSTRUCTOR + // -------------------------------------------------------------------------------------------- + fn new( + trace_info: TraceInfo, + pub_inputs: Self::BaseField, + options: ProofOptions, + ) -> Self { + let degrees = vec![TransitionConstraintDegree::new(1), TransitionConstraintDegree::new(1)]; + assert_eq!(TRACE_WIDTH, trace_info.width()); + FibSmall { + context: AirContext::new(trace_info, degrees, 3, options), + result: pub_inputs, + } + } + + fn context(&self) -> &AirContext { + &self.context + } + + fn evaluate_transition>( + &self, + frame: &EvaluationFrame, + _periodic_values: &[E], + result: &mut [E], + ) { + let current = frame.current(); + let next = frame.next(); + // expected state width is 2 field elements + debug_assert_eq!(TRACE_WIDTH, current.len()); + debug_assert_eq!(TRACE_WIDTH, next.len()); + + // constraints of Fibonacci sequence (2 terms per step): + // s_{0, i+1} = s_{0, i} + s_{1, i} + // s_{1, i+1} = s_{1, i} + s_{0, i+1} + result[0] = are_equal(next[0], current[0] + current[1]); + result[1] = are_equal(next[1], current[1] + next[0]); + } + + fn get_assertions(&self) -> Vec> { + // a valid Fibonacci sequence should start with two ones and terminate with + // the expected result + let last_step = self.trace_length() - 1; + vec![ + Assertion::single(0, 0, Self::BaseField::ONE), + Assertion::single(1, 0, Self::BaseField::ONE), + Assertion::single(1, last_step, self.result), + ] + } +} + +pub struct DummyProver { + options: ProofOptions, +} + +impl DummyProver { + pub fn new() -> Self { + Self { + options: ProofOptions::new(1, 2, 1, FieldExtension::None, 2, 127), + } + } + + /// Builds an execution trace for computing a Fibonacci sequence of the specified length such + /// that each row advances the sequence by 2 terms. + pub fn build_trace( + &self, + sequence_length: usize, + ) -> TraceTable { + assert!(sequence_length.is_power_of_two(), "sequence length must be a power of 2"); + + let mut trace = TraceTable::new(TRACE_WIDTH, sequence_length / 2); + trace.fill( + |state| { + state[0] = BaseElement::ONE; + state[1] = BaseElement::ONE; + }, + |_, state| { + state[0] += state[1]; + state[1] += state[0]; + }, + ); + + trace + } +} + +impl Prover for DummyProver { + type BaseField = BaseElement; + type Air = FibSmall; + type Trace = TraceTable; + type HashFn = Blake3_192; + type RandomCoin = DefaultRandomCoin; + type TraceLde> = + DefaultTraceLde>; + type ConstraintEvaluator<'a, E: FieldElement> = + DefaultConstraintEvaluator<'a, FibSmall, E>; + + fn get_pub_inputs( + &self, + trace: &Self::Trace, + ) -> BaseElement { + let last_step = trace.length() - 1; + trace.get(1, last_step) + } + + fn options(&self) -> &ProofOptions { + &self.options + } + + fn new_trace_lde>( + &self, + trace_info: &TraceInfo, + main_trace: &ColMatrix, + domain: &StarkDomain, + ) -> (Self::TraceLde, TracePolyTable) { + DefaultTraceLde::new(trace_info, main_trace, domain) + } + + fn new_evaluator<'a, E: FieldElement>( + &self, + air: &'a FibSmall, + aux_rand_elements: AuxTraceRandElements, + composition_coefficients: ConstraintCompositionCoefficients, + ) -> Self::ConstraintEvaluator<'a, E> { + DefaultConstraintEvaluator::new(air, aux_rand_elements, composition_coefficients) + } +} From 3392c7e41ef4db645c8f460e4a7a9d7b09443e88 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 10 Nov 2023 06:31:34 -0500 Subject: [PATCH 060/166] move test structs to test_utils --- .../src/state_view/tests/apply_block.rs | 2 + block-producer/src/state_view/tests/mod.rs | 180 +----------------- .../src/state_view/tests/verify_tx.rs | 2 + block-producer/src/test_utils/account.rs | 48 +++++ block-producer/src/test_utils/mod.rs | 12 ++ block-producer/src/test_utils/store.rs | 127 ++++++++++++ 6 files changed, 193 insertions(+), 178 deletions(-) create mode 100644 block-producer/src/test_utils/account.rs create mode 100644 block-producer/src/test_utils/store.rs diff --git a/block-producer/src/state_view/tests/apply_block.rs b/block-producer/src/state_view/tests/apply_block.rs index 7ff969e0c..31d082570 100644 --- a/block-producer/src/state_view/tests/apply_block.rs +++ b/block-producer/src/state_view/tests/apply_block.rs @@ -6,6 +6,8 @@ use std::iter; +use crate::test_utils::MockStoreSuccess; + use super::*; /// Tests requirement AB1 diff --git a/block-producer/src/state_view/tests/mod.rs b/block-producer/src/state_view/tests/mod.rs index a2fc96d07..466ba7d9c 100644 --- a/block-producer/src/state_view/tests/mod.rs +++ b/block-producer/src/state_view/tests/mod.rs @@ -1,188 +1,12 @@ use super::*; -use std::collections::BTreeMap; +use miden_objects::{transaction::ConsumedNoteInfo, BlockHeader, Felt, Hasher}; -use miden_node_proto::domain::BlockInputs; -use miden_objects::{ - accounts::get_account_seed, transaction::ConsumedNoteInfo, BlockHeader, Felt, Hasher, -}; - -use crate::{ - store::{BlockInputsError, TxInputsError}, - test_utils::DummyProvenTxGenerator, -}; +use crate::test_utils::{DummyProvenTxGenerator, MockPrivateAccount}; mod apply_block; mod verify_tx; -// MOCK STORES -// ------------------------------------------------------------------------------------------------- - -#[derive(Default)] -struct MockStoreSuccess { - /// Map account id -> account hash - accounts: Arc>>, - - /// Stores the nullifiers of the notes that were consumed - consumed_nullifiers: Arc>>, - - /// The number of times `apply_block()` was called - num_apply_block_called: Arc>, -} - -impl MockStoreSuccess { - /// Initializes the known accounts from provided mock accounts, where the account hash in the - /// store is the first state in `MockAccount.states`. - fn new( - accounts: impl Iterator, - consumed_nullifiers: BTreeSet, - ) -> Self { - let store_accounts: BTreeMap = - accounts.map(|account| (account.id, account.states[0])).collect(); - - Self { - accounts: Arc::new(RwLock::new(store_accounts)), - consumed_nullifiers: Arc::new(RwLock::new(consumed_nullifiers)), - num_apply_block_called: Arc::new(RwLock::new(0)), - } - } -} - -#[async_trait] -impl ApplyBlock for MockStoreSuccess { - async fn apply_block( - &self, - block: Arc, - ) -> Result<(), ApplyBlockError> { - // Intentionally, we take and hold both locks, to prevent calls to `get_tx_inputs()` from going through while we're updating the store's data structure - let mut locked_accounts = self.accounts.write().await; - let mut locked_consumed_nullifiers = self.consumed_nullifiers.write().await; - - for &(account_id, account_hash) in block.updated_accounts.iter() { - locked_accounts.insert(account_id, account_hash); - } - - let mut new_nullifiers: BTreeSet = - block.produced_nullifiers.iter().cloned().collect(); - locked_consumed_nullifiers.append(&mut new_nullifiers); - - *self.num_apply_block_called.write().await += 1; - - Ok(()) - } -} - -#[async_trait] -impl Store for MockStoreSuccess { - async fn get_tx_inputs( - &self, - proven_tx: SharedProvenTx, - ) -> Result { - let locked_accounts = self.accounts.read().await; - let locked_consumed_nullifiers = self.consumed_nullifiers.read().await; - - let account_hash = locked_accounts.get(&proven_tx.account_id()).cloned(); - - let nullifiers = proven_tx - .consumed_notes() - .iter() - .map(|note| (note.nullifier(), locked_consumed_nullifiers.contains(¬e.nullifier()))) - .collect(); - - Ok(TxInputs { - account_hash, - nullifiers, - }) - } - - async fn get_block_inputs( - &self, - _updated_accounts: impl Iterator + Send, - _produced_nullifiers: impl Iterator + Send, - ) -> Result { - unimplemented!() - } -} - -#[derive(Default)] -struct MockStoreFailure; - -#[async_trait] -impl ApplyBlock for MockStoreFailure { - async fn apply_block( - &self, - _block: Arc, - ) -> Result<(), ApplyBlockError> { - Err(ApplyBlockError::Dummy) - } -} - -#[async_trait] -impl Store for MockStoreFailure { - async fn get_tx_inputs( - &self, - _proven_tx: SharedProvenTx, - ) -> Result { - Err(TxInputsError::Dummy) - } - - async fn get_block_inputs( - &self, - _updated_accounts: impl Iterator + Send, - _produced_nullifiers: impl Iterator + Send, - ) -> Result { - unimplemented!() - } -} - -// MOCK PRIVATE ACCOUNT -// ------------------------------------------------------------------------------------------------- - -/// A mock representation fo private accounts. An account starts in state `states[0]`, is modified -/// to state `states[1]`, and so on. -#[derive(Clone, Copy, Debug)] -pub struct MockPrivateAccount { - pub id: AccountId, - - // Sequence states that the account goes into. - pub states: [Digest; NUM_STATES], -} - -impl MockPrivateAccount { - fn new(init_seed: [u8; 32]) -> Self { - let account_seed = get_account_seed( - init_seed, - miden_objects::accounts::AccountType::RegularAccountUpdatableCode, - false, - Digest::default(), - Digest::default(), - ) - .unwrap(); - - let mut states = [Digest::default(); NUM_STATES]; - - states[0] = Hasher::hash(&init_seed); - for idx in 1..NUM_STATES { - states[idx] = Hasher::hash(&states[idx - 1].as_bytes()); - } - - Self { - id: AccountId::new(account_seed, Digest::default(), Digest::default()).unwrap(), - states, - } - } -} - -impl From for MockPrivateAccount { - /// Each index gives rise to a different account ID - fn from(index: u8) -> Self { - let mut init_seed: [u8; 32] = [0; 32]; - init_seed[0] = index; - - Self::new(init_seed) - } -} - // HELPERS // ------------------------------------------------------------------------------------------------- diff --git a/block-producer/src/state_view/tests/verify_tx.rs b/block-producer/src/state_view/tests/verify_tx.rs index 2c080c14a..1a290ea92 100644 --- a/block-producer/src/state_view/tests/verify_tx.rs +++ b/block-producer/src/state_view/tests/verify_tx.rs @@ -14,6 +14,8 @@ use std::iter; use tokio::task::JoinSet; +use crate::test_utils::MockStoreSuccess; + use super::*; /// Tests the happy path where 3 transactions who modify different accounts and consume different diff --git a/block-producer/src/test_utils/account.rs b/block-producer/src/test_utils/account.rs new file mode 100644 index 000000000..a54430683 --- /dev/null +++ b/block-producer/src/test_utils/account.rs @@ -0,0 +1,48 @@ +use miden_objects::{accounts::get_account_seed, Hasher}; + +use super::*; + +/// A mock representation fo private accounts. An account starts in state `states[0]`, is modified +/// to state `states[1]`, and so on. +#[derive(Clone, Copy, Debug)] +pub struct MockPrivateAccount { + pub id: AccountId, + + // Sequence states that the account goes into. + pub states: [Digest; NUM_STATES], +} + +impl MockPrivateAccount { + fn new(init_seed: [u8; 32]) -> Self { + let account_seed = get_account_seed( + init_seed, + miden_objects::accounts::AccountType::RegularAccountUpdatableCode, + false, + Digest::default(), + Digest::default(), + ) + .unwrap(); + + let mut states = [Digest::default(); NUM_STATES]; + + states[0] = Hasher::hash(&init_seed); + for idx in 1..NUM_STATES { + states[idx] = Hasher::hash(&states[idx - 1].as_bytes()); + } + + Self { + id: AccountId::new(account_seed, Digest::default(), Digest::default()).unwrap(), + states, + } + } +} + +impl From for MockPrivateAccount { + /// Each index gives rise to a different account ID + fn from(index: u8) -> Self { + let mut init_seed: [u8; 32] = [0; 32]; + init_seed[0] = index; + + Self::new(init_seed) + } +} diff --git a/block-producer/src/test_utils/mod.rs b/block-producer/src/test_utils/mod.rs index 26b85a2c8..f23f1fc3a 100644 --- a/block-producer/src/test_utils/mod.rs +++ b/block-producer/src/test_utils/mod.rs @@ -1,2 +1,14 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; +use tokio::sync::RwLock; + +use miden_objects::{accounts::AccountId, Digest}; + mod proven_tx; pub use proven_tx::DummyProvenTxGenerator; + +mod store; +pub use store::{MockStoreFailure, MockStoreSuccess}; + +mod account; +pub use account::MockPrivateAccount; diff --git a/block-producer/src/test_utils/store.rs b/block-producer/src/test_utils/store.rs new file mode 100644 index 000000000..a9d9a2c5a --- /dev/null +++ b/block-producer/src/test_utils/store.rs @@ -0,0 +1,127 @@ +use async_trait::async_trait; +use miden_node_proto::domain::BlockInputs; + +use crate::{ + block::Block, + store::{ApplyBlock, ApplyBlockError, BlockInputsError, Store, TxInputs, TxInputsError}, + SharedProvenTx, +}; + +use super::*; + +#[derive(Default)] +pub struct MockStoreSuccess { + /// Map account id -> account hash + accounts: Arc>>, + + /// Stores the nullifiers of the notes that were consumed + consumed_nullifiers: Arc>>, + + /// The number of times `apply_block()` was called + pub num_apply_block_called: Arc>, +} + +impl MockStoreSuccess { + /// Initializes the known accounts from provided mock accounts, where the account hash in the + /// store is the first state in `MockAccount.states`. + pub fn new( + accounts: impl Iterator, + consumed_nullifiers: BTreeSet, + ) -> Self { + let store_accounts: BTreeMap = + accounts.map(|account| (account.id, account.states[0])).collect(); + + Self { + accounts: Arc::new(RwLock::new(store_accounts)), + consumed_nullifiers: Arc::new(RwLock::new(consumed_nullifiers)), + num_apply_block_called: Arc::new(RwLock::new(0)), + } + } +} + +#[async_trait] +impl ApplyBlock for MockStoreSuccess { + async fn apply_block( + &self, + block: Arc, + ) -> Result<(), ApplyBlockError> { + // Intentionally, we take and hold both locks, to prevent calls to `get_tx_inputs()` from going through while we're updating the store's data structure + let mut locked_accounts = self.accounts.write().await; + let mut locked_consumed_nullifiers = self.consumed_nullifiers.write().await; + + for &(account_id, account_hash) in block.updated_accounts.iter() { + locked_accounts.insert(account_id, account_hash); + } + + let mut new_nullifiers: BTreeSet = + block.produced_nullifiers.iter().cloned().collect(); + locked_consumed_nullifiers.append(&mut new_nullifiers); + + *self.num_apply_block_called.write().await += 1; + + Ok(()) + } +} + +#[async_trait] +impl Store for MockStoreSuccess { + async fn get_tx_inputs( + &self, + proven_tx: SharedProvenTx, + ) -> Result { + let locked_accounts = self.accounts.read().await; + let locked_consumed_nullifiers = self.consumed_nullifiers.read().await; + + let account_hash = locked_accounts.get(&proven_tx.account_id()).cloned(); + + let nullifiers = proven_tx + .consumed_notes() + .iter() + .map(|note| (note.nullifier(), locked_consumed_nullifiers.contains(¬e.nullifier()))) + .collect(); + + Ok(TxInputs { + account_hash, + nullifiers, + }) + } + + async fn get_block_inputs( + &self, + _updated_accounts: impl Iterator + Send, + _produced_nullifiers: impl Iterator + Send, + ) -> Result { + unimplemented!() + } +} + +#[derive(Default)] +pub struct MockStoreFailure; + +#[async_trait] +impl ApplyBlock for MockStoreFailure { + async fn apply_block( + &self, + _block: Arc, + ) -> Result<(), ApplyBlockError> { + Err(ApplyBlockError::Dummy) + } +} + +#[async_trait] +impl Store for MockStoreFailure { + async fn get_tx_inputs( + &self, + _proven_tx: SharedProvenTx, + ) -> Result { + Err(TxInputsError::Dummy) + } + + async fn get_block_inputs( + &self, + _updated_accounts: impl Iterator + Send, + _produced_nullifiers: impl Iterator + Send, + ) -> Result { + unimplemented!() + } +} From f222ca16af3e55698b04249f10bcf6760db118e2 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 10 Nov 2023 06:54:33 -0500 Subject: [PATCH 061/166] change MockStoreSuccess::new --- .../src/state_view/tests/apply_block.rs | 21 +++++++++-- .../src/state_view/tests/verify_tx.rs | 36 ++++++++++++++----- block-producer/src/test_utils/store.rs | 7 ++-- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/block-producer/src/state_view/tests/apply_block.rs b/block-producer/src/state_view/tests/apply_block.rs index 31d082570..3c7ffd082 100644 --- a/block-producer/src/state_view/tests/apply_block.rs +++ b/block-producer/src/state_view/tests/apply_block.rs @@ -16,7 +16,10 @@ async fn test_apply_block_ab1() { let tx_gen = DummyProvenTxGenerator::new(); let account: MockPrivateAccount<3> = MockPrivateAccount::from(0); - let store = Arc::new(MockStoreSuccess::new(iter::once(account), BTreeSet::new())); + let store = Arc::new(MockStoreSuccess::new( + iter::once((account.id, account.states[0])), + BTreeSet::new(), + )); let tx = tx_gen.dummy_proven_tx_with_params( account.id, @@ -46,7 +49,13 @@ async fn test_apply_block_ab2() { let (txs, accounts): (Vec<_>, Vec<_>) = get_txs_and_accounts(&tx_gen, 3).unzip(); - let store = Arc::new(MockStoreSuccess::new(accounts.clone().into_iter(), BTreeSet::new())); + let store = Arc::new(MockStoreSuccess::new( + accounts + .clone() + .into_iter() + .map(|mock_account| (mock_account.id, mock_account.states[0])), + BTreeSet::new(), + )); let state_view = DefaulStateView::new(store.clone()); @@ -78,7 +87,13 @@ async fn test_apply_block_ab3() { let (txs, accounts): (Vec<_>, Vec<_>) = get_txs_and_accounts(&tx_gen, 3).unzip(); - let store = Arc::new(MockStoreSuccess::new(accounts.clone().into_iter(), BTreeSet::new())); + let store = Arc::new(MockStoreSuccess::new( + accounts + .clone() + .into_iter() + .map(|mock_account| (mock_account.id, mock_account.states[0])), + BTreeSet::new(), + )); let state_view = DefaulStateView::new(store.clone()); diff --git a/block-producer/src/state_view/tests/verify_tx.rs b/block-producer/src/state_view/tests/verify_tx.rs index 1a290ea92..e345dd7b7 100644 --- a/block-producer/src/state_view/tests/verify_tx.rs +++ b/block-producer/src/state_view/tests/verify_tx.rs @@ -26,7 +26,12 @@ async fn test_verify_tx_happy_path() { let (txs, accounts): (Vec, Vec) = get_txs_and_accounts(&tx_gen, 3).unzip(); - let store = Arc::new(MockStoreSuccess::new(accounts.into_iter(), BTreeSet::new())); + let store = Arc::new(MockStoreSuccess::new( + accounts + .into_iter() + .map(|mock_account| (mock_account.id, mock_account.states[0])), + BTreeSet::new(), + )); let state_view = DefaulStateView::new(store); @@ -45,7 +50,12 @@ async fn test_verify_tx_happy_path_concurrent() { let (txs, accounts): (Vec, Vec) = get_txs_and_accounts(&tx_gen, 3).unzip(); - let store = Arc::new(MockStoreSuccess::new(accounts.into_iter(), BTreeSet::new())); + let store = Arc::new(MockStoreSuccess::new( + accounts + .into_iter() + .map(|mock_account| (mock_account.id, mock_account.states[0])), + BTreeSet::new(), + )); let state_view = Arc::new(DefaulStateView::new(store)); @@ -66,9 +76,12 @@ async fn test_verify_tx_happy_path_concurrent() { async fn test_verify_tx_vt1() { let tx_gen = DummyProvenTxGenerator::new(); - let account = MockPrivateAccount::from(0); + let account = MockPrivateAccount::<3>::from(0); - let store = Arc::new(MockStoreSuccess::new(iter::once(account), BTreeSet::new())); + let store = Arc::new(MockStoreSuccess::new( + iter::once((account.id, account.states[0])), + BTreeSet::new(), + )); // The transaction's initial account hash uses `account.states[1]`, where the store expects // `account.states[0]` @@ -133,7 +146,7 @@ async fn test_verify_tx_vt3() { // Notice: `consumed_note_in_store` is added to the store let store = Arc::new(MockStoreSuccess::new( - iter::once(account), + iter::once((account.id, account.states[0])), BTreeSet::from_iter(iter::once(consumed_note_in_store.nullifier())), )); @@ -163,7 +176,10 @@ async fn test_verify_tx_vt4() { let account: MockPrivateAccount<3> = MockPrivateAccount::from(0); - let store = Arc::new(MockStoreSuccess::new(iter::once(account), BTreeSet::new())); + let store = Arc::new(MockStoreSuccess::new( + iter::once((account.id, account.states[0])), + BTreeSet::new(), + )); let tx1 = tx_gen.dummy_proven_tx_with_params( account.id, @@ -203,8 +219,12 @@ async fn test_verify_tx_vt5() { let consumed_note_in_both_txs = consumed_note_by_index(0); // Notice: `consumed_note_in_both_txs` is NOT in the store - let store = - Arc::new(MockStoreSuccess::new(vec![account_1, account_2].into_iter(), BTreeSet::new())); + let store = Arc::new(MockStoreSuccess::new( + vec![account_1, account_2] + .into_iter() + .map(|account| (account.id, account.states[0])), + BTreeSet::new(), + )); let tx1 = tx_gen.dummy_proven_tx_with_params( account_1.id, diff --git a/block-producer/src/test_utils/store.rs b/block-producer/src/test_utils/store.rs index a9d9a2c5a..d35035b6b 100644 --- a/block-producer/src/test_utils/store.rs +++ b/block-producer/src/test_utils/store.rs @@ -25,11 +25,10 @@ impl MockStoreSuccess { /// Initializes the known accounts from provided mock accounts, where the account hash in the /// store is the first state in `MockAccount.states`. pub fn new( - accounts: impl Iterator, + accounts: impl Iterator, consumed_nullifiers: BTreeSet, ) -> Self { - let store_accounts: BTreeMap = - accounts.map(|account| (account.id, account.states[0])).collect(); + let store_accounts: BTreeMap = accounts.collect(); Self { accounts: Arc::new(RwLock::new(store_accounts)), @@ -122,6 +121,6 @@ impl Store for MockStoreFailure { _updated_accounts: impl Iterator + Send, _produced_nullifiers: impl Iterator + Send, ) -> Result { - unimplemented!() + Err(BlockInputsError::Dummy) } } From 36cc1963f053712913d6546a4171e61001278de6 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 10 Nov 2023 07:20:33 -0500 Subject: [PATCH 062/166] Use SimpleSmt in MockStoreSuccess --- block-producer/src/test_utils/mod.rs | 2 +- block-producer/src/test_utils/store.rs | 23 ++++++++++++++++++----- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/block-producer/src/test_utils/mod.rs b/block-producer/src/test_utils/mod.rs index f23f1fc3a..9cdb64614 100644 --- a/block-producer/src/test_utils/mod.rs +++ b/block-producer/src/test_utils/mod.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeSet; use std::sync::Arc; use tokio::sync::RwLock; diff --git a/block-producer/src/test_utils/store.rs b/block-producer/src/test_utils/store.rs index d35035b6b..b754b016a 100644 --- a/block-producer/src/test_utils/store.rs +++ b/block-producer/src/test_utils/store.rs @@ -1,5 +1,7 @@ use async_trait::async_trait; use miden_node_proto::domain::BlockInputs; +use miden_objects::EMPTY_WORD; +use miden_vm::crypto::SimpleSmt; use crate::{ block::Block, @@ -9,10 +11,9 @@ use crate::{ use super::*; -#[derive(Default)] pub struct MockStoreSuccess { /// Map account id -> account hash - accounts: Arc>>, + accounts: Arc>, /// Stores the nullifiers of the notes that were consumed consumed_nullifiers: Arc>>, @@ -28,7 +29,11 @@ impl MockStoreSuccess { accounts: impl Iterator, consumed_nullifiers: BTreeSet, ) -> Self { - let store_accounts: BTreeMap = accounts.collect(); + let accounts: Vec<_> = accounts + .into_iter() + .map(|(account_id, hash)| (account_id.into(), hash.into())) + .collect(); + let store_accounts = SimpleSmt::with_leaves(64, accounts).unwrap(); Self { accounts: Arc::new(RwLock::new(store_accounts)), @@ -49,7 +54,7 @@ impl ApplyBlock for MockStoreSuccess { let mut locked_consumed_nullifiers = self.consumed_nullifiers.write().await; for &(account_id, account_hash) in block.updated_accounts.iter() { - locked_accounts.insert(account_id, account_hash); + locked_accounts.update_leaf(account_id.into(), account_hash.into()).unwrap(); } let mut new_nullifiers: BTreeSet = @@ -71,7 +76,15 @@ impl Store for MockStoreSuccess { let locked_accounts = self.accounts.read().await; let locked_consumed_nullifiers = self.consumed_nullifiers.read().await; - let account_hash = locked_accounts.get(&proven_tx.account_id()).cloned(); + let account_hash = { + let account_hash = locked_accounts.get_leaf(proven_tx.account_id().into()).unwrap(); + + if account_hash == EMPTY_WORD { + None + } else { + Some(account_hash.into()) + } + }; let nullifiers = proven_tx .consumed_notes() From 397e070d0200e29a8b22fcfb179b924b545f6ad6 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 10 Nov 2023 07:29:55 -0500 Subject: [PATCH 063/166] implement MockStoreSuccess::get_block_inputs --- block-producer/src/test_utils/store.rs | 36 +++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/block-producer/src/test_utils/store.rs b/block-producer/src/test_utils/store.rs index b754b016a..01413c4a8 100644 --- a/block-producer/src/test_utils/store.rs +++ b/block-producer/src/test_utils/store.rs @@ -1,6 +1,8 @@ use async_trait::async_trait; -use miden_node_proto::domain::BlockInputs; -use miden_objects::EMPTY_WORD; +use miden_air::Felt; +use miden_mock::mock::block::mock_block_header; +use miden_node_proto::domain::{AccountInputRecord, BlockInputs}; +use miden_objects::{crypto::merkle::MmrPeaks, EMPTY_WORD}; use miden_vm::crypto::SimpleSmt; use crate::{ @@ -100,10 +102,36 @@ impl Store for MockStoreSuccess { async fn get_block_inputs( &self, - _updated_accounts: impl Iterator + Send, + updated_accounts: impl Iterator + Send, _produced_nullifiers: impl Iterator + Send, ) -> Result { - unimplemented!() + let block_header = mock_block_header(Felt::from(0u64), None, None, &[]); + let chain_peaks = MmrPeaks::new(0, Vec::new()).unwrap(); + + let account_states = { + let locked_accounts = self.accounts.read().await; + + updated_accounts + .map(|&account_id| { + let account_hash = locked_accounts.get_leaf(account_id.into()).unwrap(); + let proof = locked_accounts.get_leaf_path(account_id.into()).unwrap(); + + AccountInputRecord { + account_id, + account_hash: account_hash.into(), + proof, + } + }) + .collect() + }; + + Ok(BlockInputs { + block_header, + chain_peaks, + account_states, + // TODO: return a proper nullifiers iterator + nullifiers: Vec::new(), + }) } } From 708ebb2e0e5ba08f70d8f1567dd9755b2bba5682 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 10 Nov 2023 07:38:57 -0500 Subject: [PATCH 064/166] MockStoreSuccess: update_accounts --- block-producer/src/test_utils/store.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/block-producer/src/test_utils/store.rs b/block-producer/src/test_utils/store.rs index 01413c4a8..27242e1e3 100644 --- a/block-producer/src/test_utils/store.rs +++ b/block-producer/src/test_utils/store.rs @@ -43,6 +43,19 @@ impl MockStoreSuccess { num_apply_block_called: Arc::new(RwLock::new(0)), } } + + /// Update some accounts in the store + pub async fn update_accounts( + &self, + updated_accounts: impl Iterator, + ) { + let mut locked_accounts = self.accounts.write().await; + for (account_id, new_account_state) in updated_accounts { + locked_accounts + .update_leaf(account_id.into(), new_account_state.into()) + .unwrap(); + } + } } #[async_trait] From ca4fedbbe263f67a87be27ae7aca099acbf8698f Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 10 Nov 2023 07:44:13 -0500 Subject: [PATCH 065/166] Use `MockStoreSuccess` in prover test --- .../src/block_builder/prover/tests.rs | 58 +++++++------------ block-producer/src/test_utils/store.rs | 8 ++- 2 files changed, 29 insertions(+), 37 deletions(-) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index c03b9c043..9e2cd7421 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -4,9 +4,12 @@ use miden_air::FieldElement; use miden_mock::mock::block::mock_block_header; use miden_node_proto::domain::AccountInputRecord; use miden_objects::crypto::merkle::MmrPeaks; -use miden_vm::crypto::SimpleSmt; -use crate::{batch_builder::TransactionBatch, test_utils::DummyProvenTxGenerator}; +use crate::{ + batch_builder::TransactionBatch, + store::Store, + test_utils::{DummyProvenTxGenerator, MockStoreSuccess}, +}; use super::*; @@ -157,8 +160,8 @@ fn test_block_witness_validation_inconsistent_account_hashes() { /// Tests that the `BlockProver` computes the proper account root. /// /// We assume an initial store with 5 accounts, and all will be updated. -#[test] -fn test_compute_account_root_success() { +#[tokio::test] +async fn test_compute_account_root_success() { let tx_gen = DummyProvenTxGenerator::new(); // Set up account states @@ -190,41 +193,19 @@ fn test_compute_account_root_success() { // Set up store's account SMT // --------------------------------------------------------------------------------------------- - // store SMT is initialized with all the accounts and their initial state - let mut store_smt = SimpleSmt::with_leaves( - 64, + let store = MockStoreSuccess::new( account_ids .iter() .zip(account_initial_states.iter()) - .map(|(&account_id, &account_hash)| (account_id.into(), account_hash)), - ) - .unwrap(); - + .map(|(&account_id, &account_hash)| (account_id.into(), account_hash.into())), + BTreeSet::new(), + ); // Block prover // --------------------------------------------------------------------------------------------- // Block inputs is initialized with all the accounts and their initial state - let block_inputs_from_store: BlockInputs = { - let block_header = mock_block_header(Felt::ZERO, None, None, &[]); - let chain_peaks = MmrPeaks::new(0, Vec::new()).unwrap(); - - let account_states = account_ids - .iter() - .zip(account_initial_states.iter()) - .map(|(&account_id, account_hash)| AccountInputRecord { - account_id, - account_hash: Digest::from(account_hash), - proof: store_smt.get_leaf_path(account_id.into()).unwrap(), - }) - .collect(); - - BlockInputs { - block_header, - chain_peaks, - account_states, - nullifiers: Vec::new(), - } - }; + let block_inputs_from_store: BlockInputs = + store.get_block_inputs(account_ids.iter(), std::iter::empty()).await.unwrap(); let batches: Vec = { let txs: Vec<_> = account_ids @@ -253,11 +234,16 @@ fn test_compute_account_root_success() { // Update SMT by hand to get new root // --------------------------------------------------------------------------------------------- - for (idx, &account_id) in account_ids.iter().enumerate() { - store_smt.update_leaf(account_id.into(), account_final_states[idx]).unwrap(); - } + store + .update_accounts( + account_ids + .iter() + .zip(account_final_states.iter()) + .map(|(&account_id, &account_hash)| (account_id.into(), account_hash.into())), + ) + .await; // Compare roots // --------------------------------------------------------------------------------------------- - assert_eq!(block_header.account_root(), store_smt.root()); + assert_eq!(block_header.account_root(), store.account_root().await); } diff --git a/block-producer/src/test_utils/store.rs b/block-producer/src/test_utils/store.rs index 27242e1e3..903d6b3a5 100644 --- a/block-producer/src/test_utils/store.rs +++ b/block-producer/src/test_utils/store.rs @@ -44,7 +44,7 @@ impl MockStoreSuccess { } } - /// Update some accounts in the store + /// Update some accounts in the store pub async fn update_accounts( &self, updated_accounts: impl Iterator, @@ -56,6 +56,12 @@ impl MockStoreSuccess { .unwrap(); } } + + pub async fn account_root(&self) -> Digest { + let locked_accounts = self.accounts.read().await; + + locked_accounts.root() + } } #[async_trait] From 1a1f227ddd81fa83d4811fcef50546a54337a680 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 10 Nov 2023 11:40:30 -0500 Subject: [PATCH 066/166] test_apply_block_called_empty_batches --- block-producer/src/block_builder/tests.rs | 26 +++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/block-producer/src/block_builder/tests.rs b/block-producer/src/block_builder/tests.rs index cf16b751b..550e4bc17 100644 --- a/block-producer/src/block_builder/tests.rs +++ b/block-producer/src/block_builder/tests.rs @@ -1,3 +1,29 @@ +use std::collections::BTreeSet; + // block builder tests (higher level) // 1. `apply_block()` is called // 2. if `apply_block()` fails, you fail too +use super::*; + +use miden_air::Felt; + +use crate::test_utils::MockStoreSuccess; + +/// Tests that `build_block()` succeeds when the transaction batches are empty +#[tokio::test] +async fn test_apply_block_called_empty_batches() { + let account_id = unsafe { AccountId::new_unchecked(42u64.into()) }; + let account_hash: Digest = + [Felt::from(1u64), Felt::from(1u64), Felt::from(1u64), Felt::from(1u64)].into(); + let store = Arc::new(MockStoreSuccess::new( + std::iter::once((account_id, account_hash)), + BTreeSet::new(), + )); + + let block_builder = DefaultBlockBuilder::new(store.clone()); + + block_builder.build_block(Vec::new()).await.unwrap(); + + // Ensure that the store's `apply_block()` was called + assert_eq!(*store.num_apply_block_called.read().await, 1); +} From 094b27205f5c84d461989b870cda7a02c6284265 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 10 Nov 2023 11:54:22 -0500 Subject: [PATCH 067/166] test_compute_account_root_empty_batches --- .../src/block_builder/prover/tests.rs | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index 9e2cd7421..5222e9037 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -247,3 +247,52 @@ async fn test_compute_account_root_success() { // --------------------------------------------------------------------------------------------- assert_eq!(block_header.account_root(), store.account_root().await); } + +/// Test that the current account root is returned if the batches are empty +#[tokio::test] +async fn test_compute_account_root_empty_batches() { + // Set up account states + // --------------------------------------------------------------------------------------------- + let account_ids = vec![ + unsafe { AccountId::new_unchecked(Felt::from(0b0000_0000_0000_0000u64)) }, + unsafe { AccountId::new_unchecked(Felt::from(0b1111_0000_0000_0000u64)) }, + unsafe { AccountId::new_unchecked(Felt::from(0b1111_1111_0000_0000u64)) }, + unsafe { AccountId::new_unchecked(Felt::from(0b1111_1111_1111_0000u64)) }, + unsafe { AccountId::new_unchecked(Felt::from(0b1111_1111_1111_1111u64)) }, + ]; + + let account_initial_states = vec![ + [Felt::from(1u64), Felt::from(1u64), Felt::from(1u64), Felt::from(1u64)], + [Felt::from(2u64), Felt::from(2u64), Felt::from(2u64), Felt::from(2u64)], + [Felt::from(3u64), Felt::from(3u64), Felt::from(3u64), Felt::from(3u64)], + [Felt::from(4u64), Felt::from(4u64), Felt::from(4u64), Felt::from(4u64)], + [Felt::from(5u64), Felt::from(5u64), Felt::from(5u64), Felt::from(5u64)], + ]; + + // Set up store's account SMT + // --------------------------------------------------------------------------------------------- + + let store = MockStoreSuccess::new( + account_ids + .iter() + .zip(account_initial_states.iter()) + .map(|(&account_id, &account_hash)| (account_id.into(), account_hash.into())), + BTreeSet::new(), + ); + // Block prover + // --------------------------------------------------------------------------------------------- + + // Block inputs is initialized with all the accounts and their initial state + let block_inputs_from_store: BlockInputs = + store.get_block_inputs(account_ids.iter(), std::iter::empty()).await.unwrap(); + + let batches = Vec::new(); + let block_witness = BlockWitness::new(block_inputs_from_store, batches).unwrap(); + + let block_prover = BlockProver::new(); + let block_header = block_prover.prove(block_witness).unwrap(); + + // Compare roots + // --------------------------------------------------------------------------------------------- + assert_eq!(block_header.account_root(), store.account_root().await); +} From 60de9ace54d8fb7fcf1ddff5dcabb4bdefd78249 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 10 Nov 2023 11:55:06 -0500 Subject: [PATCH 068/166] kernel: check if we should enter loop --- block-producer/src/block_builder/prover/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 1f9693b0f..96abac1f0 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -30,7 +30,10 @@ const BLOCK_KERNEL_MASM: &str = " use.std::collections::smt64 begin - push.1 + dup neq.0 + # => [0 or 1, num_accounts_updated, OLD_ACCOUNT_ROOT, + # NEW_ACCOUNT_HASH_0, account_id_0, ... , NEW_ACCOUNT_HASH_n, account_id_n] + while.true # stack: [counter, ROOT_0, ..., NEW_ACCOUNT_HASH_i, account_id_i , ...] From 2b835e6512239897c5793a7285d836d6c36b8791 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 10 Nov 2023 13:50:59 -0500 Subject: [PATCH 069/166] fix test --- .../src/block_builder/prover/tests.rs | 2 +- block-producer/src/test_utils/store.rs | 29 ++++++++++++++++--- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index 5222e9037..b45576146 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -284,7 +284,7 @@ async fn test_compute_account_root_empty_batches() { // Block inputs is initialized with all the accounts and their initial state let block_inputs_from_store: BlockInputs = - store.get_block_inputs(account_ids.iter(), std::iter::empty()).await.unwrap(); + store.get_block_inputs(std::iter::empty(), std::iter::empty()).await.unwrap(); let batches = Vec::new(); let block_witness = BlockWitness::new(block_inputs_from_store, batches).unwrap(); diff --git a/block-producer/src/test_utils/store.rs b/block-producer/src/test_utils/store.rs index 903d6b3a5..31a6e9750 100644 --- a/block-producer/src/test_utils/store.rs +++ b/block-producer/src/test_utils/store.rs @@ -1,8 +1,7 @@ use async_trait::async_trait; -use miden_air::Felt; -use miden_mock::mock::block::mock_block_header; +use miden_air::{Felt, FieldElement}; use miden_node_proto::domain::{AccountInputRecord, BlockInputs}; -use miden_objects::{crypto::merkle::MmrPeaks, EMPTY_WORD}; +use miden_objects::{crypto::merkle::MmrPeaks, BlockHeader, EMPTY_WORD}; use miden_vm::crypto::SimpleSmt; use crate::{ @@ -124,7 +123,29 @@ impl Store for MockStoreSuccess { updated_accounts: impl Iterator + Send, _produced_nullifiers: impl Iterator + Send, ) -> Result { - let block_header = mock_block_header(Felt::from(0u64), None, None, &[]); + let block_header = { + let prev_hash: Digest = Digest::default(); + let chain_root: Digest = Digest::default(); + let acct_root: Digest = self.account_root().await; + let nullifier_root: Digest = Digest::default(); + let note_root: Digest = Digest::default(); + let batch_root: Digest = Digest::default(); + let proof_hash: Digest = Digest::default(); + + BlockHeader::new( + prev_hash, + Felt::ZERO, + chain_root, + acct_root, + nullifier_root, + note_root, + batch_root, + proof_hash, + Felt::ZERO, + Felt::ONE, + ) + }; + let chain_peaks = MmrPeaks::new(0, Vec::new()).unwrap(); let account_states = { From a04bc2b33dfbdb5df01cbd2b70e0d4a3df5cc02c Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 10 Nov 2023 13:57:13 -0500 Subject: [PATCH 070/166] test_build_block_failure --- block-producer/src/block_builder/tests.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/block-producer/src/block_builder/tests.rs b/block-producer/src/block_builder/tests.rs index 550e4bc17..1d08e0eb6 100644 --- a/block-producer/src/block_builder/tests.rs +++ b/block-producer/src/block_builder/tests.rs @@ -2,12 +2,11 @@ use std::collections::BTreeSet; // block builder tests (higher level) // 1. `apply_block()` is called -// 2. if `apply_block()` fails, you fail too use super::*; use miden_air::Felt; -use crate::test_utils::MockStoreSuccess; +use crate::test_utils::{MockStoreFailure, MockStoreSuccess}; /// Tests that `build_block()` succeeds when the transaction batches are empty #[tokio::test] @@ -27,3 +26,16 @@ async fn test_apply_block_called_empty_batches() { // Ensure that the store's `apply_block()` was called assert_eq!(*store.num_apply_block_called.read().await, 1); } + +/// Tests that `build_block()` fails when `get_block_inputs()` fails +#[tokio::test] +async fn test_build_block_failure() { + let store = Arc::new(MockStoreFailure::default()); + + let block_builder = DefaultBlockBuilder::new(store.clone()); + + let result = block_builder.build_block(Vec::new()).await; + + // Ensure that the store's `apply_block()` was called + assert!(matches!(result, Err(BuildBlockError::GetBlockInputsFailed(_)))); +} From 82826b809fff219facddacda79654fc61f195ca7 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 10 Nov 2023 14:04:38 -0500 Subject: [PATCH 071/166] test_compute_account_root_empty_batches --- block-producer/src/block_builder/tests.rs | 39 ++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/block-producer/src/block_builder/tests.rs b/block-producer/src/block_builder/tests.rs index 1d08e0eb6..990931955 100644 --- a/block-producer/src/block_builder/tests.rs +++ b/block-producer/src/block_builder/tests.rs @@ -6,7 +6,44 @@ use super::*; use miden_air::Felt; -use crate::test_utils::{MockStoreFailure, MockStoreSuccess}; +use crate::{ + batch_builder::TransactionBatch, + test_utils::{DummyProvenTxGenerator, MockStoreFailure, MockStoreSuccess}, +}; + +/// Tests that `build_block()` succeeds when the transaction batches are not empty +#[tokio::test] +async fn test_apply_block_called_nonempty_batches() { + let tx_gen = DummyProvenTxGenerator::new(); + let account_id = unsafe { AccountId::new_unchecked(42u64.into()) }; + let account_initial_hash: Digest = + [Felt::from(1u64), Felt::from(1u64), Felt::from(1u64), Felt::from(1u64)].into(); + let store = Arc::new(MockStoreSuccess::new( + std::iter::once((account_id, account_initial_hash)), + BTreeSet::new(), + )); + + let block_builder = DefaultBlockBuilder::new(store.clone()); + + let batches: Vec = { + let batch_1 = { + let tx = Arc::new(tx_gen.dummy_proven_tx_with_params( + account_id, + account_initial_hash, + [Felt::from(2u64), Felt::from(2u64), Felt::from(2u64), Felt::from(2u64)].into(), + Vec::new(), + )); + + Arc::new(TransactionBatch::new(vec![tx])) + }; + + vec![batch_1] + }; + block_builder.build_block(batches).await.unwrap(); + + // Ensure that the store's `apply_block()` was called + assert_eq!(*store.num_apply_block_called.read().await, 1); +} /// Tests that `build_block()` succeeds when the transaction batches are empty #[tokio::test] From 74864c4f820ffa9007e66166a5339987dda086ad Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Mon, 13 Nov 2023 06:02:51 -0500 Subject: [PATCH 072/166] use `Felt::from(u64)` --- block-producer/src/block_builder/prover/mod.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 96abac1f0..5d10e5e18 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -138,10 +138,11 @@ impl BlockProver { let digest_elements: Vec = stack_output .iter() - .map(|&num| Felt::try_from(num).map_err(|_|BlockProverError::InvalidRootReturned)) + .cloned() + .map(Felt::from) // We reverse, since a word `[a, b, c, d]` will be stored on the stack as `[d, c, b, a]` .rev() - .collect::>()?; + .collect(); let digest_elements: [Felt; 4] = digest_elements.try_into().map_err(|_| BlockProverError::InvalidRootReturned)?; From 67f2b30d5bf6a55b4f257de573cbd7bb2d83009b Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Mon, 13 Nov 2023 09:42:43 -0500 Subject: [PATCH 073/166] remove commented out fields --- block-producer/src/block_builder/prover/mod.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 5d10e5e18..7f4452d45 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -158,8 +158,6 @@ impl BlockProver { #[derive(Debug, PartialEq, Eq)] pub(super) struct BlockWitness { updated_accounts: BTreeMap, - // account_states: Vec, - // account_updates: Vec<(AccountId, Digest)>, prev_header: BlockHeader, } From d0acbd16b25897e4170b1914d0b93743f152947d Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 14 Nov 2023 07:25:51 -0500 Subject: [PATCH 074/166] block kernel masm: move account root computation to proc --- block-producer/src/block_builder/prover/mod.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 7f4452d45..c89ba08e3 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -29,7 +29,12 @@ mod tests; const BLOCK_KERNEL_MASM: &str = " use.std::collections::smt64 -begin +#! Compute the account root +#! +#! Stack: [num_accounts_updated, OLD_ACCOUNT_ROOT, +#! NEW_ACCOUNT_HASH_0, account_id_0, ... , NEW_ACCOUNT_HASH_n, account_id_n] +#! Output: [NEW_ACCOUNT_ROOT] +proc.compute_account_root dup neq.0 # => [0 or 1, num_accounts_updated, OLD_ACCOUNT_ROOT, # NEW_ACCOUNT_HASH_0, account_id_0, ... , NEW_ACCOUNT_HASH_n, account_id_n] @@ -57,6 +62,10 @@ begin drop # => [ROOT_{n-1}] end + +begin + exec.compute_account_root +end "; #[derive(Debug)] From 98c0856f40d6642c5543c6105e3f3e797c9431f8 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 14 Nov 2023 07:33:08 -0500 Subject: [PATCH 075/166] main proc --- block-producer/src/block_builder/prover/mod.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index c89ba08e3..1d42901b4 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -63,8 +63,15 @@ proc.compute_account_root # => [ROOT_{n-1}] end +proc.main.1 + exec.compute_account_root loc_storew.0 dropw + + # Load output on stack + loc_loadw.0 +end + begin - exec.compute_account_root + exec.main end "; From 5cb6bbccaeb29e3f3c4d0d9d981b52a44300fabc Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 14 Nov 2023 08:45:55 -0500 Subject: [PATCH 076/166] proc.compute_note_root --- .../src/block_builder/prover/mod.rs | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 1d42901b4..b0018c818 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -63,6 +63,63 @@ proc.compute_account_root # => [ROOT_{n-1}] end +#! TODO: put a version of this in stdlib? +#! +#! Inserts the specified value under the specified key in a Sparse Merkle Tree of depth 8 defined by the +#! specified root. If the insert is successful, the old value located under the specified key +#! is returned via the stack. +#! +#! Inputs: +#! Operand stack: [VALUE, key, ROOT, ...] +#! +#! Outputs: +#! Operand stack: [OLD_VALUE, NEW_ROOT, ...] +export.mtree_8_set + # prepare the stack for mtree_set operation + movup.4 movdn.8 swapw movup.8 push.8 + # => [8, key, ROOT, VALUE, ...] + + mtree_set + # => [OLD_VALUE, NEW_ROOT, ...] +end + +#! Compute the note root +#! +#! Stack: [num_notes_updated, SMT_EMPTY_ROOT_DEPTH_8, NOTE_HASH_0, ... , NOTE_HASH_{n-1}] +#! Output: [NOTES_ROOT] +proc.compute_note_root + # assess if we should loop + push.0 dup.1 dup.1 neq + #=> [should_loop, loop_counter=0, num_notes_updated, SMT_EMPTY_ROOT_DEPTH_8, NOTE_HASH_0, ... , NOTE_HASH_{n-1}] + + while.true + #=> [loop_counter, num_notes_updated, ROOT_i, NOTE_HASH_i, ... ] + + # Move loop_counter and `num_notes_updated` down for next iteration + # Keep a copy of `loop_counter`; we use it as the insert key for the ith note hash + dup movdn.10 swap movdn.10 + #=> [loop_counter, ROOT_i, NOTE_HASH_i, loop_counter, num_notes_updated, ... ] + + # Prepare stack for mtree_8_set + movdn.4 + #=> [ROOT_i, loop_counter, NOTE_HASH_i, loop_counter, num_notes_updated, ... ] + + exec.mtree_8_set dropw + #=> [ROOT_{i+1}, loop_counter, num_notes_updated, ... ] + + # Prepare stack for loop counter check + movdn.5 movdn.5 movdn.5 movdn.5 + #=> [loop_counter, num_notes_updated, ROOT_{i+1}, ... ] + + # loop counter + add.1 dup.1 dup.1 neq + #=> [should_loop, loop_counter + 1, num_notes_updated, ROOT_{i+1}, ... ] + end + + drop drop + # => [ROOT_{n-1}] +end + proc.main.1 exec.compute_account_root loc_storew.0 dropw From a1e544f40e36047dff634090f08cf3db1c42da77 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 14 Nov 2023 08:52:42 -0500 Subject: [PATCH 077/166] call compute_note_root from main --- block-producer/src/block_builder/prover/mod.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index b0018c818..a3287a8ea 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -120,11 +120,17 @@ proc.compute_note_root # => [ROOT_{n-1}] end -proc.main.1 +# Stack: [, ] +proc.main.2 exec.compute_account_root loc_storew.0 dropw + #=> [] + + exec.compute_note_root loc_storew.1 dropw + #=> [ ] # Load output on stack - loc_loadw.0 + loc_loadw.1 padw loc_loadw.0 + #=> [ ACCOUNT_ROOT, NOTE_ROOT] end begin From ad4d5c1815a4e1fe902762ca47a45222882a27ec Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 14 Nov 2023 09:46:37 -0500 Subject: [PATCH 078/166] Retrieve note root from program --- .../src/block_builder/prover/mod.rs | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index a3287a8ea..30e592c86 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -74,7 +74,7 @@ end #! #! Outputs: #! Operand stack: [OLD_VALUE, NEW_ROOT, ...] -export.mtree_8_set +proc.mtree_8_set # prepare the stack for mtree_set operation movup.4 movdn.8 swapw movup.8 push.8 # => [8, key, ROOT, VALUE, ...] @@ -169,10 +169,10 @@ impl BlockProver { let block_num = witness.prev_header.block_num(); let version = witness.prev_header.version(); + let (account_root, note_root) = self.compute_roots(witness)?; + let chain_root = Digest::default(); - let account_root = self.compute_new_account_root(witness)?; let nullifier_root = Digest::default(); - let note_root = Digest::default(); let batch_root = Digest::default(); let proof_hash = Digest::default(); let timestamp: Felt = SystemTime::now() @@ -195,12 +195,10 @@ impl BlockProver { )) } - /// `current_account_states`: iterator of (account id, node hash, Merkle path) - /// `account_updates`: iterator of (account id, new account hash) - fn compute_new_account_root( + fn compute_roots( &self, witness: BlockWitness, - ) -> Result { + ) -> Result<(Digest, Digest), BlockProverError> { let (advice_inputs, stack_inputs) = witness.into_parts()?; let host = { let advice_provider = MemAdviceProvider::from(advice_inputs); @@ -212,10 +210,30 @@ impl BlockProver { execute(&self.kernel, stack_inputs, host, ExecutionOptions::default()) .map_err(BlockProverError::ProgramExecutionFailed)?; + // TODO: Use `StackOutputs::pop_digest()` once merged + let (account_root_output, note_root_output) = { + let root_outputs: Vec<_> = execution_output.stack_outputs().stack().chunks(4).collect(); + + (root_outputs[0], root_outputs[1]) + }; + let new_account_root = { - let stack_output = execution_output.stack_outputs().stack_truncated(4); + let digest_elements: Vec = account_root_output + .iter() + .cloned() + .map(Felt::from) + // We reverse, since a word `[a, b, c, d]` will be stored on the stack as `[d, c, b, a]` + .rev() + .collect(); + + let digest_elements: [Felt; 4] = + digest_elements.try_into().map_err(|_| BlockProverError::InvalidRootReturned)?; + + digest_elements.into() + }; - let digest_elements: Vec = stack_output + let new_note_root = { + let digest_elements: Vec = note_root_output .iter() .cloned() .map(Felt::from) @@ -229,7 +247,7 @@ impl BlockProver { digest_elements.into() }; - Ok(new_account_root) + Ok((new_account_root, new_note_root)) } } From 320f975587832199a679554231dee30683eb08ee Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 14 Nov 2023 10:14:25 -0500 Subject: [PATCH 079/166] TransactionBatch: process on `new()` --- block-producer/src/batch_builder/mod.rs | 69 ++++++++++++++++--------- 1 file changed, 46 insertions(+), 23 deletions(-) diff --git a/block-producer/src/batch_builder/mod.rs b/block-producer/src/batch_builder/mod.rs index 8d7eeda68..b325bc499 100644 --- a/block-producer/src/batch_builder/mod.rs +++ b/block-producer/src/batch_builder/mod.rs @@ -18,55 +18,78 @@ mod tests; /// /// Note: Until recursive proofs are available in the Miden VM, we don't include the common proof. pub struct TransactionBatch { - txs: Vec, + account_initial_states: Vec<(AccountId, Digest)>, + updated_accounts: Vec<(AccountId, Digest)>, + consumed_notes_script_roots: Vec, + produced_nullifiers: Vec, + created_notes: Vec, } impl TransactionBatch { pub fn new(txs: Vec) -> Self { - Self { txs } + let account_initial_states = + txs.iter().map(|tx| (tx.account_id(), tx.initial_account_hash())).collect(); + let updated_accounts = + txs.iter().map(|tx| (tx.account_id(), tx.final_account_hash())).collect(); + + let consumed_notes_script_roots = { + let mut script_roots: Vec = txs + .iter() + .flat_map(|tx| tx.consumed_notes()) + .map(|consumed_note| consumed_note.script_root()) + .collect(); + + script_roots.sort(); + + // Removes duplicates in consecutive items + script_roots.into_iter().dedup().collect() + }; + let produced_nullifiers = txs + .iter() + .flat_map(|tx| tx.consumed_notes()) + .map(|consumed_note| consumed_note.nullifier()) + .collect(); + + let created_notes = txs + .iter() + .flat_map(|tx| tx.created_notes()) + .map(|note_envelope| note_envelope.note_hash()) + .collect(); + + Self { + account_initial_states, + updated_accounts, + consumed_notes_script_roots, + produced_nullifiers, + created_notes, + } } /// Returns an iterator over account ids that were modified in the transaction batch, and their /// corresponding initial hash pub fn account_initial_states(&self) -> impl Iterator + '_ { - self.txs.iter().map(|tx| (tx.account_id(), tx.initial_account_hash())) + self.account_initial_states.iter().cloned() } /// Returns an iterator over account ids that were modified in the transaction batch, and their /// corresponding new hash pub fn updated_accounts(&self) -> impl Iterator + '_ { - self.txs.iter().map(|tx| (tx.account_id(), tx.final_account_hash())) + self.updated_accounts.iter().cloned() } /// Returns the script root of all consumed notes pub fn consumed_notes_script_roots(&self) -> impl Iterator + '_ { - let mut script_roots: Vec = self - .txs - .iter() - .flat_map(|tx| tx.consumed_notes()) - .map(|consumed_note| consumed_note.script_root()) - .collect(); - - script_roots.sort(); - - // Removes duplicates in consecutive items - script_roots.into_iter().dedup() + self.consumed_notes_script_roots.iter().cloned() } /// Returns the nullifier of all consumed notes pub fn produced_nullifiers(&self) -> impl Iterator + '_ { - self.txs - .iter() - .flat_map(|tx| tx.consumed_notes()) - .map(|consumed_note| consumed_note.nullifier()) + self.produced_nullifiers.iter().cloned() } /// Returns the hash of created notes pub fn created_notes(&self) -> impl Iterator + '_ { - self.txs - .iter() - .flat_map(|tx| tx.created_notes()) - .map(|note_envelope| note_envelope.note_hash()) + self.created_notes.iter().cloned() } } From b1f042119d218a6970122ff48f5d65a0c573ac63 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 14 Nov 2023 10:28:54 -0500 Subject: [PATCH 080/166] TransactionBatch: created_notes_root --- block-producer/src/batch_builder/mod.rs | 34 +++++++++++++++++++------ 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/block-producer/src/batch_builder/mod.rs b/block-producer/src/batch_builder/mod.rs index b325bc499..a1b141d9d 100644 --- a/block-producer/src/batch_builder/mod.rs +++ b/block-producer/src/batch_builder/mod.rs @@ -3,6 +3,7 @@ use std::{cmp::min, fmt::Debug, sync::Arc, time::Duration}; use async_trait::async_trait; use itertools::Itertools; use miden_objects::{accounts::AccountId, Digest}; +use miden_vm::crypto::SimpleSmt; use tokio::{sync::RwLock, time}; use crate::{block_builder::BlockBuilder, SharedProvenTx, SharedRwVec, SharedTxBatch}; @@ -10,6 +11,8 @@ use crate::{block_builder::BlockBuilder, SharedProvenTx, SharedRwVec, SharedTxBa #[cfg(test)] mod tests; +const CREATED_NOTES_SMT_DEPTH: u8 = 12; + // TRANSACTION BATCH // ================================================================================================ @@ -22,7 +25,7 @@ pub struct TransactionBatch { updated_accounts: Vec<(AccountId, Digest)>, consumed_notes_script_roots: Vec, produced_nullifiers: Vec, - created_notes: Vec, + created_notes_smt: SimpleSmt, } impl TransactionBatch { @@ -50,18 +53,28 @@ impl TransactionBatch { .map(|consumed_note| consumed_note.nullifier()) .collect(); - let created_notes = txs - .iter() - .flat_map(|tx| tx.created_notes()) - .map(|note_envelope| note_envelope.note_hash()) - .collect(); + let created_notes_smt = { + let created_notes: Vec<_> = txs + .iter() + .flat_map(|tx| tx.created_notes()) + .map(|note_envelope| note_envelope.note_hash()) + .collect(); + + SimpleSmt::with_leaves( + CREATED_NOTES_SMT_DEPTH, + created_notes.into_iter().enumerate().map(|(idx, note_hash)| { + (idx.try_into().expect("TODO: very large index?"), note_hash.into()) + }), + ) + .expect("TODO: on failure?") + }; Self { account_initial_states, updated_accounts, consumed_notes_script_roots, produced_nullifiers, - created_notes, + created_notes_smt, } } @@ -89,7 +102,12 @@ impl TransactionBatch { /// Returns the hash of created notes pub fn created_notes(&self) -> impl Iterator + '_ { - self.created_notes.iter().cloned() + self.created_notes_smt.leaves().map(|(_key, leaf)| leaf.into()) + } + + /// Returns the root of the created notes SMT + pub fn created_notes_root(&self) -> Digest { + self.created_notes_smt.root() } } From d76ac6c15c5dd69ba62827950649cda0840b4381 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 14 Nov 2023 10:39:15 -0500 Subject: [PATCH 081/166] BuildBatchError::TooManyNotes --- block-producer/src/batch_builder/errors.rs | 16 ++++++++++++++++ block-producer/src/batch_builder/mod.rs | 10 ++++------ block-producer/src/txqueue/tests/mod.rs | 2 +- 3 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 block-producer/src/batch_builder/errors.rs diff --git a/block-producer/src/batch_builder/errors.rs b/block-producer/src/batch_builder/errors.rs new file mode 100644 index 000000000..18d7f7c7d --- /dev/null +++ b/block-producer/src/batch_builder/errors.rs @@ -0,0 +1,16 @@ +use thiserror::Error; + +use super::CREATED_NOTES_SMT_DEPTH; + +const MAX_NUM_CREATED_NOTES_PER_BATCH: usize = 2usize.pow(CREATED_NOTES_SMT_DEPTH as u32); + +#[derive(Error, Debug, PartialEq)] +pub enum BuildBatchError { + #[error("dummy")] + Dummy, + #[error( + "Too many notes in the batch. Got: {0}, max: {}", + MAX_NUM_CREATED_NOTES_PER_BATCH + )] + TooManyNotes(usize), +} diff --git a/block-producer/src/batch_builder/mod.rs b/block-producer/src/batch_builder/mod.rs index a1b141d9d..3ca1e9cea 100644 --- a/block-producer/src/batch_builder/mod.rs +++ b/block-producer/src/batch_builder/mod.rs @@ -1,4 +1,4 @@ -use std::{cmp::min, fmt::Debug, sync::Arc, time::Duration}; +use std::{cmp::min, sync::Arc, time::Duration}; use async_trait::async_trait; use itertools::Itertools; @@ -8,6 +8,9 @@ use tokio::{sync::RwLock, time}; use crate::{block_builder::BlockBuilder, SharedProvenTx, SharedRwVec, SharedTxBatch}; +use self::errors::BuildBatchError; + +pub mod errors; #[cfg(test)] mod tests; @@ -114,11 +117,6 @@ impl TransactionBatch { // BATCH BUILDER // ================================================================================================ -#[derive(Debug, PartialEq)] -pub enum BuildBatchError { - Dummy, -} - #[async_trait] pub trait BatchBuilder: Send + Sync + 'static { async fn build_batch( diff --git a/block-producer/src/txqueue/tests/mod.rs b/block-producer/src/txqueue/tests/mod.rs index 910d499fe..9b872928e 100644 --- a/block-producer/src/txqueue/tests/mod.rs +++ b/block-producer/src/txqueue/tests/mod.rs @@ -1,6 +1,6 @@ use super::*; use crate::{ - batch_builder::{BuildBatchError, TransactionBatch}, + batch_builder::{errors::BuildBatchError, TransactionBatch}, test_utils::DummyProvenTxGenerator, SharedTxBatch, }; From 5f2ea0469d03d0bea10aa88d6eaecd7e2226092e Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 14 Nov 2023 10:44:17 -0500 Subject: [PATCH 082/166] TransactionBatch::new returns Result --- block-producer/src/batch_builder/errors.rs | 4 +--- block-producer/src/batch_builder/mod.rs | 18 +++++++++++++----- block-producer/src/batch_builder/tests/mod.rs | 2 +- .../src/block_builder/prover/tests.rs | 12 ++++++------ block-producer/src/block_builder/tests.rs | 2 +- block-producer/src/txqueue/tests/mod.rs | 2 +- 6 files changed, 23 insertions(+), 17 deletions(-) diff --git a/block-producer/src/batch_builder/errors.rs b/block-producer/src/batch_builder/errors.rs index 18d7f7c7d..04e32fc05 100644 --- a/block-producer/src/batch_builder/errors.rs +++ b/block-producer/src/batch_builder/errors.rs @@ -1,8 +1,6 @@ use thiserror::Error; -use super::CREATED_NOTES_SMT_DEPTH; - -const MAX_NUM_CREATED_NOTES_PER_BATCH: usize = 2usize.pow(CREATED_NOTES_SMT_DEPTH as u32); +use super::MAX_NUM_CREATED_NOTES_PER_BATCH; #[derive(Error, Debug, PartialEq)] pub enum BuildBatchError { diff --git a/block-producer/src/batch_builder/mod.rs b/block-producer/src/batch_builder/mod.rs index 3ca1e9cea..4abe17024 100644 --- a/block-producer/src/batch_builder/mod.rs +++ b/block-producer/src/batch_builder/mod.rs @@ -15,6 +15,7 @@ pub mod errors; mod tests; const CREATED_NOTES_SMT_DEPTH: u8 = 12; +const MAX_NUM_CREATED_NOTES_PER_BATCH: usize = 2usize.pow(CREATED_NOTES_SMT_DEPTH as u32); // TRANSACTION BATCH // ================================================================================================ @@ -32,7 +33,7 @@ pub struct TransactionBatch { } impl TransactionBatch { - pub fn new(txs: Vec) -> Self { + pub fn new(txs: Vec) -> Result { let account_initial_states = txs.iter().map(|tx| (tx.account_id(), tx.initial_account_hash())).collect(); let updated_accounts = @@ -63,22 +64,29 @@ impl TransactionBatch { .map(|note_envelope| note_envelope.note_hash()) .collect(); + if created_notes.len() > MAX_NUM_CREATED_NOTES_PER_BATCH { + return Err(BuildBatchError::TooManyNotes(created_notes.len())); + } + SimpleSmt::with_leaves( CREATED_NOTES_SMT_DEPTH, created_notes.into_iter().enumerate().map(|(idx, note_hash)| { - (idx.try_into().expect("TODO: very large index?"), note_hash.into()) + ( + idx.try_into().expect("already checked not for too many notes"), + note_hash.into(), + ) }), ) .expect("TODO: on failure?") }; - Self { + Ok(Self { account_initial_states, updated_accounts, consumed_notes_script_roots, produced_nullifiers, created_notes_smt, - } + }) } /// Returns an iterator over account ids that were modified in the transaction batch, and their @@ -199,7 +207,7 @@ where &self, txs: Vec, ) -> Result<(), BuildBatchError> { - let batch = Arc::new(TransactionBatch::new(txs)); + let batch = Arc::new(TransactionBatch::new(txs).unwrap()); self.ready_batches.write().await.push(batch); Ok(()) diff --git a/block-producer/src/batch_builder/tests/mod.rs b/block-producer/src/batch_builder/tests/mod.rs index 826404a3b..93b757083 100644 --- a/block-producer/src/batch_builder/tests/mod.rs +++ b/block-producer/src/batch_builder/tests/mod.rs @@ -169,5 +169,5 @@ fn dummy_tx_batch( .map(|_| Arc::new(tx_gen.dummy_proven_tx())) .collect(); - Arc::new(TransactionBatch::new(txs)) + Arc::new(TransactionBatch::new(txs).unwrap()) } diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index b45576146..254285891 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -58,7 +58,7 @@ fn test_block_witness_validation_inconsistent_account_ids() { Vec::new(), )); - Arc::new(TransactionBatch::new(vec![tx])) + Arc::new(TransactionBatch::new(vec![tx]).unwrap()) }; let batch_2 = { @@ -69,7 +69,7 @@ fn test_block_witness_validation_inconsistent_account_ids() { Vec::new(), )); - Arc::new(TransactionBatch::new(vec![tx])) + Arc::new(TransactionBatch::new(vec![tx]).unwrap()) }; vec![batch_1, batch_2] @@ -132,7 +132,7 @@ fn test_block_witness_validation_inconsistent_account_hashes() { Vec::new(), )); - Arc::new(TransactionBatch::new(vec![tx])) + Arc::new(TransactionBatch::new(vec![tx]).unwrap()) }; let batch_2 = { @@ -143,7 +143,7 @@ fn test_block_witness_validation_inconsistent_account_hashes() { Vec::new(), )); - Arc::new(TransactionBatch::new(vec![tx])) + Arc::new(TransactionBatch::new(vec![tx]).unwrap()) }; vec![batch_1, batch_2] @@ -221,8 +221,8 @@ async fn test_compute_account_root_success() { }) .collect(); - let batch_1 = Arc::new(TransactionBatch::new(txs[..2].to_vec())); - let batch_2 = Arc::new(TransactionBatch::new(txs[2..].to_vec())); + let batch_1 = Arc::new(TransactionBatch::new(txs[..2].to_vec()).unwrap()); + let batch_2 = Arc::new(TransactionBatch::new(txs[2..].to_vec()).unwrap()); vec![batch_1, batch_2] }; diff --git a/block-producer/src/block_builder/tests.rs b/block-producer/src/block_builder/tests.rs index 990931955..0683f077d 100644 --- a/block-producer/src/block_builder/tests.rs +++ b/block-producer/src/block_builder/tests.rs @@ -34,7 +34,7 @@ async fn test_apply_block_called_nonempty_batches() { Vec::new(), )); - Arc::new(TransactionBatch::new(vec![tx])) + Arc::new(TransactionBatch::new(vec![tx]).unwrap()) }; vec![batch_1] diff --git a/block-producer/src/txqueue/tests/mod.rs b/block-producer/src/txqueue/tests/mod.rs index 9b872928e..d734ed151 100644 --- a/block-producer/src/txqueue/tests/mod.rs +++ b/block-producer/src/txqueue/tests/mod.rs @@ -47,7 +47,7 @@ impl BatchBuilder for BatchBuilderSuccess { &self, txs: Vec, ) -> Result<(), BuildBatchError> { - let batch = Arc::new(TransactionBatch::new(txs)); + let batch = Arc::new(TransactionBatch::new(txs).unwrap()); self.ready_batches.write().await.push(batch); Ok(()) From 4528e68be48607b5ec07062a1b1603e4864c2098 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 14 Nov 2023 10:45:39 -0500 Subject: [PATCH 083/166] remove expect() --- block-producer/src/batch_builder/errors.rs | 3 +++ block-producer/src/batch_builder/mod.rs | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/block-producer/src/batch_builder/errors.rs b/block-producer/src/batch_builder/errors.rs index 04e32fc05..be67881c8 100644 --- a/block-producer/src/batch_builder/errors.rs +++ b/block-producer/src/batch_builder/errors.rs @@ -1,3 +1,4 @@ +use miden_vm::crypto::MerkleError; use thiserror::Error; use super::MAX_NUM_CREATED_NOTES_PER_BATCH; @@ -11,4 +12,6 @@ pub enum BuildBatchError { MAX_NUM_CREATED_NOTES_PER_BATCH )] TooManyNotes(usize), + #[error("failed to create notes SMT: {0}")] + NotesSmtError(#[from] MerkleError) } diff --git a/block-producer/src/batch_builder/mod.rs b/block-producer/src/batch_builder/mod.rs index 4abe17024..7c2a9bbf9 100644 --- a/block-producer/src/batch_builder/mod.rs +++ b/block-producer/src/batch_builder/mod.rs @@ -76,8 +76,7 @@ impl TransactionBatch { note_hash.into(), ) }), - ) - .expect("TODO: on failure?") + )? }; Ok(Self { From 50be10e83b3afdadb03b4d0975309edbf852c064 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 14 Nov 2023 14:56:53 -0500 Subject: [PATCH 084/166] test_compute_note_root_success --- .../src/block_builder/prover/tests.rs | 84 ++++++++++++++++++- 1 file changed, 83 insertions(+), 1 deletion(-) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index 254285891..bf0c423a4 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -3,7 +3,11 @@ use std::sync::Arc; use miden_air::FieldElement; use miden_mock::mock::block::mock_block_header; use miden_node_proto::domain::AccountInputRecord; -use miden_objects::crypto::merkle::MmrPeaks; +use miden_objects::{ + crypto::merkle::MmrPeaks, + notes::{NoteEnvelope, NoteMetadata}, +}; +use miden_vm::crypto::SimpleSmt; use crate::{ batch_builder::TransactionBatch, @@ -13,6 +17,9 @@ use crate::{ use super::*; +// BLOCK WITNESS TESTS +// ================================================================================================= + /// Tests that `BlockWitness` constructor fails if the store and transaction batches contain a /// different set of account ids. /// @@ -157,6 +164,9 @@ fn test_block_witness_validation_inconsistent_account_hashes() { ); } +// ACCOUNT ROOT TESTS +// ================================================================================================= + /// Tests that the `BlockProver` computes the proper account root. /// /// We assume an initial store with 5 accounts, and all will be updated. @@ -296,3 +306,75 @@ async fn test_compute_account_root_empty_batches() { // --------------------------------------------------------------------------------------------- assert_eq!(block_header.account_root(), store.account_root().await); } + +// NOTE ROOT TESTS +// ================================================================================================= + +#[tokio::test] +async fn test_compute_note_root_success() { + let tx_gen = DummyProvenTxGenerator::new(); + + let dummy_account_id = unsafe { AccountId::new_unchecked(Felt::from(0u64)) }; + + let notes_created: Vec = vec![ + Digest::from([Felt::from(1u64), Felt::from(1u64), Felt::from(1u64), Felt::from(1u64)]), + Digest::from([Felt::from(2u64), Felt::from(2u64), Felt::from(2u64), Felt::from(2u64)]), + Digest::from([Felt::from(3u64), Felt::from(3u64), Felt::from(3u64), Felt::from(3u64)]), + ] + .into_iter() + .map(|note_digest| { + NoteEnvelope::new( + note_digest, + NoteMetadata::new(dummy_account_id, Felt::from(1u64), Felt::from(0u64)), + ) + }) + .collect(); + + // Set up store + // --------------------------------------------------------------------------------------------- + + let store = MockStoreSuccess::new(std::iter::empty(), BTreeSet::new()); + + // Block prover + // --------------------------------------------------------------------------------------------- + + // Block inputs is initialized with all the accounts and their initial state + let block_inputs_from_store: BlockInputs = + store.get_block_inputs(std::iter::empty(), std::iter::empty()).await.unwrap(); + + let batches: Vec = { + let txs: Vec<_> = notes_created + .iter() + .map(|note| { + Arc::new(tx_gen.dummy_proven_tx_with_params( + dummy_account_id, + Digest::default(), + Digest::default(), + Vec::new(), + )) + }) + .collect(); + + let batch_1 = Arc::new(TransactionBatch::new(txs[..2].to_vec()).unwrap()); + let batch_2 = Arc::new(TransactionBatch::new(txs[2..].to_vec()).unwrap()); + + vec![batch_1, batch_2] + }; + + let block_witness = BlockWitness::new(block_inputs_from_store, batches).unwrap(); + + let block_prover = BlockProver::new(); + let block_header = block_prover.prove(block_witness).unwrap(); + + // Create SMT by hand to get new root + // --------------------------------------------------------------------------------------------- + let notes_smt = SimpleSmt::with_leaves( + 8, + notes_created.into_iter().enumerate().map(|(idx, note)| (idx as u64, note.note_hash().into())), + ) + .unwrap(); + + // Compare roots + // --------------------------------------------------------------------------------------------- + assert_eq!(block_header.note_root(), notes_smt.root()); +} From 34a3c6d56a5d6ea43fcb012fbc9d567524b1a71e Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 14 Nov 2023 15:08:39 -0500 Subject: [PATCH 085/166] add arg to tx_gen --- block-producer/src/batch_builder/errors.rs | 2 +- .../src/block_builder/prover/tests.rs | 29 ++++++++++++++----- block-producer/src/block_builder/tests.rs | 1 + .../src/state_view/tests/apply_block.rs | 2 ++ block-producer/src/state_view/tests/mod.rs | 1 + .../src/state_view/tests/verify_tx.rs | 7 +++++ block-producer/src/test_utils/proven_tx.rs | 4 ++- 7 files changed, 37 insertions(+), 9 deletions(-) diff --git a/block-producer/src/batch_builder/errors.rs b/block-producer/src/batch_builder/errors.rs index be67881c8..f653d40d8 100644 --- a/block-producer/src/batch_builder/errors.rs +++ b/block-producer/src/batch_builder/errors.rs @@ -13,5 +13,5 @@ pub enum BuildBatchError { )] TooManyNotes(usize), #[error("failed to create notes SMT: {0}")] - NotesSmtError(#[from] MerkleError) + NotesSmtError(#[from] MerkleError), } diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index bf0c423a4..cafa4ce92 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -63,6 +63,7 @@ fn test_block_witness_validation_inconsistent_account_ids() { Digest::default(), Digest::default(), Vec::new(), + Vec::new(), )); Arc::new(TransactionBatch::new(vec![tx]).unwrap()) @@ -74,6 +75,7 @@ fn test_block_witness_validation_inconsistent_account_ids() { Digest::default(), Digest::default(), Vec::new(), + Vec::new(), )); Arc::new(TransactionBatch::new(vec![tx]).unwrap()) @@ -137,6 +139,7 @@ fn test_block_witness_validation_inconsistent_account_hashes() { account_1_hash_batches, Digest::default(), Vec::new(), + Vec::new(), )); Arc::new(TransactionBatch::new(vec![tx]).unwrap()) @@ -148,6 +151,7 @@ fn test_block_witness_validation_inconsistent_account_hashes() { Digest::default(), Digest::default(), Vec::new(), + Vec::new(), )); Arc::new(TransactionBatch::new(vec![tx]).unwrap()) @@ -227,6 +231,7 @@ async fn test_compute_account_root_success() { account_initial_states[idx].into(), account_final_states[idx].into(), Vec::new(), + Vec::new(), )) }) .collect(); @@ -314,7 +319,11 @@ async fn test_compute_account_root_empty_batches() { async fn test_compute_note_root_success() { let tx_gen = DummyProvenTxGenerator::new(); - let dummy_account_id = unsafe { AccountId::new_unchecked(Felt::from(0u64)) }; + let account_ids = vec![ + unsafe { AccountId::new_unchecked(Felt::from(0u64)) }, + unsafe { AccountId::new_unchecked(Felt::from(1u64)) }, + unsafe { AccountId::new_unchecked(Felt::from(2u64)) }, + ]; let notes_created: Vec = vec![ Digest::from([Felt::from(1u64), Felt::from(1u64), Felt::from(1u64), Felt::from(1u64)]), @@ -322,10 +331,11 @@ async fn test_compute_note_root_success() { Digest::from([Felt::from(3u64), Felt::from(3u64), Felt::from(3u64), Felt::from(3u64)]), ] .into_iter() - .map(|note_digest| { + .zip(account_ids.iter()) + .map(|(note_digest, &account_id)| { NoteEnvelope::new( note_digest, - NoteMetadata::new(dummy_account_id, Felt::from(1u64), Felt::from(0u64)), + NoteMetadata::new(account_id, Felt::from(1u64), Felt::from(0u64)), ) }) .collect(); @@ -340,17 +350,19 @@ async fn test_compute_note_root_success() { // Block inputs is initialized with all the accounts and their initial state let block_inputs_from_store: BlockInputs = - store.get_block_inputs(std::iter::empty(), std::iter::empty()).await.unwrap(); + store.get_block_inputs(account_ids.iter(), std::iter::empty()).await.unwrap(); let batches: Vec = { let txs: Vec<_> = notes_created .iter() - .map(|note| { + .zip(account_ids.iter()) + .map(|(note, &account_id)| { Arc::new(tx_gen.dummy_proven_tx_with_params( - dummy_account_id, + account_id, Digest::default(), Digest::default(), Vec::new(), + vec![note.clone()], )) }) .collect(); @@ -370,7 +382,10 @@ async fn test_compute_note_root_success() { // --------------------------------------------------------------------------------------------- let notes_smt = SimpleSmt::with_leaves( 8, - notes_created.into_iter().enumerate().map(|(idx, note)| (idx as u64, note.note_hash().into())), + notes_created + .into_iter() + .enumerate() + .map(|(idx, note)| (idx as u64, note.note_hash().into())), ) .unwrap(); diff --git a/block-producer/src/block_builder/tests.rs b/block-producer/src/block_builder/tests.rs index 0683f077d..007a5c558 100644 --- a/block-producer/src/block_builder/tests.rs +++ b/block-producer/src/block_builder/tests.rs @@ -32,6 +32,7 @@ async fn test_apply_block_called_nonempty_batches() { account_initial_hash, [Felt::from(2u64), Felt::from(2u64), Felt::from(2u64), Felt::from(2u64)].into(), Vec::new(), + Vec::new(), )); Arc::new(TransactionBatch::new(vec![tx]).unwrap()) diff --git a/block-producer/src/state_view/tests/apply_block.rs b/block-producer/src/state_view/tests/apply_block.rs index 3c7ffd082..fe927b761 100644 --- a/block-producer/src/state_view/tests/apply_block.rs +++ b/block-producer/src/state_view/tests/apply_block.rs @@ -26,6 +26,7 @@ async fn test_apply_block_ab1() { account.states[0], account.states[1], Vec::new(), + Vec::new(), ); let state_view = DefaulStateView::new(store.clone()); @@ -115,6 +116,7 @@ async fn test_apply_block_ab3() { accounts[0].states[1], accounts[0].states[2], txs[0].consumed_notes().to_vec(), + Vec::new(), ); let verify_tx_res = state_view.verify_tx(tx_new.into()).await; diff --git a/block-producer/src/state_view/tests/mod.rs b/block-producer/src/state_view/tests/mod.rs index 466ba7d9c..5874b792b 100644 --- a/block-producer/src/state_view/tests/mod.rs +++ b/block-producer/src/state_view/tests/mod.rs @@ -27,6 +27,7 @@ pub fn get_txs_and_accounts<'a>( account.states[0], account.states[1], vec![consumed_note_by_index(index)], + Vec::new(), ); (Arc::new(tx), account) diff --git a/block-producer/src/state_view/tests/verify_tx.rs b/block-producer/src/state_view/tests/verify_tx.rs index e345dd7b7..0f1bafd2d 100644 --- a/block-producer/src/state_view/tests/verify_tx.rs +++ b/block-producer/src/state_view/tests/verify_tx.rs @@ -90,6 +90,7 @@ async fn test_verify_tx_vt1() { account.states[1], account.states[2], vec![consumed_note_by_index(0)], + Vec::new(), ); let state_view = DefaulStateView::new(store); @@ -120,6 +121,7 @@ async fn test_verify_tx_vt2() { account_not_in_store.states[0], account_not_in_store.states[1], vec![consumed_note_by_index(0)], + Vec::new(), ); let state_view = DefaulStateView::new(store); @@ -155,6 +157,7 @@ async fn test_verify_tx_vt3() { account.states[0], account.states[1], vec![consumed_note_in_store], + Vec::new(), ); let state_view = DefaulStateView::new(store); @@ -186,6 +189,7 @@ async fn test_verify_tx_vt4() { account.states[0], account.states[1], Vec::new(), + Vec::new(), ); // Notice: tx2 modifies the same account as tx1, even though from a different initial state, @@ -195,6 +199,7 @@ async fn test_verify_tx_vt4() { account.states[1], account.states[2], Vec::new(), + Vec::new(), ); let state_view = DefaulStateView::new(store); @@ -231,6 +236,7 @@ async fn test_verify_tx_vt5() { account_1.states[0], account_1.states[1], vec![consumed_note_in_both_txs], + Vec::new(), ); // Notice: tx2 modifies the same account as tx1, even though from a different initial state, @@ -240,6 +246,7 @@ async fn test_verify_tx_vt5() { account_2.states[1], account_2.states[2], vec![consumed_note_in_both_txs], + Vec::new(), ); let state_view = DefaulStateView::new(store); diff --git a/block-producer/src/test_utils/proven_tx.rs b/block-producer/src/test_utils/proven_tx.rs index 466e5e4de..9ffad77d3 100644 --- a/block-producer/src/test_utils/proven_tx.rs +++ b/block-producer/src/test_utils/proven_tx.rs @@ -2,6 +2,7 @@ use miden_air::{ExecutionProof, HashFunction}; use miden_mock::constants::ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_ON_CHAIN; use miden_objects::{ accounts::AccountId, + notes::NoteEnvelope, transaction::{ConsumedNoteInfo, ProvenTransaction}, Digest, }; @@ -51,13 +52,14 @@ impl DummyProvenTxGenerator { initial_account_hash: Digest, final_account_hash: Digest, consumed_notes: Vec, + created_notes: Vec, ) -> ProvenTransaction { ProvenTransaction::new( account_id, initial_account_hash, final_account_hash, consumed_notes, - Vec::new(), + created_notes, None, Digest::default(), ExecutionProof::new(self.stark_proof.clone(), HashFunction::Blake3_192), From 28da2f9689a442eb6bfab6e79a17e4f61fab4c37 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 14 Nov 2023 15:56:44 -0500 Subject: [PATCH 086/166] fix BlockWitness --- .../src/block_builder/prover/mod.rs | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 30e592c86..5dbe5fe1d 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -6,8 +6,10 @@ use std::{ use miden_air::ExecutionOptions; use miden_node_proto::domain::BlockInputs; use miden_objects::{ - accounts::AccountId, assembly::Assembler, crypto::merkle::MerkleStore, BlockHeader, Digest, - Felt, + accounts::AccountId, + assembly::Assembler, + crypto::merkle::{EmptySubtreeRoots, MerkleStore}, + BlockHeader, Digest, Felt, }; use miden_stdlib::StdLibrary; use miden_vm::{ @@ -255,6 +257,8 @@ impl BlockProver { #[derive(Debug, PartialEq, Eq)] pub(super) struct BlockWitness { updated_accounts: BTreeMap, + /// collection of all batches' created notes SMT roots + batch_created_notes_roots: Vec, prev_header: BlockHeader, } @@ -298,8 +302,12 @@ impl BlockWitness { .collect() }; + let batch_created_notes_roots = + batches.iter().map(|batch| batch.created_notes_root()).collect(); + Ok(Self { updated_accounts, + batch_created_notes_roots, prev_header: block_inputs.block_header, }) } @@ -376,7 +384,19 @@ impl BlockWitness { // from the bottom to the top let mut stack_inputs = Vec::new(); - // append all insert key/values + // Notes stack inputs + { + let num_created_notes_roots = self.batch_created_notes_roots.len(); + for batch_created_notes_root in self.batch_created_notes_roots { + let root_eles: [Felt; 4] = batch_created_notes_root.into(); + stack_inputs.extend(root_eles); + } + let empty_root_depth_8 = EmptySubtreeRoots::entry(8, 0); + stack_inputs.extend(*empty_root_depth_8); + stack_inputs.push(Felt::from(num_created_notes_roots as u64)); + } + + // Account stack inputs let mut num_accounts_updated: u64 = 0; for (idx, (&account_id, account_update)) in self.updated_accounts.iter().enumerate() { stack_inputs.push(account_id.into()); From 8256842a6ec12782eaaffaa4cf17fb5b8d172898 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 14 Nov 2023 15:58:10 -0500 Subject: [PATCH 087/166] rename error --- block-producer/src/block_builder/errors.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/block-producer/src/block_builder/errors.rs b/block-producer/src/block_builder/errors.rs index 0ceeb7f27..1a682e04f 100644 --- a/block-producer/src/block_builder/errors.rs +++ b/block-producer/src/block_builder/errors.rs @@ -6,10 +6,10 @@ use crate::store::{ApplyBlockError, BlockInputsError}; #[derive(Debug, Error, PartialEq, Eq)] pub enum BuildBlockError { - #[error("failed to update account root: {0}")] - AccountRootUpdateFailed(#[from] BlockProverError), #[error("failed to apply block: {0}")] ApplyBlockFailed(#[from] ApplyBlockError), + #[error("failed to compute new block header: {0}")] + ComputeBlockHeaderFailed(#[from] BlockProverError), #[error("failed to get block inputs from store: {0}")] GetBlockInputsFailed(#[from] BlockInputsError), #[error("transaction batches and store don't modify the same account IDs. Offending accounts: {0:?}")] From 4f4115953e4bef22c0f71ff51e6de3f6950dff30 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 16 Nov 2023 08:15:12 -0500 Subject: [PATCH 088/166] test: empty notes --- .../src/block_builder/prover/tests.rs | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index cafa4ce92..2a18005e8 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -315,6 +315,40 @@ async fn test_compute_account_root_empty_batches() { // NOTE ROOT TESTS // ================================================================================================= +/// Tests that the block kernel returns the empty tree if no notes were created +#[tokio::test] +async fn test_compute_note_root_empty_notes_success() { + // Set up store + // --------------------------------------------------------------------------------------------- + + let store = MockStoreSuccess::new(std::iter::empty(), BTreeSet::new()); + + // Block prover + // --------------------------------------------------------------------------------------------- + + // Block inputs is initialized with all the accounts and their initial state + let block_inputs_from_store: BlockInputs = + store.get_block_inputs(std::iter::empty(), std::iter::empty()).await.unwrap(); + + let batches: Vec = { + let batch = Arc::new(TransactionBatch::new(Vec::new()).unwrap()); + vec![batch] + }; + + let block_witness = BlockWitness::new(block_inputs_from_store, batches).unwrap(); + + let block_prover = BlockProver::new(); + let block_header = block_prover.prove(block_witness).unwrap(); + + // Create SMT by hand to get empty root + // --------------------------------------------------------------------------------------------- + let notes_smt = SimpleSmt::new(8).unwrap(); + + // Compare roots + // --------------------------------------------------------------------------------------------- + assert_eq!(block_header.note_root(), notes_smt.root()); +} + #[tokio::test] async fn test_compute_note_root_success() { let tx_gen = DummyProvenTxGenerator::new(); From 58254411b7826264cdc2876001c9c687f8d8cad4 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 16 Nov 2023 09:53:04 -0500 Subject: [PATCH 089/166] comment --- block-producer/src/block_builder/prover/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 5dbe5fe1d..b13eee2e6 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -302,6 +302,7 @@ impl BlockWitness { .collect() }; + // TODO: Validate that there are less than 2^8 roots (and in StateView) let batch_created_notes_roots = batches.iter().map(|batch| batch.created_notes_root()).collect(); @@ -391,8 +392,9 @@ impl BlockWitness { let root_eles: [Felt; 4] = batch_created_notes_root.into(); stack_inputs.extend(root_eles); } - let empty_root_depth_8 = EmptySubtreeRoots::entry(8, 0); - stack_inputs.extend(*empty_root_depth_8); + + let empty_root_depth_20 = EmptySubtreeRoots::entry(20, 0); + stack_inputs.extend(*empty_root_depth_20); stack_inputs.push(Felt::from(num_created_notes_roots as u64)); } From 007ec56691dc939f9ff667fe1d53b086a6d38583 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 16 Nov 2023 10:42:20 -0500 Subject: [PATCH 090/166] test note root for empty batches --- .../src/block_builder/prover/mod.rs | 6 ++-- .../src/block_builder/prover/tests.rs | 32 +++++++++++++++++-- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index b13eee2e6..b20e11350 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -392,9 +392,9 @@ impl BlockWitness { let root_eles: [Felt; 4] = batch_created_notes_root.into(); stack_inputs.extend(root_eles); } - - let empty_root_depth_20 = EmptySubtreeRoots::entry(20, 0); - stack_inputs.extend(*empty_root_depth_20); + + let empty_root_depth_8 = EmptySubtreeRoots::entry(8, 0); + stack_inputs.extend(*empty_root_depth_8); stack_inputs.push(Felt::from(num_created_notes_roots as u64)); } diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index 2a18005e8..f7659bd8b 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -315,7 +315,35 @@ async fn test_compute_account_root_empty_batches() { // NOTE ROOT TESTS // ================================================================================================= -/// Tests that the block kernel returns the empty tree if no notes were created +/// Tests that the block kernel returns the empty tree (depth 8) if no notes were created, but which contains at least 1 batch. +#[tokio::test] +async fn test_compute_note_root_empty_batches_success() { + // Set up store + // --------------------------------------------------------------------------------------------- + + let store = MockStoreSuccess::new(std::iter::empty(), BTreeSet::new()); + + // Block prover + // --------------------------------------------------------------------------------------------- + + // Block inputs is initialized with all the accounts and their initial state + let block_inputs_from_store: BlockInputs = + store.get_block_inputs(std::iter::empty(), std::iter::empty()).await.unwrap(); + + let batches: Vec = Vec::new(); + + let block_witness = BlockWitness::new(block_inputs_from_store, batches).unwrap(); + + let block_prover = BlockProver::new(); + let block_header = block_prover.prove(block_witness).unwrap(); + + // Compare roots + // --------------------------------------------------------------------------------------------- + let empty_root_depth_8 = EmptySubtreeRoots::entry(8, 0); + assert_eq!(block_header.note_root(), *empty_root_depth_8); +} + +/// Tests that the block kernel returns the proper tree if no notes were created, but which contains at least 1 batch. #[tokio::test] async fn test_compute_note_root_empty_notes_success() { // Set up store @@ -342,7 +370,7 @@ async fn test_compute_note_root_empty_notes_success() { // Create SMT by hand to get empty root // --------------------------------------------------------------------------------------------- - let notes_smt = SimpleSmt::new(8).unwrap(); + let notes_smt = SimpleSmt::new(20).unwrap(); // Compare roots // --------------------------------------------------------------------------------------------- From 372a6111b9342b967bb34c182abcd4fd4e2cbd8e Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 16 Nov 2023 10:50:33 -0500 Subject: [PATCH 091/166] test_compute_note_root_empty_notes_success --- block-producer/src/batch_builder/mod.rs | 2 +- block-producer/src/block_builder/prover/tests.rs | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/block-producer/src/batch_builder/mod.rs b/block-producer/src/batch_builder/mod.rs index 7c2a9bbf9..f21a0dd5a 100644 --- a/block-producer/src/batch_builder/mod.rs +++ b/block-producer/src/batch_builder/mod.rs @@ -14,7 +14,7 @@ pub mod errors; #[cfg(test)] mod tests; -const CREATED_NOTES_SMT_DEPTH: u8 = 12; +pub const CREATED_NOTES_SMT_DEPTH: u8 = 12; const MAX_NUM_CREATED_NOTES_PER_BATCH: usize = 2usize.pow(CREATED_NOTES_SMT_DEPTH as u32); // TRANSACTION BATCH diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index f7659bd8b..a3a81a087 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -10,7 +10,7 @@ use miden_objects::{ use miden_vm::crypto::SimpleSmt; use crate::{ - batch_builder::TransactionBatch, + batch_builder::{TransactionBatch, CREATED_NOTES_SMT_DEPTH}, store::Store, test_utils::{DummyProvenTxGenerator, MockStoreSuccess}, }; @@ -370,7 +370,12 @@ async fn test_compute_note_root_empty_notes_success() { // Create SMT by hand to get empty root // --------------------------------------------------------------------------------------------- - let notes_smt = SimpleSmt::new(20).unwrap(); + + let notes_smt = { + // Since there's 1 batch (no notes created), the SMT contains 1 leaf with an empty root + let batch_notes_empty_root = EmptySubtreeRoots::entry(CREATED_NOTES_SMT_DEPTH, 0); + SimpleSmt::with_leaves(8, std::iter::once((0, batch_notes_empty_root.into()))).unwrap() + }; // Compare roots // --------------------------------------------------------------------------------------------- From 582d8ca5d12aef578e0e89243ee46baaa01fa011 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 16 Nov 2023 13:07:20 -0500 Subject: [PATCH 092/166] constants --- block-producer/src/block_builder/prover/mod.rs | 10 +++++++++- block-producer/src/block_builder/prover/tests.rs | 11 ++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index b20e11350..7061f5c40 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -16,10 +16,17 @@ use miden_vm::{ crypto::MerklePath, execute, AdviceInputs, DefaultHost, MemAdviceProvider, Program, StackInputs, }; -use crate::SharedTxBatch; +use crate::{batch_builder, SharedTxBatch}; use super::{errors::BlockProverError, BuildBlockError}; +/// The depth at which we insert roots from the batches. +pub const CREATED_NOTES_TREE_INSERTION_DEPTH: u8 = 8; + +/// The depth of the created notes tree in the block. +pub const CREATED_NOTES_TREE_DEPTH: u8 = + CREATED_NOTES_TREE_INSERTION_DEPTH + batch_builder::CREATED_NOTES_SMT_DEPTH; + #[cfg(test)] mod tests; @@ -390,6 +397,7 @@ impl BlockWitness { let num_created_notes_roots = self.batch_created_notes_roots.len(); for batch_created_notes_root in self.batch_created_notes_roots { let root_eles: [Felt; 4] = batch_created_notes_root.into(); + println!("Adding batch root: {:?}", root_eles); stack_inputs.extend(root_eles); } diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index a3a81a087..d213d0cf8 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -339,7 +339,7 @@ async fn test_compute_note_root_empty_batches_success() { // Compare roots // --------------------------------------------------------------------------------------------- - let empty_root_depth_8 = EmptySubtreeRoots::entry(8, 0); + let empty_root_depth_8 = EmptySubtreeRoots::entry(20, 0); assert_eq!(block_header.note_root(), *empty_root_depth_8); } @@ -374,11 +374,20 @@ async fn test_compute_note_root_empty_notes_success() { let notes_smt = { // Since there's 1 batch (no notes created), the SMT contains 1 leaf with an empty root let batch_notes_empty_root = EmptySubtreeRoots::entry(CREATED_NOTES_SMT_DEPTH, 0); + { + let empty: [Felt; 4] = batch_notes_empty_root.into(); + println!("empty 12: {empty:?}"); + } SimpleSmt::with_leaves(8, std::iter::once((0, batch_notes_empty_root.into()))).unwrap() }; // Compare roots // --------------------------------------------------------------------------------------------- + let batch_notes_empty_root = EmptySubtreeRoots::entry(8, 0); + { + let empty: [Felt; 4] = batch_notes_empty_root.into(); + println!("empty 8: {empty:?}"); + } assert_eq!(block_header.note_root(), notes_smt.root()); } From 7206d68de07b9f241225c7d146ba3bfcb002e02a Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 16 Nov 2023 13:17:03 -0500 Subject: [PATCH 093/166] adjust tests to reflect new tree design i.e. depth 20, insert at 8 --- block-producer/src/batch_builder/mod.rs | 2 +- .../src/block_builder/prover/mod.rs | 6 +-- .../src/block_builder/prover/tests.rs | 37 +++++++------------ 3 files changed, 17 insertions(+), 28 deletions(-) diff --git a/block-producer/src/batch_builder/mod.rs b/block-producer/src/batch_builder/mod.rs index f21a0dd5a..8a129fca9 100644 --- a/block-producer/src/batch_builder/mod.rs +++ b/block-producer/src/batch_builder/mod.rs @@ -14,7 +14,7 @@ pub mod errors; #[cfg(test)] mod tests; -pub const CREATED_NOTES_SMT_DEPTH: u8 = 12; +pub(crate) const CREATED_NOTES_SMT_DEPTH: u8 = 12; const MAX_NUM_CREATED_NOTES_PER_BATCH: usize = 2usize.pow(CREATED_NOTES_SMT_DEPTH as u32); // TRANSACTION BATCH diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 7061f5c40..32e6d72a6 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -21,10 +21,10 @@ use crate::{batch_builder, SharedTxBatch}; use super::{errors::BlockProverError, BuildBlockError}; /// The depth at which we insert roots from the batches. -pub const CREATED_NOTES_TREE_INSERTION_DEPTH: u8 = 8; +pub(crate) const CREATED_NOTES_TREE_INSERTION_DEPTH: u8 = 8; /// The depth of the created notes tree in the block. -pub const CREATED_NOTES_TREE_DEPTH: u8 = +pub(crate) const CREATED_NOTES_TREE_DEPTH: u8 = CREATED_NOTES_TREE_INSERTION_DEPTH + batch_builder::CREATED_NOTES_SMT_DEPTH; #[cfg(test)] @@ -401,7 +401,7 @@ impl BlockWitness { stack_inputs.extend(root_eles); } - let empty_root_depth_8 = EmptySubtreeRoots::entry(8, 0); + let empty_root_depth_8 = EmptySubtreeRoots::entry(CREATED_NOTES_TREE_DEPTH, 0); stack_inputs.extend(*empty_root_depth_8); stack_inputs.push(Felt::from(num_created_notes_roots as u64)); } diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index d213d0cf8..c23a7cdd3 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -10,7 +10,7 @@ use miden_objects::{ use miden_vm::crypto::SimpleSmt; use crate::{ - batch_builder::{TransactionBatch, CREATED_NOTES_SMT_DEPTH}, + batch_builder::TransactionBatch, store::Store, test_utils::{DummyProvenTxGenerator, MockStoreSuccess}, }; @@ -315,7 +315,8 @@ async fn test_compute_account_root_empty_batches() { // NOTE ROOT TESTS // ================================================================================================= -/// Tests that the block kernel returns the empty tree (depth 8) if no notes were created, but which contains at least 1 batch. +/// Tests that the block kernel returns the empty tree (depth 20) if no notes were created, and +/// contains no batches #[tokio::test] async fn test_compute_note_root_empty_batches_success() { // Set up store @@ -339,11 +340,12 @@ async fn test_compute_note_root_empty_batches_success() { // Compare roots // --------------------------------------------------------------------------------------------- - let empty_root_depth_8 = EmptySubtreeRoots::entry(20, 0); - assert_eq!(block_header.note_root(), *empty_root_depth_8); + let created_notes_empty_root = EmptySubtreeRoots::entry(CREATED_NOTES_TREE_DEPTH, 0); + assert_eq!(block_header.note_root(), *created_notes_empty_root); } -/// Tests that the block kernel returns the proper tree if no notes were created, but which contains at least 1 batch. +/// Tests that the block kernel returns the empty tree (depth 20) if no notes were created, but +/// which contains at least 1 batch. #[tokio::test] async fn test_compute_note_root_empty_notes_success() { // Set up store @@ -368,29 +370,14 @@ async fn test_compute_note_root_empty_notes_success() { let block_prover = BlockProver::new(); let block_header = block_prover.prove(block_witness).unwrap(); - // Create SMT by hand to get empty root - // --------------------------------------------------------------------------------------------- - - let notes_smt = { - // Since there's 1 batch (no notes created), the SMT contains 1 leaf with an empty root - let batch_notes_empty_root = EmptySubtreeRoots::entry(CREATED_NOTES_SMT_DEPTH, 0); - { - let empty: [Felt; 4] = batch_notes_empty_root.into(); - println!("empty 12: {empty:?}"); - } - SimpleSmt::with_leaves(8, std::iter::once((0, batch_notes_empty_root.into()))).unwrap() - }; - // Compare roots // --------------------------------------------------------------------------------------------- - let batch_notes_empty_root = EmptySubtreeRoots::entry(8, 0); - { - let empty: [Felt; 4] = batch_notes_empty_root.into(); - println!("empty 8: {empty:?}"); - } - assert_eq!(block_header.note_root(), notes_smt.root()); + let created_notes_empty_root = EmptySubtreeRoots::entry(CREATED_NOTES_TREE_DEPTH, 0); + assert_eq!(block_header.note_root(), *created_notes_empty_root); } +/// Tests that the block kernel returns the expected tree when multiple notes were created across +/// many batches. #[tokio::test] async fn test_compute_note_root_success() { let tx_gen = DummyProvenTxGenerator::new(); @@ -468,4 +455,6 @@ async fn test_compute_note_root_success() { // Compare roots // --------------------------------------------------------------------------------------------- assert_eq!(block_header.note_root(), notes_smt.root()); + + panic!("Rework test. We need to be able to create a SimpleSmt of depth 20 where we insert at depth 8."); } From 779c611470ee701ae9642d70b2f4d856b7b4d6fd Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 16 Nov 2023 13:33:36 -0500 Subject: [PATCH 094/166] only send non-empty created notes roots --- .../src/block_builder/prover/mod.rs | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 32e6d72a6..8bbb92294 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -260,12 +260,15 @@ impl BlockProver { } } +// BLOCK WITNESS +// ================================================================================================= + /// Provides inputs to the `BlockKernel` so that it can generate the new header #[derive(Debug, PartialEq, Eq)] pub(super) struct BlockWitness { updated_accounts: BTreeMap, - /// collection of all batches' created notes SMT roots - batch_created_notes_roots: Vec, + /// (batch_index, created_notes_root) for batches that contain notes + batch_created_notes_roots: Vec<(usize, Digest)>, prev_header: BlockHeader, } @@ -310,8 +313,17 @@ impl BlockWitness { }; // TODO: Validate that there are less than 2^8 roots (and in StateView) - let batch_created_notes_roots = - batches.iter().map(|batch| batch.created_notes_root()).collect(); + let batch_created_notes_roots = batches + .iter() + .enumerate() + .filter_map(|(batch_index, batch)| { + if batch.created_notes().next().is_none() { + None + } else { + Some((batch_index, batch.created_notes_root())) + } + }) + .collect(); Ok(Self { updated_accounts, @@ -395,14 +407,16 @@ impl BlockWitness { // Notes stack inputs { let num_created_notes_roots = self.batch_created_notes_roots.len(); - for batch_created_notes_root in self.batch_created_notes_roots { - let root_eles: [Felt; 4] = batch_created_notes_root.into(); - println!("Adding batch root: {:?}", root_eles); - stack_inputs.extend(root_eles); + for (batch_index, batch_created_notes_root) in self.batch_created_notes_roots { + let batch_index = u64::try_from(batch_index) + .expect("can't be more than 2^64 - 1 notes created"); + stack_inputs.push(Felt::from(batch_index)); + + stack_inputs.extend(batch_created_notes_root); } - let empty_root_depth_8 = EmptySubtreeRoots::entry(CREATED_NOTES_TREE_DEPTH, 0); - stack_inputs.extend(*empty_root_depth_8); + let empty_root = EmptySubtreeRoots::entry(CREATED_NOTES_TREE_DEPTH, 0); + stack_inputs.extend(*empty_root); stack_inputs.push(Felt::from(num_created_notes_roots as u64)); } From 6985a12e3c5c7d7118072652e9b5eca9f75fc99f Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 16 Nov 2023 13:56:07 -0500 Subject: [PATCH 095/166] fix block_kernel masm given previous changes --- .../src/block_builder/prover/mod.rs | 39 +++++++------------ 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 8bbb92294..459767493 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -72,21 +72,18 @@ proc.compute_account_root # => [ROOT_{n-1}] end -#! TODO: put a version of this in stdlib? -#! #! Inserts the specified value under the specified key in a Sparse Merkle Tree of depth 8 defined by the #! specified root. If the insert is successful, the old value located under the specified key #! is returned via the stack. #! #! Inputs: -#! Operand stack: [VALUE, key, ROOT, ...] +#! Operand stack: [ROOT, key, VALUE, ...] #! #! Outputs: #! Operand stack: [OLD_VALUE, NEW_ROOT, ...] proc.mtree_8_set - # prepare the stack for mtree_set operation - movup.4 movdn.8 swapw movup.8 push.8 - # => [8, key, ROOT, VALUE, ...] + movup.4 push.8 + # => [depth=8, key, ROOT, VALUE, ...] mtree_set # => [OLD_VALUE, NEW_ROOT, ...] @@ -94,38 +91,30 @@ end #! Compute the note root #! -#! Stack: [num_notes_updated, SMT_EMPTY_ROOT_DEPTH_8, NOTE_HASH_0, ... , NOTE_HASH_{n-1}] +#! Stack: [num_notes_updated, SMT_EMPTY_ROOT, note_key_0, NOTE_HASH_0, ... , note_key_{n-1}, NOTE_HASH_{n-1}] #! Output: [NOTES_ROOT] proc.compute_note_root # assess if we should loop - push.0 dup.1 dup.1 neq - #=> [should_loop, loop_counter=0, num_notes_updated, SMT_EMPTY_ROOT_DEPTH_8, NOTE_HASH_0, ... , NOTE_HASH_{n-1}] + dup neq.0 + #=> [0 or 1, num_notes_updated, SMT_EMPTY_ROOT, ... ] while.true - #=> [loop_counter, num_notes_updated, ROOT_i, NOTE_HASH_i, ... ] - - # Move loop_counter and `num_notes_updated` down for next iteration - # Keep a copy of `loop_counter`; we use it as the insert key for the ith note hash - dup movdn.10 swap movdn.10 - #=> [loop_counter, ROOT_i, NOTE_HASH_i, loop_counter, num_notes_updated, ... ] + #=> [notes_left_to_update, ROOT_i, note_key_i, NOTE_HASH_i, ... ] - # Prepare stack for mtree_8_set - movdn.4 - #=> [ROOT_i, loop_counter, NOTE_HASH_i, loop_counter, num_notes_updated, ... ] + # Move `notes_left_to_update` down for next iteration + movdn.9 + #=> [ROOT_i, note_key_i, NOTE_HASH_i, notes_left_to_update, ... ] exec.mtree_8_set dropw - #=> [ROOT_{i+1}, loop_counter, num_notes_updated, ... ] + #=> [ROOT_{i+1}, notes_left_to_update, ... ] - # Prepare stack for loop counter check - movdn.5 movdn.5 movdn.5 movdn.5 - #=> [loop_counter, num_notes_updated, ROOT_{i+1}, ... ] # loop counter - add.1 dup.1 dup.1 neq - #=> [should_loop, loop_counter + 1, num_notes_updated, ROOT_{i+1}, ... ] + movup.4 sub.1 dup neq.0 + #=> [0 or 1, notes_left_to_update - 1, ROOT_{i+1}, ... ] end - drop drop + drop # => [ROOT_{n-1}] end From a982a530c781ff530920236780f5ef28d86a27ed Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 16 Nov 2023 14:03:13 -0500 Subject: [PATCH 096/166] fix test_compute_note_root_success --- .../src/block_builder/prover/tests.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index c23a7cdd3..d3f5073b8 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -443,18 +443,23 @@ async fn test_compute_note_root_success() { // Create SMT by hand to get new root // --------------------------------------------------------------------------------------------- + + // The current logic is hardcoded to a depth of 20 + assert_eq!(CREATED_NOTES_TREE_DEPTH, 20); + + // The first 2 txs were put in the first batch; the 3rd was put in the second. It will lie in + // the second subtree of depth 12 let notes_smt = SimpleSmt::with_leaves( - 8, - notes_created - .into_iter() - .enumerate() - .map(|(idx, note)| (idx as u64, note.note_hash().into())), + CREATED_NOTES_TREE_DEPTH, + vec![ + (0u64, notes_created[0].note_hash().into()), + (1u64, notes_created[1].note_hash().into()), + (2u64.pow(12), notes_created[2].note_hash().into()), + ], ) .unwrap(); // Compare roots // --------------------------------------------------------------------------------------------- assert_eq!(block_header.note_root(), notes_smt.root()); - - panic!("Rework test. We need to be able to create a SimpleSmt of depth 20 where we insert at depth 8."); } From 25de72928e6a5dddd498f56e7645f6e2f8dacff1 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 16 Nov 2023 14:04:38 -0500 Subject: [PATCH 097/166] fix `into_parts()` --- block-producer/src/block_builder/prover/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 459767493..615c44562 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -397,11 +397,11 @@ impl BlockWitness { { let num_created_notes_roots = self.batch_created_notes_roots.len(); for (batch_index, batch_created_notes_root) in self.batch_created_notes_roots { + stack_inputs.extend(batch_created_notes_root); + let batch_index = u64::try_from(batch_index) .expect("can't be more than 2^64 - 1 notes created"); stack_inputs.push(Felt::from(batch_index)); - - stack_inputs.extend(batch_created_notes_root); } let empty_root = EmptySubtreeRoots::entry(CREATED_NOTES_TREE_DEPTH, 0); From 5180a2f8c0f7ed7730014b113cea08d6589720bf Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 16 Nov 2023 14:10:41 -0500 Subject: [PATCH 098/166] build_batch: remove `unwrap` --- block-producer/src/batch_builder/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block-producer/src/batch_builder/mod.rs b/block-producer/src/batch_builder/mod.rs index 8a129fca9..0b27c79af 100644 --- a/block-producer/src/batch_builder/mod.rs +++ b/block-producer/src/batch_builder/mod.rs @@ -206,7 +206,7 @@ where &self, txs: Vec, ) -> Result<(), BuildBatchError> { - let batch = Arc::new(TransactionBatch::new(txs).unwrap()); + let batch = Arc::new(TransactionBatch::new(txs)?); self.ready_batches.write().await.push(batch); Ok(()) From afb3568f3dbd1e71f536bb36376c86f6652450ab Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 16 Nov 2023 14:23:04 -0500 Subject: [PATCH 099/166] validate not too many batches in block --- block-producer/src/batch_builder/errors.rs | 2 +- block-producer/src/batch_builder/mod.rs | 2 +- block-producer/src/block_builder/errors.rs | 7 +++++++ block-producer/src/block_builder/prover/mod.rs | 6 +++++- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/block-producer/src/batch_builder/errors.rs b/block-producer/src/batch_builder/errors.rs index f653d40d8..1da3b7e65 100644 --- a/block-producer/src/batch_builder/errors.rs +++ b/block-producer/src/batch_builder/errors.rs @@ -11,7 +11,7 @@ pub enum BuildBatchError { "Too many notes in the batch. Got: {0}, max: {}", MAX_NUM_CREATED_NOTES_PER_BATCH )] - TooManyNotes(usize), + TooManyNotesCreated(usize), #[error("failed to create notes SMT: {0}")] NotesSmtError(#[from] MerkleError), } diff --git a/block-producer/src/batch_builder/mod.rs b/block-producer/src/batch_builder/mod.rs index 0b27c79af..7522aac02 100644 --- a/block-producer/src/batch_builder/mod.rs +++ b/block-producer/src/batch_builder/mod.rs @@ -65,7 +65,7 @@ impl TransactionBatch { .collect(); if created_notes.len() > MAX_NUM_CREATED_NOTES_PER_BATCH { - return Err(BuildBatchError::TooManyNotes(created_notes.len())); + return Err(BuildBatchError::TooManyNotesCreated(created_notes.len())); } SimpleSmt::with_leaves( diff --git a/block-producer/src/block_builder/errors.rs b/block-producer/src/block_builder/errors.rs index 1a682e04f..31b518772 100644 --- a/block-producer/src/block_builder/errors.rs +++ b/block-producer/src/block_builder/errors.rs @@ -4,6 +4,8 @@ use thiserror::Error; use crate::store::{ApplyBlockError, BlockInputsError}; +use super::prover::CREATED_NOTES_TREE_INSERTION_DEPTH; + #[derive(Debug, Error, PartialEq, Eq)] pub enum BuildBlockError { #[error("failed to apply block: {0}")] @@ -16,6 +18,11 @@ pub enum BuildBlockError { InconsistentAccountIds(Vec), #[error("transaction batches and store contain different hashes for some accounts. Offending accounts: {0:?}")] InconsistentAccountStates(Vec), + #[error( + "Too many batches in block. Got: {0}, max: 2^{}", + CREATED_NOTES_TREE_INSERTION_DEPTH + )] + TooManyBatchesInBlock(usize), #[error("dummy")] Dummy, } diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 615c44562..bfc3823db 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -301,7 +301,6 @@ impl BlockWitness { .collect() }; - // TODO: Validate that there are less than 2^8 roots (and in StateView) let batch_created_notes_roots = batches .iter() .enumerate() @@ -328,6 +327,11 @@ impl BlockWitness { // TODO: // - Block height returned for each nullifier is 0. + // Validate that there aren't too many batches in the block. + if batches.len() > 2usize.pow(CREATED_NOTES_TREE_INSERTION_DEPTH.into()) { + return Err(BuildBlockError::TooManyBatchesInBlock(batches.len())); + } + Self::validate_account_states(block_inputs, batches)?; Ok(()) From e871a79164dd4972cdb1937e8b3e6b8e423a385d Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 21 Nov 2023 07:10:31 -0500 Subject: [PATCH 100/166] remove unsafe --- .../src/block_builder/prover/tests.rs | 36 +++++++++---------- block-producer/src/block_builder/tests.rs | 4 +-- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index d3f5073b8..b747e7d01 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -27,9 +27,9 @@ use super::*; #[test] fn test_block_witness_validation_inconsistent_account_ids() { let tx_gen = DummyProvenTxGenerator::new(); - let account_id_1 = unsafe { AccountId::new_unchecked(Felt::ZERO) }; - let account_id_2 = unsafe { AccountId::new_unchecked(Felt::ONE) }; - let account_id_3 = unsafe { AccountId::new_unchecked(Felt::new(42)) }; + let account_id_1 = AccountId::new_unchecked(Felt::ZERO); + let account_id_2 = AccountId::new_unchecked(Felt::ONE); + let account_id_3 = AccountId::new_unchecked(Felt::new(42)); let block_inputs_from_store: BlockInputs = { let block_header = mock_block_header(Felt::ZERO, None, None, &[]); @@ -99,8 +99,8 @@ fn test_block_witness_validation_inconsistent_account_ids() { #[test] fn test_block_witness_validation_inconsistent_account_hashes() { let tx_gen = DummyProvenTxGenerator::new(); - let account_id_1 = unsafe { AccountId::new_unchecked(Felt::ZERO) }; - let account_id_2 = unsafe { AccountId::new_unchecked(Felt::ONE) }; + let account_id_1 = AccountId::new_unchecked(Felt::ZERO); + let account_id_2 = AccountId::new_unchecked(Felt::ONE); let account_1_hash_store = Digest::new([Felt::from(1u64), Felt::from(2u64), Felt::from(3u64), Felt::from(4u64)]); @@ -181,11 +181,11 @@ async fn test_compute_account_root_success() { // Set up account states // --------------------------------------------------------------------------------------------- let account_ids = vec![ - unsafe { AccountId::new_unchecked(Felt::from(0b0000_0000_0000_0000u64)) }, - unsafe { AccountId::new_unchecked(Felt::from(0b1111_0000_0000_0000u64)) }, - unsafe { AccountId::new_unchecked(Felt::from(0b1111_1111_0000_0000u64)) }, - unsafe { AccountId::new_unchecked(Felt::from(0b1111_1111_1111_0000u64)) }, - unsafe { AccountId::new_unchecked(Felt::from(0b1111_1111_1111_1111u64)) }, + AccountId::new_unchecked(Felt::from(0b0000_0000_0000_0000u64)), + AccountId::new_unchecked(Felt::from(0b1111_0000_0000_0000u64)), + AccountId::new_unchecked(Felt::from(0b1111_1111_0000_0000u64)), + AccountId::new_unchecked(Felt::from(0b1111_1111_1111_0000u64)), + AccountId::new_unchecked(Felt::from(0b1111_1111_1111_1111u64)), ]; let account_initial_states = vec![ @@ -269,11 +269,11 @@ async fn test_compute_account_root_empty_batches() { // Set up account states // --------------------------------------------------------------------------------------------- let account_ids = vec![ - unsafe { AccountId::new_unchecked(Felt::from(0b0000_0000_0000_0000u64)) }, - unsafe { AccountId::new_unchecked(Felt::from(0b1111_0000_0000_0000u64)) }, - unsafe { AccountId::new_unchecked(Felt::from(0b1111_1111_0000_0000u64)) }, - unsafe { AccountId::new_unchecked(Felt::from(0b1111_1111_1111_0000u64)) }, - unsafe { AccountId::new_unchecked(Felt::from(0b1111_1111_1111_1111u64)) }, + AccountId::new_unchecked(Felt::from(0b0000_0000_0000_0000u64)), + AccountId::new_unchecked(Felt::from(0b1111_0000_0000_0000u64)), + AccountId::new_unchecked(Felt::from(0b1111_1111_0000_0000u64)), + AccountId::new_unchecked(Felt::from(0b1111_1111_1111_0000u64)), + AccountId::new_unchecked(Felt::from(0b1111_1111_1111_1111u64)), ]; let account_initial_states = vec![ @@ -383,9 +383,9 @@ async fn test_compute_note_root_success() { let tx_gen = DummyProvenTxGenerator::new(); let account_ids = vec![ - unsafe { AccountId::new_unchecked(Felt::from(0u64)) }, - unsafe { AccountId::new_unchecked(Felt::from(1u64)) }, - unsafe { AccountId::new_unchecked(Felt::from(2u64)) }, + AccountId::new_unchecked(Felt::from(0u64)), + AccountId::new_unchecked(Felt::from(1u64)), + AccountId::new_unchecked(Felt::from(2u64)), ]; let notes_created: Vec = vec![ diff --git a/block-producer/src/block_builder/tests.rs b/block-producer/src/block_builder/tests.rs index 007a5c558..1bd42d077 100644 --- a/block-producer/src/block_builder/tests.rs +++ b/block-producer/src/block_builder/tests.rs @@ -15,7 +15,7 @@ use crate::{ #[tokio::test] async fn test_apply_block_called_nonempty_batches() { let tx_gen = DummyProvenTxGenerator::new(); - let account_id = unsafe { AccountId::new_unchecked(42u64.into()) }; + let account_id = AccountId::new_unchecked(42u64.into()); let account_initial_hash: Digest = [Felt::from(1u64), Felt::from(1u64), Felt::from(1u64), Felt::from(1u64)].into(); let store = Arc::new(MockStoreSuccess::new( @@ -49,7 +49,7 @@ async fn test_apply_block_called_nonempty_batches() { /// Tests that `build_block()` succeeds when the transaction batches are empty #[tokio::test] async fn test_apply_block_called_empty_batches() { - let account_id = unsafe { AccountId::new_unchecked(42u64.into()) }; + let account_id = AccountId::new_unchecked(42u64.into()); let account_hash: Digest = [Felt::from(1u64), Felt::from(1u64), Felt::from(1u64), Felt::from(1u64)].into(); let store = Arc::new(MockStoreSuccess::new( From 1ea87e7528ffa289a4ce8ff731b90103b8350f5d Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 21 Nov 2023 07:11:19 -0500 Subject: [PATCH 101/166] remove capital letter --- block-producer/src/block_builder/errors.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block-producer/src/block_builder/errors.rs b/block-producer/src/block_builder/errors.rs index 31b518772..f63af9217 100644 --- a/block-producer/src/block_builder/errors.rs +++ b/block-producer/src/block_builder/errors.rs @@ -19,7 +19,7 @@ pub enum BuildBlockError { #[error("transaction batches and store contain different hashes for some accounts. Offending accounts: {0:?}")] InconsistentAccountStates(Vec), #[error( - "Too many batches in block. Got: {0}, max: 2^{}", + "too many batches in block. Got: {0}, max: 2^{}", CREATED_NOTES_TREE_INSERTION_DEPTH )] TooManyBatchesInBlock(usize), From f3b56044117abde8c744872a1eb503a814609526 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 21 Nov 2023 07:20:22 -0500 Subject: [PATCH 102/166] remove script roots from transaction batch --- block-producer/Cargo.toml | 1 - block-producer/src/batch_builder/mod.rs | 20 -------------------- 2 files changed, 21 deletions(-) diff --git a/block-producer/Cargo.toml b/block-producer/Cargo.toml index 14e14b2b7..10ba5b60f 100644 --- a/block-producer/Cargo.toml +++ b/block-producer/Cargo.toml @@ -15,7 +15,6 @@ winterfell = "0.7" [dependencies] async-trait = "0.1" -itertools = "0.11" miden-air = { package = "miden-air", git = "https://github.com/0xPolygonMiden/miden-vm.git", branch = "next", default-features = false } miden-node-proto = { path = "../proto", version = "0.1"} miden_objects = { workspace = true } diff --git a/block-producer/src/batch_builder/mod.rs b/block-producer/src/batch_builder/mod.rs index 7522aac02..f8790b86e 100644 --- a/block-producer/src/batch_builder/mod.rs +++ b/block-producer/src/batch_builder/mod.rs @@ -1,7 +1,6 @@ use std::{cmp::min, sync::Arc, time::Duration}; use async_trait::async_trait; -use itertools::Itertools; use miden_objects::{accounts::AccountId, Digest}; use miden_vm::crypto::SimpleSmt; use tokio::{sync::RwLock, time}; @@ -27,7 +26,6 @@ const MAX_NUM_CREATED_NOTES_PER_BATCH: usize = 2usize.pow(CREATED_NOTES_SMT_DEPT pub struct TransactionBatch { account_initial_states: Vec<(AccountId, Digest)>, updated_accounts: Vec<(AccountId, Digest)>, - consumed_notes_script_roots: Vec, produced_nullifiers: Vec, created_notes_smt: SimpleSmt, } @@ -39,18 +37,6 @@ impl TransactionBatch { let updated_accounts = txs.iter().map(|tx| (tx.account_id(), tx.final_account_hash())).collect(); - let consumed_notes_script_roots = { - let mut script_roots: Vec = txs - .iter() - .flat_map(|tx| tx.consumed_notes()) - .map(|consumed_note| consumed_note.script_root()) - .collect(); - - script_roots.sort(); - - // Removes duplicates in consecutive items - script_roots.into_iter().dedup().collect() - }; let produced_nullifiers = txs .iter() .flat_map(|tx| tx.consumed_notes()) @@ -82,7 +68,6 @@ impl TransactionBatch { Ok(Self { account_initial_states, updated_accounts, - consumed_notes_script_roots, produced_nullifiers, created_notes_smt, }) @@ -100,11 +85,6 @@ impl TransactionBatch { self.updated_accounts.iter().cloned() } - /// Returns the script root of all consumed notes - pub fn consumed_notes_script_roots(&self) -> impl Iterator + '_ { - self.consumed_notes_script_roots.iter().cloned() - } - /// Returns the nullifier of all consumed notes pub fn produced_nullifiers(&self) -> impl Iterator + '_ { self.produced_nullifiers.iter().cloned() From 7bbef20852d633e84574e1f34f3e04d040580978 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 21 Nov 2023 07:40:05 -0500 Subject: [PATCH 103/166] Use BTreeMap for account updates in TransactionBatch --- block-producer/src/batch_builder/mod.rs | 37 ++++++++++++++++++------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/block-producer/src/batch_builder/mod.rs b/block-producer/src/batch_builder/mod.rs index f8790b86e..c58a42369 100644 --- a/block-producer/src/batch_builder/mod.rs +++ b/block-producer/src/batch_builder/mod.rs @@ -1,4 +1,4 @@ -use std::{cmp::min, sync::Arc, time::Duration}; +use std::{cmp::min, collections::BTreeMap, sync::Arc, time::Duration}; use async_trait::async_trait; use miden_objects::{accounts::AccountId, Digest}; @@ -24,18 +24,25 @@ const MAX_NUM_CREATED_NOTES_PER_BATCH: usize = 2usize.pow(CREATED_NOTES_SMT_DEPT /// /// Note: Until recursive proofs are available in the Miden VM, we don't include the common proof. pub struct TransactionBatch { - account_initial_states: Vec<(AccountId, Digest)>, - updated_accounts: Vec<(AccountId, Digest)>, + updated_accounts: BTreeMap, produced_nullifiers: Vec, created_notes_smt: SimpleSmt, } impl TransactionBatch { pub fn new(txs: Vec) -> Result { - let account_initial_states = - txs.iter().map(|tx| (tx.account_id(), tx.initial_account_hash())).collect(); - let updated_accounts = - txs.iter().map(|tx| (tx.account_id(), tx.final_account_hash())).collect(); + let updated_accounts = txs + .iter() + .map(|tx| { + ( + tx.account_id(), + AccountStates { + initial_state: tx.initial_account_hash(), + final_state: tx.final_account_hash(), + }, + ) + }) + .collect(); let produced_nullifiers = txs .iter() @@ -66,7 +73,6 @@ impl TransactionBatch { }; Ok(Self { - account_initial_states, updated_accounts, produced_nullifiers, created_notes_smt, @@ -76,13 +82,17 @@ impl TransactionBatch { /// Returns an iterator over account ids that were modified in the transaction batch, and their /// corresponding initial hash pub fn account_initial_states(&self) -> impl Iterator + '_ { - self.account_initial_states.iter().cloned() + self.updated_accounts + .iter() + .map(|(account_id, account_states)| (*account_id, account_states.initial_state)) } /// Returns an iterator over account ids that were modified in the transaction batch, and their /// corresponding new hash pub fn updated_accounts(&self) -> impl Iterator + '_ { - self.updated_accounts.iter().cloned() + self.updated_accounts + .iter() + .map(|(account_id, account_states)| (*account_id, account_states.final_state)) } /// Returns the nullifier of all consumed notes @@ -101,6 +111,13 @@ impl TransactionBatch { } } +/// Stores the initial state (before the transaction) and final state (after the transaction) of an +/// account +struct AccountStates { + initial_state: Digest, + final_state: Digest, +} + // BATCH BUILDER // ================================================================================================ From b7d2e09a7d1f44f87e875a80458d05ef0bdb3a10 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 22 Nov 2023 11:32:37 -0500 Subject: [PATCH 104/166] inline mtree_8_set function --- .../src/block_builder/prover/mod.rs | 25 +++---------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index bfc3823db..021072af0 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -72,23 +72,6 @@ proc.compute_account_root # => [ROOT_{n-1}] end -#! Inserts the specified value under the specified key in a Sparse Merkle Tree of depth 8 defined by the -#! specified root. If the insert is successful, the old value located under the specified key -#! is returned via the stack. -#! -#! Inputs: -#! Operand stack: [ROOT, key, VALUE, ...] -#! -#! Outputs: -#! Operand stack: [OLD_VALUE, NEW_ROOT, ...] -proc.mtree_8_set - movup.4 push.8 - # => [depth=8, key, ROOT, VALUE, ...] - - mtree_set - # => [OLD_VALUE, NEW_ROOT, ...] -end - #! Compute the note root #! #! Stack: [num_notes_updated, SMT_EMPTY_ROOT, note_key_0, NOTE_HASH_0, ... , note_key_{n-1}, NOTE_HASH_{n-1}] @@ -101,11 +84,11 @@ proc.compute_note_root while.true #=> [notes_left_to_update, ROOT_i, note_key_i, NOTE_HASH_i, ... ] - # Move `notes_left_to_update` down for next iteration - movdn.9 - #=> [ROOT_i, note_key_i, NOTE_HASH_i, notes_left_to_update, ... ] + # Prepare stack for mtree_set + movdn.9 movup.4 push.8 + #=> [depth=8, note_key_i, ROOT_i, NOTE_HASH_i, notes_left_to_update, ... ] - exec.mtree_8_set dropw + mtree_set dropw #=> [ROOT_{i+1}, notes_left_to_update, ... ] From 2d37e71775d18089f744ded07bd36cced446b7b4 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 22 Nov 2023 11:34:34 -0500 Subject: [PATCH 105/166] remove whitespace --- block-producer/src/block_builder/prover/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 021072af0..63e3842c0 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -91,7 +91,6 @@ proc.compute_note_root mtree_set dropw #=> [ROOT_{i+1}, notes_left_to_update, ... ] - # loop counter movup.4 sub.1 dup neq.0 #=> [0 or 1, notes_left_to_update - 1, ROOT_{i+1}, ... ] From 1206d301640b61355bd52b1ad172c6026160dd37 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 22 Nov 2023 16:17:34 -0500 Subject: [PATCH 106/166] compute_chain_mmr_root routine --- .../src/block_builder/prover/mod.rs | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 63e3842c0..e10b8187b 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -37,6 +37,9 @@ mod tests; /// NEW_ACCOUNT_HASH_n, account_id_n] const BLOCK_KERNEL_MASM: &str = " use.std::collections::smt64 +use.std::collections::mmr + +const.CHAIN_MMR_PTR=1000 #! Compute the account root #! @@ -100,6 +103,48 @@ proc.compute_note_root # => [ROOT_{n-1}] end +#! Compute the chain MMR root +#! +#! Stack: [NUM_LEAVES, num_peaks, PEAK_0, ..., PEAK_{n-1}, PREV_BLOCK_HASH_TO_INSERT] +#! Output: [CHAIN_MMR_ROOT] +proc.compute_chain_mmr_root + # initialize memory: num leaves + push.CHAIN_MMR_PTR mem_storew dropw + # => [num_peaks, PEAK_0, ..., PEAK_{n-1}, PREV_BLOCK_HASH_TO_INSERT] + + # prepare stack for loop that stores peaks + push.CHAIN_MMR_PTR add.1 dup add + # => [write_ptr, write_ptr_end, PEAK_0, ..., PEAK_{n-1}, PREV_BLOCK_HASH_TO_INSERT] + + dup.1 dup.1 neq + while.true + # => [write_ptr, write_ptr_end, PEAK_i, ..., PEAK_{n-1}, PREV_BLOCK_HASH_TO_INSERT] + + movup.5 movup.5 movup.5 movup.5 dup.4 + # => [write_ptr, PEAK_i, write_ptr, write_ptr_end, PEAK_{i+1}, ..., PEAK_{n-1}, PREV_BLOCK_HASH_TO_INSERT] + + mem_storew dropw + # => [write_ptr, write_ptr_end, PEAK_{i+1}, ..., PEAK_{n-1}, PREV_BLOCK_HASH_TO_INSERT] + + # check if done looping + dup.1 dup.1 neq + # => [0 or 1, write_ptr, write_ptr_end, PEAK_{i+1}, ..., PEAK_{n-1}, PREV_BLOCK_HASH_TO_INSERT] + end + # => [ PREV_BLOCK_HASH_TO_INSERT ] + + # prepare stack to add new peak + push.CHAIN_MMR_PTR movdn.4 + # => [ PREV_BLOCK_HASH_TO_INSERT, chain_mmr_ptr ] + + # add new peak + exec.mmr::add + # => [ ] + + # Compute new MMR root + push.CHAIN_MMR_PTR exec.mmr::pack + # => [ CHAIN_MMR_ROOT ] +end + # Stack: [, ] proc.main.2 exec.compute_account_root loc_storew.0 dropw From 8d03172b4ba1dfe98c60c9369c681150c593777d Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 22 Nov 2023 16:20:33 -0500 Subject: [PATCH 107/166] call chain mmr root routine from main --- block-producer/src/block_builder/prover/mod.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index e10b8187b..851fbd536 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -145,17 +145,20 @@ proc.compute_chain_mmr_root # => [ CHAIN_MMR_ROOT ] end -# Stack: [, ] -proc.main.2 +# Stack: [, , ] +proc.main.3 exec.compute_account_root loc_storew.0 dropw - #=> [] + # => [, ] exec.compute_note_root loc_storew.1 dropw - #=> [ ] + # => [ ] + + exec.compute_chain_mmr_root loc_storew.2 dropw + # => [ ] # Load output on stack - loc_loadw.1 padw loc_loadw.0 - #=> [ ACCOUNT_ROOT, NOTE_ROOT] + loc_loadw.2 padw loc_loadw.1 padw loc_loadw.0 + #=> [ ACCOUNT_ROOT, NOTE_ROOT, CHAIN_MMR_ROOT ] end begin From 6b351133bb3424e8ca2df98ca3fe793447080c1c Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Mon, 27 Nov 2023 10:03:07 -0500 Subject: [PATCH 108/166] use new StackOutputs API --- block-producer/src/block_builder/errors.rs | 4 +- .../src/block_builder/prover/mod.rs | 51 ++++++------------- 2 files changed, 17 insertions(+), 38 deletions(-) diff --git a/block-producer/src/block_builder/errors.rs b/block-producer/src/block_builder/errors.rs index 3e6dc8739..93a70f496 100644 --- a/block-producer/src/block_builder/errors.rs +++ b/block-producer/src/block_builder/errors.rs @@ -32,6 +32,6 @@ pub enum BlockProverError { InvalidMerklePaths(MerkleError), #[error("program execution failed")] ProgramExecutionFailed(ExecutionError), - #[error("invalid return value on stack (not a hash)")] - InvalidRootReturned, + #[error("failed to retrieve {0} root from stack outputs")] + InvalidRootOutput(String), } diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 63e3842c0..2c47fbe10 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -20,6 +20,12 @@ use crate::{batch_builder, SharedTxBatch}; use super::{errors::BlockProverError, BuildBlockError}; +/// The index of the word at which the account root is stored on the output stack. +pub const ACCOUNT_ROOT_WORD_IDX: usize = 0; + +/// The index of the word at which the note root is stored on the output stack. +pub const NOTE_ROOT_WORD_IDX: usize = 4; + /// The depth at which we insert roots from the batches. pub(crate) const CREATED_NOTES_TREE_INSERTION_DEPTH: u8 = 8; @@ -190,44 +196,17 @@ impl BlockProver { execute(&self.kernel, stack_inputs, host, ExecutionOptions::default()) .map_err(BlockProverError::ProgramExecutionFailed)?; - // TODO: Use `StackOutputs::pop_digest()` once merged - let (account_root_output, note_root_output) = { - let root_outputs: Vec<_> = execution_output.stack_outputs().stack().chunks(4).collect(); - - (root_outputs[0], root_outputs[1]) - }; - - let new_account_root = { - let digest_elements: Vec = account_root_output - .iter() - .cloned() - .map(Felt::from) - // We reverse, since a word `[a, b, c, d]` will be stored on the stack as `[d, c, b, a]` - .rev() - .collect(); - - let digest_elements: [Felt; 4] = - digest_elements.try_into().map_err(|_| BlockProverError::InvalidRootReturned)?; + let new_account_root = execution_output + .stack_outputs() + .get_stack_word(ACCOUNT_ROOT_WORD_IDX) + .ok_or(BlockProverError::InvalidRootOutput("account".to_string()))?; - digest_elements.into() - }; - - let new_note_root = { - let digest_elements: Vec = note_root_output - .iter() - .cloned() - .map(Felt::from) - // We reverse, since a word `[a, b, c, d]` will be stored on the stack as `[d, c, b, a]` - .rev() - .collect(); - - let digest_elements: [Felt; 4] = - digest_elements.try_into().map_err(|_| BlockProverError::InvalidRootReturned)?; - - digest_elements.into() - }; + let new_note_root = execution_output + .stack_outputs() + .get_stack_word(NOTE_ROOT_WORD_IDX) + .ok_or(BlockProverError::InvalidRootOutput("note".to_string()))?; - Ok((new_account_root, new_note_root)) + Ok((new_account_root.into(), new_note_root.into())) } } From 25f933ff0c12c9f2977b5974613c8eebc7aee82c Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Mon, 27 Nov 2023 10:45:50 -0500 Subject: [PATCH 109/166] add explanation to `compute_note_root` --- block-producer/src/block_builder/prover/mod.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 2c47fbe10..9abd813eb 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -78,7 +78,14 @@ proc.compute_account_root # => [ROOT_{n-1}] end -#! Compute the note root +#! Compute the note root. +#! +#! Each batch contains a tree of depth 12 for its created notes. The block's created notes tree is created +#! by aggregating up to 2^8 tree roots coming from the batches contained in the block. +#! +#! `SMT_EMPTY_ROOT` must be `E20`, the root of the empty tree of depth 20. If less than 2^8 batches are +#! contained in the block, `E12` is used as the padding value; this is derived from the fact that +#! `SMT_EMPTY_ROOT` is `E20`, and that our tree has depth 8. #! #! Stack: [num_notes_updated, SMT_EMPTY_ROOT, note_key_0, NOTE_HASH_0, ... , note_key_{n-1}, NOTE_HASH_{n-1}] #! Output: [NOTES_ROOT] From 71bf52d993b03e1981b6135b1fc731dabec844cf Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 28 Nov 2023 10:27:13 -0500 Subject: [PATCH 110/166] convert `as u64` to try_into --- block-producer/src/block_builder/prover/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 9abd813eb..e1812b802 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -378,7 +378,10 @@ impl BlockWitness { let empty_root = EmptySubtreeRoots::entry(CREATED_NOTES_TREE_DEPTH, 0); stack_inputs.extend(*empty_root); - stack_inputs.push(Felt::from(num_created_notes_roots as u64)); + stack_inputs.push(Felt::from( + u64::try_from(num_created_notes_roots) + .expect("can't be more than 2^64 - 1 notes created"), + )); } // Account stack inputs From d01f13a537b24f4bb79c86441ceba627921f5030 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 28 Nov 2023 10:38:59 -0500 Subject: [PATCH 111/166] fix created notes smt: depth 13 instead of 12 --- block-producer/src/batch_builder/mod.rs | 31 +++++++++++++------------ 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/block-producer/src/batch_builder/mod.rs b/block-producer/src/batch_builder/mod.rs index c58a42369..5af7ca2db 100644 --- a/block-producer/src/batch_builder/mod.rs +++ b/block-producer/src/batch_builder/mod.rs @@ -1,7 +1,7 @@ use std::{cmp::min, collections::BTreeMap, sync::Arc, time::Duration}; use async_trait::async_trait; -use miden_objects::{accounts::AccountId, Digest}; +use miden_objects::{accounts::AccountId, notes::NoteEnvelope, Digest}; use miden_vm::crypto::SimpleSmt; use tokio::{sync::RwLock, time}; @@ -13,8 +13,12 @@ pub mod errors; #[cfg(test)] mod tests; -pub(crate) const CREATED_NOTES_SMT_DEPTH: u8 = 12; -const MAX_NUM_CREATED_NOTES_PER_BATCH: usize = 2usize.pow(CREATED_NOTES_SMT_DEPTH as u32); +pub(crate) const CREATED_NOTES_SMT_DEPTH: u8 = 13; + +/// The created notes tree uses an extra depth to store the 2 components of `NoteEnvelope`. +/// That is, conceptually, notes sit at depth 12; where in reality, depth 12 contains the +/// hash of level 13, where both the `note_hash()` and metadata are stored (one per node). +const MAX_NUM_CREATED_NOTES_PER_BATCH: usize = 2usize.pow((CREATED_NOTES_SMT_DEPTH - 1) as u32); // TRANSACTION BATCH // ================================================================================================ @@ -51,24 +55,21 @@ impl TransactionBatch { .collect(); let created_notes_smt = { - let created_notes: Vec<_> = txs - .iter() - .flat_map(|tx| tx.created_notes()) - .map(|note_envelope| note_envelope.note_hash()) - .collect(); + let created_notes: Vec<&NoteEnvelope> = + txs.iter().flat_map(|tx| tx.created_notes()).collect(); if created_notes.len() > MAX_NUM_CREATED_NOTES_PER_BATCH { return Err(BuildBatchError::TooManyNotesCreated(created_notes.len())); } - SimpleSmt::with_leaves( + SimpleSmt::with_contiguous_leaves( CREATED_NOTES_SMT_DEPTH, - created_notes.into_iter().enumerate().map(|(idx, note_hash)| { - ( - idx.try_into().expect("already checked not for too many notes"), - note_hash.into(), - ) - }), + created_notes + .into_iter() + .flat_map(|note_envelope| { + [note_envelope.note_hash().into(), note_envelope.metadata().into()] + }) + .collect::>(), )? }; From 959d8831793ed8a4e1e7b9871664d1db2d91e2a8 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 28 Nov 2023 10:47:07 -0500 Subject: [PATCH 112/166] fix test --- block-producer/src/block_builder/prover/tests.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index b747e7d01..0da38bd4a 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -444,8 +444,10 @@ async fn test_compute_note_root_success() { // Create SMT by hand to get new root // --------------------------------------------------------------------------------------------- - // The current logic is hardcoded to a depth of 20 - assert_eq!(CREATED_NOTES_TREE_DEPTH, 20); + // The current logic is hardcoded to a depth of 21 + // Specifically, we assume the block has up to 2^8 batches, and each batch up to 2^12 created notes, + // where each note is stored at depth 13 in the batch as 2 contiguous nodes: note hash, then metadata. + assert_eq!(CREATED_NOTES_TREE_DEPTH, 21); // The first 2 txs were put in the first batch; the 3rd was put in the second. It will lie in // the second subtree of depth 12 @@ -453,8 +455,11 @@ async fn test_compute_note_root_success() { CREATED_NOTES_TREE_DEPTH, vec![ (0u64, notes_created[0].note_hash().into()), - (1u64, notes_created[1].note_hash().into()), - (2u64.pow(12), notes_created[2].note_hash().into()), + (1u64, notes_created[0].metadata().into()), + (2u64, notes_created[1].note_hash().into()), + (3u64, notes_created[1].metadata().into()), + (2u64.pow(13), notes_created[2].note_hash().into()), + (2u64.pow(13) + 1, notes_created[2].metadata().into()), ], ) .unwrap(); From d4b87934bbd436127e6b941dbef91d8bccf1cce2 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 28 Nov 2023 10:55:15 -0500 Subject: [PATCH 113/166] clippy tests --- block-producer/src/batch_builder/tests/mod.rs | 7 ++---- .../src/block_builder/prover/tests.rs | 22 +++++++++---------- block-producer/src/block_builder/tests.rs | 2 +- .../src/state_view/tests/apply_block.rs | 4 ++-- block-producer/src/state_view/tests/mod.rs | 6 ++--- block-producer/src/test_utils/proven_tx.rs | 9 +++++++- block-producer/src/txqueue/tests/mod.rs | 2 +- 7 files changed, 28 insertions(+), 24 deletions(-) diff --git a/block-producer/src/batch_builder/tests/mod.rs b/block-producer/src/batch_builder/tests/mod.rs index 93b757083..0dab548b0 100644 --- a/block-producer/src/batch_builder/tests/mod.rs +++ b/block-producer/src/batch_builder/tests/mod.rs @@ -122,7 +122,7 @@ async fn test_batches_added_back_to_queue_on_block_build_failure() { let block_frequency = Duration::from_millis(20); let max_batches_per_block = 2; - let block_builder = Arc::new(BlockBuilderFailure::default()); + let block_builder = Arc::new(BlockBuilderFailure); let batch_builder = DefaultBatchBuilder::new( block_builder.clone(), @@ -164,10 +164,7 @@ fn dummy_tx_batch( tx_gen: &DummyProvenTxGenerator, num_txs_in_batch: usize, ) -> SharedTxBatch { - let txs: Vec<_> = (0..num_txs_in_batch) - .into_iter() - .map(|_| Arc::new(tx_gen.dummy_proven_tx())) - .collect(); + let txs: Vec<_> = (0..num_txs_in_batch).map(|_| Arc::new(tx_gen.dummy_proven_tx())).collect(); Arc::new(TransactionBatch::new(txs).unwrap()) } diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index 0da38bd4a..506e62bf5 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -180,7 +180,7 @@ async fn test_compute_account_root_success() { // Set up account states // --------------------------------------------------------------------------------------------- - let account_ids = vec![ + let account_ids = [ AccountId::new_unchecked(Felt::from(0b0000_0000_0000_0000u64)), AccountId::new_unchecked(Felt::from(0b1111_0000_0000_0000u64)), AccountId::new_unchecked(Felt::from(0b1111_1111_0000_0000u64)), @@ -188,7 +188,7 @@ async fn test_compute_account_root_success() { AccountId::new_unchecked(Felt::from(0b1111_1111_1111_1111u64)), ]; - let account_initial_states = vec![ + let account_initial_states = [ [Felt::from(1u64), Felt::from(1u64), Felt::from(1u64), Felt::from(1u64)], [Felt::from(2u64), Felt::from(2u64), Felt::from(2u64), Felt::from(2u64)], [Felt::from(3u64), Felt::from(3u64), Felt::from(3u64), Felt::from(3u64)], @@ -196,7 +196,7 @@ async fn test_compute_account_root_success() { [Felt::from(5u64), Felt::from(5u64), Felt::from(5u64), Felt::from(5u64)], ]; - let account_final_states = vec![ + let account_final_states = [ [Felt::from(2u64), Felt::from(2u64), Felt::from(2u64), Felt::from(2u64)], [Felt::from(3u64), Felt::from(3u64), Felt::from(3u64), Felt::from(3u64)], [Felt::from(4u64), Felt::from(4u64), Felt::from(4u64), Felt::from(4u64)], @@ -211,7 +211,7 @@ async fn test_compute_account_root_success() { account_ids .iter() .zip(account_initial_states.iter()) - .map(|(&account_id, &account_hash)| (account_id.into(), account_hash.into())), + .map(|(&account_id, &account_hash)| (account_id, account_hash.into())), BTreeSet::new(), ); // Block prover @@ -254,7 +254,7 @@ async fn test_compute_account_root_success() { account_ids .iter() .zip(account_final_states.iter()) - .map(|(&account_id, &account_hash)| (account_id.into(), account_hash.into())), + .map(|(&account_id, &account_hash)| (account_id, account_hash.into())), ) .await; @@ -268,7 +268,7 @@ async fn test_compute_account_root_success() { async fn test_compute_account_root_empty_batches() { // Set up account states // --------------------------------------------------------------------------------------------- - let account_ids = vec![ + let account_ids = [ AccountId::new_unchecked(Felt::from(0b0000_0000_0000_0000u64)), AccountId::new_unchecked(Felt::from(0b1111_0000_0000_0000u64)), AccountId::new_unchecked(Felt::from(0b1111_1111_0000_0000u64)), @@ -276,7 +276,7 @@ async fn test_compute_account_root_empty_batches() { AccountId::new_unchecked(Felt::from(0b1111_1111_1111_1111u64)), ]; - let account_initial_states = vec![ + let account_initial_states = [ [Felt::from(1u64), Felt::from(1u64), Felt::from(1u64), Felt::from(1u64)], [Felt::from(2u64), Felt::from(2u64), Felt::from(2u64), Felt::from(2u64)], [Felt::from(3u64), Felt::from(3u64), Felt::from(3u64), Felt::from(3u64)], @@ -291,7 +291,7 @@ async fn test_compute_account_root_empty_batches() { account_ids .iter() .zip(account_initial_states.iter()) - .map(|(&account_id, &account_hash)| (account_id.into(), account_hash.into())), + .map(|(&account_id, &account_hash)| (account_id, account_hash.into())), BTreeSet::new(), ); // Block prover @@ -382,13 +382,13 @@ async fn test_compute_note_root_empty_notes_success() { async fn test_compute_note_root_success() { let tx_gen = DummyProvenTxGenerator::new(); - let account_ids = vec![ + let account_ids = [ AccountId::new_unchecked(Felt::from(0u64)), AccountId::new_unchecked(Felt::from(1u64)), AccountId::new_unchecked(Felt::from(2u64)), ]; - let notes_created: Vec = vec![ + let notes_created: Vec = [ Digest::from([Felt::from(1u64), Felt::from(1u64), Felt::from(1u64), Felt::from(1u64)]), Digest::from([Felt::from(2u64), Felt::from(2u64), Felt::from(2u64), Felt::from(2u64)]), Digest::from([Felt::from(3u64), Felt::from(3u64), Felt::from(3u64), Felt::from(3u64)]), @@ -425,7 +425,7 @@ async fn test_compute_note_root_success() { Digest::default(), Digest::default(), Vec::new(), - vec![note.clone()], + vec![*note], )) }) .collect(); diff --git a/block-producer/src/block_builder/tests.rs b/block-producer/src/block_builder/tests.rs index 1bd42d077..c21860d94 100644 --- a/block-producer/src/block_builder/tests.rs +++ b/block-producer/src/block_builder/tests.rs @@ -68,7 +68,7 @@ async fn test_apply_block_called_empty_batches() { /// Tests that `build_block()` fails when `get_block_inputs()` fails #[tokio::test] async fn test_build_block_failure() { - let store = Arc::new(MockStoreFailure::default()); + let store = Arc::new(MockStoreFailure); let block_builder = DefaultBlockBuilder::new(store.clone()); diff --git a/block-producer/src/state_view/tests/apply_block.rs b/block-producer/src/state_view/tests/apply_block.rs index fe927b761..b39ee8180 100644 --- a/block-producer/src/state_view/tests/apply_block.rs +++ b/block-producer/src/state_view/tests/apply_block.rs @@ -62,7 +62,7 @@ async fn test_apply_block_ab2() { // Verify transactions so it can be tracked in state view for tx in txs { - let verify_tx_res = state_view.verify_tx(tx.into()).await; + let verify_tx_res = state_view.verify_tx(tx).await; assert!(verify_tx_res.is_ok()); } @@ -123,7 +123,7 @@ async fn test_apply_block_ab3() { assert_eq!( verify_tx_res, Err(VerifyTxError::ConsumedNotesAlreadyConsumed( - txs[0].consumed_notes().into_iter().map(|note| note.nullifier()).collect() + txs[0].consumed_notes().iter().map(|note| note.nullifier()).collect() )) ); } diff --git a/block-producer/src/state_view/tests/mod.rs b/block-producer/src/state_view/tests/mod.rs index 5874b792b..6ecda9fae 100644 --- a/block-producer/src/state_view/tests/mod.rs +++ b/block-producer/src/state_view/tests/mod.rs @@ -16,10 +16,10 @@ pub fn consumed_note_by_index(index: u8) -> ConsumedNoteInfo { /// Returns `num` transactions, and the corresponding account they modify. /// The transactions each consume a single different note -pub fn get_txs_and_accounts<'a>( - tx_gen: &'a DummyProvenTxGenerator, +pub fn get_txs_and_accounts( + tx_gen: &DummyProvenTxGenerator, num: u8, -) -> impl Iterator + 'a { +) -> impl Iterator + '_ { (0..num).map(|index| { let account = MockPrivateAccount::from(index); let tx = tx_gen.dummy_proven_tx_with_params( diff --git a/block-producer/src/test_utils/proven_tx.rs b/block-producer/src/test_utils/proven_tx.rs index 9ffad77d3..b4c672745 100644 --- a/block-producer/src/test_utils/proven_tx.rs +++ b/block-producer/src/test_utils/proven_tx.rs @@ -1,3 +1,5 @@ +//! FibSmall taken from the `fib_small` example in `winterfell` + use miden_air::{ExecutionProof, HashFunction}; use miden_mock::constants::ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_ON_CHAIN; use miden_objects::{ @@ -6,7 +8,6 @@ use miden_objects::{ transaction::{ConsumedNoteInfo, ProvenTransaction}, Digest, }; -///! FibSmall taken from the `fib_small` example in `winterfell` use winterfell::{ crypto::{hashers::Blake3_192, DefaultRandomCoin}, math::fields::f64::BaseElement, @@ -67,6 +68,12 @@ impl DummyProvenTxGenerator { } } +impl Default for DummyProvenTxGenerator { + fn default() -> Self { + Self::new() + } +} + const TRACE_WIDTH: usize = 2; pub fn are_equal( diff --git a/block-producer/src/txqueue/tests/mod.rs b/block-producer/src/txqueue/tests/mod.rs index d734ed151..34931c43e 100644 --- a/block-producer/src/txqueue/tests/mod.rs +++ b/block-producer/src/txqueue/tests/mod.rs @@ -152,7 +152,7 @@ async fn test_build_batch_failure() { let build_batch_frequency = Duration::from_millis(30); let batch_size = 3; - let batch_builder = Arc::new(BatchBuilderFailure::default()); + let batch_builder = Arc::new(BatchBuilderFailure); let tx_queue = DefaultTransactionQueue::new( Arc::new(TransactionVerifierSuccess), From 980f921c74d798e22843bc459082e3c09e7ca344 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 28 Nov 2023 11:04:53 -0500 Subject: [PATCH 114/166] fix masm comments --- .../src/block_builder/prover/mod.rs | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index e1812b802..3aa605f83 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -80,14 +80,17 @@ end #! Compute the note root. #! -#! Each batch contains a tree of depth 12 for its created notes. The block's created notes tree is created +#! Each batch contains a tree of depth 13 for its created notes. The block's created notes tree is created #! by aggregating up to 2^8 tree roots coming from the batches contained in the block. #! -#! `SMT_EMPTY_ROOT` must be `E20`, the root of the empty tree of depth 20. If less than 2^8 batches are -#! contained in the block, `E12` is used as the padding value; this is derived from the fact that -#! `SMT_EMPTY_ROOT` is `E20`, and that our tree has depth 8. +#! `SMT_EMPTY_ROOT` must be `E21`, the root of the empty tree of depth 21. If less than 2^8 batches are +#! contained in the block, `E13` is used as the padding value; this is derived from the fact that +#! `SMT_EMPTY_ROOT` is `E21`, and that our tree has depth 8. #! -#! Stack: [num_notes_updated, SMT_EMPTY_ROOT, note_key_0, NOTE_HASH_0, ... , note_key_{n-1}, NOTE_HASH_{n-1}] +#! Stack: [num_notes_updated, SMT_EMPTY_ROOT, +#! batch_note_root_idx_0, BATCH_NOTE_TREE_ROOT_0, +#! ... , +#! batch_note_root_idx_{n-1}, BATCH_NOTE_TREE_ROOT_{n-1}] #! Output: [NOTES_ROOT] proc.compute_note_root # assess if we should loop @@ -95,18 +98,18 @@ proc.compute_note_root #=> [0 or 1, num_notes_updated, SMT_EMPTY_ROOT, ... ] while.true - #=> [notes_left_to_update, ROOT_i, note_key_i, NOTE_HASH_i, ... ] + #=> [note_roots_left_to_update, ROOT_i, batch_note_root_idx_i, BATCH_NOTE_TREE_ROOT_i, ... ] # Prepare stack for mtree_set movdn.9 movup.4 push.8 - #=> [depth=8, note_key_i, ROOT_i, NOTE_HASH_i, notes_left_to_update, ... ] + #=> [depth=8, batch_note_root_idx_i, ROOT_i, BATCH_NOTE_TREE_ROOT_i, note_roots_left_to_update, ... ] mtree_set dropw - #=> [ROOT_{i+1}, notes_left_to_update, ... ] + #=> [ROOT_{i+1}, note_roots_left_to_update, ... ] # loop counter movup.4 sub.1 dup neq.0 - #=> [0 or 1, notes_left_to_update - 1, ROOT_{i+1}, ... ] + #=> [0 or 1, note_roots_left_to_update - 1, ROOT_{i+1}, ... ] end drop From 4aeb7d12bec3ad748415a5c17abff23aa74290f0 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 28 Nov 2023 13:57:39 -0500 Subject: [PATCH 115/166] fix compute_chain_mmr_root --- .../src/block_builder/prover/mod.rs | 34 +++++-------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 142c4ead1..73a27f6d8 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -121,38 +121,22 @@ end #! Compute the chain MMR root #! -#! Stack: [NUM_LEAVES, num_peaks, PEAK_0, ..., PEAK_{n-1}, PREV_BLOCK_HASH_TO_INSERT] -#! Output: [CHAIN_MMR_ROOT] +#! Stack: [ PREV_CHAIN_MMR_HASH, PREV_BLOCK_HASH_TO_INSERT ] +#! Advice map: PREV_CHAIN_MMR_HASH -> num_leaves || peak_0 || .. || peak_{n-1} +#! +#! Output: [ CHAIN_MMR_ROOT ] proc.compute_chain_mmr_root - # initialize memory: num leaves - push.CHAIN_MMR_PTR mem_storew dropw - # => [num_peaks, PEAK_0, ..., PEAK_{n-1}, PREV_BLOCK_HASH_TO_INSERT] - - # prepare stack for loop that stores peaks - push.CHAIN_MMR_PTR add.1 dup add - # => [write_ptr, write_ptr_end, PEAK_0, ..., PEAK_{n-1}, PREV_BLOCK_HASH_TO_INSERT] - - dup.1 dup.1 neq - while.true - # => [write_ptr, write_ptr_end, PEAK_i, ..., PEAK_{n-1}, PREV_BLOCK_HASH_TO_INSERT] - - movup.5 movup.5 movup.5 movup.5 dup.4 - # => [write_ptr, PEAK_i, write_ptr, write_ptr_end, PEAK_{i+1}, ..., PEAK_{n-1}, PREV_BLOCK_HASH_TO_INSERT] - - mem_storew dropw - # => [write_ptr, write_ptr_end, PEAK_{i+1}, ..., PEAK_{n-1}, PREV_BLOCK_HASH_TO_INSERT] + push.CHAIN_MMR_PTR movdn.4 + # => [ PREV_CHAIN_MMR_HASH, chain_mmr_ptr, PREV_BLOCK_HASH_TO_INSERT ] - # check if done looping - dup.1 dup.1 neq - # => [0 or 1, write_ptr, write_ptr_end, PEAK_{i+1}, ..., PEAK_{n-1}, PREV_BLOCK_HASH_TO_INSERT] - end + # load the chain MMR (as of previous block) at memory location CHAIN_MMR_PTR + exec.mmr::unpack # => [ PREV_BLOCK_HASH_TO_INSERT ] - # prepare stack to add new peak push.CHAIN_MMR_PTR movdn.4 # => [ PREV_BLOCK_HASH_TO_INSERT, chain_mmr_ptr ] - # add new peak + # add PREV_BLOCK_HASH_TO_INSERT to chain MMR exec.mmr::add # => [ ] From 1d769dedc882ad24bac28a550ce0fbfe26be9118 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 28 Nov 2023 14:04:08 -0500 Subject: [PATCH 116/166] fix comment --- block-producer/src/block_builder/prover/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 73a27f6d8..031b01660 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -122,7 +122,7 @@ end #! Compute the chain MMR root #! #! Stack: [ PREV_CHAIN_MMR_HASH, PREV_BLOCK_HASH_TO_INSERT ] -#! Advice map: PREV_CHAIN_MMR_HASH -> num_leaves || peak_0 || .. || peak_{n-1} +#! Advice map: PREV_CHAIN_MMR_HASH -> NUM_LEAVES || peak_0 || .. || peak_{n-1} || #! #! Output: [ CHAIN_MMR_ROOT ] proc.compute_chain_mmr_root From a1be343a21afbc1163fca9c1809636d2f38c398f Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 28 Nov 2023 14:29:28 -0500 Subject: [PATCH 117/166] chain MMR stack inputs --- block-producer/src/block_builder/prover/mod.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 031b01660..be35851af 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -8,7 +8,7 @@ use miden_node_proto::domain::BlockInputs; use miden_objects::{ accounts::AccountId, assembly::Assembler, - crypto::merkle::{EmptySubtreeRoots, MerkleStore}, + crypto::merkle::{EmptySubtreeRoots, MerkleStore, MmrPeaks}, BlockHeader, Digest, Felt, }; use miden_stdlib::StdLibrary; @@ -256,11 +256,12 @@ impl BlockProver { // ================================================================================================= /// Provides inputs to the `BlockKernel` so that it can generate the new header -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq)] pub(super) struct BlockWitness { updated_accounts: BTreeMap, /// (batch_index, created_notes_root) for batches that contain notes batch_created_notes_roots: Vec<(usize, Digest)>, + chain_peaks: MmrPeaks, prev_header: BlockHeader, } @@ -319,6 +320,7 @@ impl BlockWitness { Ok(Self { updated_accounts, batch_created_notes_roots, + chain_peaks: block_inputs.chain_peaks, prev_header: block_inputs.block_header, }) } @@ -400,6 +402,12 @@ impl BlockWitness { // from the bottom to the top let mut stack_inputs = Vec::new(); + // Chain MMR stack inputs + { + stack_inputs.extend(self.prev_header.hash()); + stack_inputs.extend(self.chain_peaks.hash_peaks()); + } + // Notes stack inputs { let num_created_notes_roots = self.batch_created_notes_roots.len(); From 894b2ef3ea3f92c729ae2705c4c3266fa481eae4 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 28 Nov 2023 15:39:01 -0500 Subject: [PATCH 118/166] remove now-useless collect() --- block-producer/src/batch_builder/mod.rs | 9 +++------ block-producer/src/test_utils/store.rs | 6 ++---- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/block-producer/src/batch_builder/mod.rs b/block-producer/src/batch_builder/mod.rs index 5af7ca2db..62bbadbef 100644 --- a/block-producer/src/batch_builder/mod.rs +++ b/block-producer/src/batch_builder/mod.rs @@ -64,12 +64,9 @@ impl TransactionBatch { SimpleSmt::with_contiguous_leaves( CREATED_NOTES_SMT_DEPTH, - created_notes - .into_iter() - .flat_map(|note_envelope| { - [note_envelope.note_hash().into(), note_envelope.metadata().into()] - }) - .collect::>(), + created_notes.into_iter().flat_map(|note_envelope| { + [note_envelope.note_hash().into(), note_envelope.metadata().into()] + }), )? }; diff --git a/block-producer/src/test_utils/store.rs b/block-producer/src/test_utils/store.rs index 31a6e9750..8da745ee5 100644 --- a/block-producer/src/test_utils/store.rs +++ b/block-producer/src/test_utils/store.rs @@ -30,10 +30,8 @@ impl MockStoreSuccess { accounts: impl Iterator, consumed_nullifiers: BTreeSet, ) -> Self { - let accounts: Vec<_> = accounts - .into_iter() - .map(|(account_id, hash)| (account_id.into(), hash.into())) - .collect(); + let accounts = + accounts.into_iter().map(|(account_id, hash)| (account_id.into(), hash.into())); let store_accounts = SimpleSmt::with_leaves(64, accounts).unwrap(); Self { From 0986a943713bab316d7739409934c172b9230e84 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 28 Nov 2023 16:30:09 -0500 Subject: [PATCH 119/166] advice provider inputs --- block-producer/src/block_builder/prover/mod.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index be35851af..50f24b19b 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -461,7 +461,12 @@ impl BlockWitness { )) .map_err(BlockProverError::InvalidMerklePaths)?; - AdviceInputs::default().with_merkle_store(merkle_store) + let mut map_data: Vec = Vec::new(); + map_data.extend([Felt::from(self.chain_peaks.num_leaves() as u64)]); + + AdviceInputs::default() + .with_merkle_store(merkle_store) + .with_map([(Digest::from(self.chain_peaks.hash_peaks()).into(), map_data)]) }; Ok((advice_inputs, stack_inputs)) From 246c12b4a37b6969528e25acfcc3f212deb13db8 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 28 Nov 2023 16:45:05 -0500 Subject: [PATCH 120/166] fill out advice inputs --- .../src/block_builder/prover/mod.rs | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 50f24b19b..26deb854d 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -1,4 +1,5 @@ use std::{ + cmp::min, collections::{BTreeMap, BTreeSet}, time::{SystemTime, UNIX_EPOCH}, }; @@ -9,7 +10,7 @@ use miden_objects::{ accounts::AccountId, assembly::Assembler, crypto::merkle::{EmptySubtreeRoots, MerkleStore, MmrPeaks}, - BlockHeader, Digest, Felt, + BlockHeader, Digest, Felt, ZERO, }; use miden_stdlib::StdLibrary; use miden_vm::{ @@ -33,6 +34,8 @@ pub(crate) const CREATED_NOTES_TREE_INSERTION_DEPTH: u8 = 8; pub(crate) const CREATED_NOTES_TREE_DEPTH: u8 = CREATED_NOTES_TREE_INSERTION_DEPTH + batch_builder::CREATED_NOTES_SMT_DEPTH; +pub(crate) const MMR_MIN_NUM_PEAKS: usize = 16; + #[cfg(test)] mod tests; @@ -461,8 +464,28 @@ impl BlockWitness { )) .map_err(BlockProverError::InvalidMerklePaths)?; - let mut map_data: Vec = Vec::new(); - map_data.extend([Felt::from(self.chain_peaks.num_leaves() as u64)]); + // advice map data is expected to be: + // [ NUM_LEAVES, peak_0, ..., peak{n-1}, ] + let map_data = { + // num leaves + let num_leaves = + [Felt::from(self.chain_peaks.num_leaves() as u64), ZERO, ZERO, ZERO]; + + // peaks + let num_peaks = self.chain_peaks.peaks().len(); + let num_padding_peaks = MMR_MIN_NUM_PEAKS - num_peaks; + let padding_peaks = vec![Digest::default(); min(num_padding_peaks, 0)]; + + let all_peaks_including_padding = + self.chain_peaks.peaks().iter().chain(padding_peaks.iter()); + + // fill out map data + let mut map_data: Vec = Vec::new(); + map_data.extend(num_leaves); + map_data.extend(all_peaks_including_padding.flat_map(|peak| peak.iter())); + + map_data + }; AdviceInputs::default() .with_merkle_store(merkle_store) From 40e573b0e547ff864466e1aac2e13ed4c1c0660e Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Tue, 28 Nov 2023 16:52:03 -0500 Subject: [PATCH 121/166] update after hash_peak change --- block-producer/src/block_builder/prover/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 26deb854d..2f4a43667 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -489,7 +489,7 @@ impl BlockWitness { AdviceInputs::default() .with_merkle_store(merkle_store) - .with_map([(Digest::from(self.chain_peaks.hash_peaks()).into(), map_data)]) + .with_map([(self.chain_peaks.hash_peaks().into(), map_data)]) }; Ok((advice_inputs, stack_inputs)) From 3324734bb59686f72ae6f9256dd866e8d84ca9f2 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 29 Nov 2023 09:23:15 -0500 Subject: [PATCH 122/166] remove main proc --- block-producer/src/block_builder/prover/mod.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 2f4a43667..14d1f84c3 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -149,24 +149,20 @@ proc.compute_chain_mmr_root end # Stack: [, , ] -proc.main.3 - exec.compute_account_root loc_storew.0 dropw +begin + exec.compute_account_root mem_storew.0 dropw # => [, ] - exec.compute_note_root loc_storew.1 dropw + exec.compute_note_root mem_storew.1 dropw # => [ ] - exec.compute_chain_mmr_root loc_storew.2 dropw + exec.compute_chain_mmr_root # => [ ] # Load output on stack - loc_loadw.2 padw loc_loadw.1 padw loc_loadw.0 + padw mem_loadw.1 padw mem_loadw.0 #=> [ ACCOUNT_ROOT, NOTE_ROOT, CHAIN_MMR_ROOT ] end - -begin - exec.main -end "; #[derive(Debug)] From 238e155111f6bc426aeadd92601eb58fb25ca546 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 29 Nov 2023 09:37:30 -0500 Subject: [PATCH 123/166] move BlockWitness in new mod --- block-producer/src/block_builder/errors.rs | 3 +- block-producer/src/block_builder/mod.rs | 2 +- .../src/block_builder/prover/block_witness.rs | 272 +++++++++++++++++ .../src/block_builder/prover/mod.rs | 278 +----------------- .../src/block_builder/prover/tests.rs | 11 +- 5 files changed, 289 insertions(+), 277 deletions(-) create mode 100644 block-producer/src/block_builder/prover/block_witness.rs diff --git a/block-producer/src/block_builder/errors.rs b/block-producer/src/block_builder/errors.rs index 93a70f496..f478d5481 100644 --- a/block-producer/src/block_builder/errors.rs +++ b/block-producer/src/block_builder/errors.rs @@ -4,7 +4,8 @@ use thiserror::Error; use crate::store::{ApplyBlockError, BlockInputsError}; -use super::prover::CREATED_NOTES_TREE_INSERTION_DEPTH; +use super::prover::block_witness::CREATED_NOTES_TREE_INSERTION_DEPTH; + #[derive(Debug, Error, PartialEq, Eq)] pub enum BuildBlockError { #[error("failed to update account root: {0}")] diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index 1c47876cd..34fdfe2ca 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -10,7 +10,7 @@ pub mod errors; mod prover; use self::{ errors::BuildBlockError, - prover::{BlockProver, BlockWitness}, + prover::{block_witness::BlockWitness, BlockProver}, }; #[cfg(test)] diff --git a/block-producer/src/block_builder/prover/block_witness.rs b/block-producer/src/block_builder/prover/block_witness.rs new file mode 100644 index 000000000..ab39aa64e --- /dev/null +++ b/block-producer/src/block_builder/prover/block_witness.rs @@ -0,0 +1,272 @@ +use std::{ + cmp::min, + collections::{BTreeMap, BTreeSet}, +}; + +use miden_node_proto::domain::BlockInputs; +use miden_objects::{ + accounts::AccountId, + crypto::merkle::{EmptySubtreeRoots, MerkleStore, MmrPeaks}, + BlockHeader, Digest, Felt, ZERO, +}; +use miden_vm::{crypto::MerklePath, AdviceInputs, StackInputs}; + +use crate::{ + batch_builder, + block_builder::errors::{BlockProverError, BuildBlockError}, + SharedTxBatch, +}; + +/// The depth at which we insert roots from the batches. +pub(crate) const CREATED_NOTES_TREE_INSERTION_DEPTH: u8 = 8; + +/// The depth of the created notes tree in the block. +pub(crate) const CREATED_NOTES_TREE_DEPTH: u8 = + CREATED_NOTES_TREE_INSERTION_DEPTH + batch_builder::CREATED_NOTES_SMT_DEPTH; + +pub(crate) const MMR_MIN_NUM_PEAKS: usize = 16; + +/// Provides inputs to the `BlockKernel` so that it can generate the new header +#[derive(Debug, PartialEq)] +pub struct BlockWitness { + pub(super) updated_accounts: BTreeMap, + /// (batch_index, created_notes_root) for batches that contain notes + pub(super) batch_created_notes_roots: Vec<(usize, Digest)>, + pub(super) chain_peaks: MmrPeaks, + pub(super) prev_header: BlockHeader, +} + +impl BlockWitness { + pub fn new( + block_inputs: BlockInputs, + batches: Vec, + ) -> Result { + Self::validate_inputs(&block_inputs, &batches)?; + + let updated_accounts = { + let mut account_initial_states: BTreeMap = + batches.iter().flat_map(|batch| batch.account_initial_states()).collect(); + + let mut account_merkle_proofs: BTreeMap = block_inputs + .account_states + .into_iter() + .map(|record| (record.account_id, record.proof)) + .collect(); + + batches + .iter() + .flat_map(|batch| batch.updated_accounts()) + .map(|(account_id, final_state_hash)| { + let initial_state_hash = account_initial_states + .remove(&account_id) + .expect("already validated that key exists"); + let proof = account_merkle_proofs + .remove(&account_id) + .expect("already validated that key exists"); + + ( + account_id, + AccountUpdate { + initial_state_hash, + final_state_hash, + proof, + }, + ) + }) + .collect() + }; + + let batch_created_notes_roots = batches + .iter() + .enumerate() + .filter_map(|(batch_index, batch)| { + if batch.created_notes().next().is_none() { + None + } else { + Some((batch_index, batch.created_notes_root())) + } + }) + .collect(); + + Ok(Self { + updated_accounts, + batch_created_notes_roots, + chain_peaks: block_inputs.chain_peaks, + prev_header: block_inputs.block_header, + }) + } + + fn validate_inputs( + block_inputs: &BlockInputs, + batches: &[SharedTxBatch], + ) -> Result<(), BuildBlockError> { + // TODO: + // - Block height returned for each nullifier is 0. + + // Validate that there aren't too many batches in the block. + if batches.len() > 2usize.pow(CREATED_NOTES_TREE_INSERTION_DEPTH.into()) { + return Err(BuildBlockError::TooManyBatchesInBlock(batches.len())); + } + + Self::validate_account_states(block_inputs, batches)?; + + Ok(()) + } + + /// Validate that initial account states coming from the batches are the same as the account + /// states returned from the store + fn validate_account_states( + block_inputs: &BlockInputs, + batches: &[SharedTxBatch], + ) -> Result<(), BuildBlockError> { + let batches_initial_states: BTreeMap = + batches.iter().flat_map(|batch| batch.account_initial_states()).collect(); + + let accounts_in_batches: BTreeSet = + batches_initial_states.keys().cloned().collect(); + let accounts_in_store: BTreeSet = block_inputs + .account_states + .iter() + .map(|record| &record.account_id) + .cloned() + .collect(); + + if accounts_in_batches == accounts_in_store { + let accounts_with_different_hashes: Vec = block_inputs + .account_states + .iter() + .filter_map(|record| { + let hash_in_store = record.account_hash; + let hash_in_batches = batches_initial_states + .get(&record.account_id) + .expect("we already verified that account id is contained in batches"); + + if hash_in_store == *hash_in_batches { + None + } else { + Some(record.account_id) + } + }) + .collect(); + + if accounts_with_different_hashes.is_empty() { + Ok(()) + } else { + Err(BuildBlockError::InconsistentAccountStates(accounts_with_different_hashes)) + } + } else { + // The batches and store don't modify the same set of accounts + let union: BTreeSet = + accounts_in_batches.union(&accounts_in_store).cloned().collect(); + let intersection: BTreeSet = + accounts_in_batches.intersection(&accounts_in_store).cloned().collect(); + + let difference: Vec = union.difference(&intersection).cloned().collect(); + + Err(BuildBlockError::InconsistentAccountIds(difference)) + } + } + + pub(super) fn into_parts(self) -> Result<(AdviceInputs, StackInputs), BlockProverError> { + let stack_inputs = { + // Note: `StackInputs::new()` reverses the input vector, so we need to construct the stack + // from the bottom to the top + let mut stack_inputs = Vec::new(); + + // Chain MMR stack inputs + { + stack_inputs.extend(self.prev_header.hash()); + stack_inputs.extend(self.chain_peaks.hash_peaks()); + } + + // Notes stack inputs + { + let num_created_notes_roots = self.batch_created_notes_roots.len(); + for (batch_index, batch_created_notes_root) in self.batch_created_notes_roots { + stack_inputs.extend(batch_created_notes_root); + + let batch_index = u64::try_from(batch_index) + .expect("can't be more than 2^64 - 1 notes created"); + stack_inputs.push(Felt::from(batch_index)); + } + + let empty_root = EmptySubtreeRoots::entry(CREATED_NOTES_TREE_DEPTH, 0); + stack_inputs.extend(*empty_root); + stack_inputs.push(Felt::from( + u64::try_from(num_created_notes_roots) + .expect("can't be more than 2^64 - 1 notes created"), + )); + } + + // Account stack inputs + let mut num_accounts_updated: u64 = 0; + for (idx, (&account_id, account_update)) in self.updated_accounts.iter().enumerate() { + stack_inputs.push(account_id.into()); + stack_inputs.extend(account_update.final_state_hash); + + let idx = u64::try_from(idx).expect("can't be more than 2^64 - 1 accounts"); + num_accounts_updated = idx + 1; + } + + // append initial account root + stack_inputs.extend(self.prev_header.account_root()); + + // append number of accounts updated + stack_inputs.push(num_accounts_updated.into()); + + StackInputs::new(stack_inputs) + }; + + let advice_inputs = { + let mut merkle_store = MerkleStore::default(); + merkle_store + .add_merkle_paths(self.updated_accounts.into_iter().map( + |( + account_id, + AccountUpdate { + initial_state_hash, + final_state_hash: _, + proof, + }, + )| { (u64::from(account_id), initial_state_hash, proof) }, + )) + .map_err(BlockProverError::InvalidMerklePaths)?; + + // advice map data is expected to be: + // [ NUM_LEAVES, peak_0, ..., peak{n-1}, ] + let map_data = { + // num leaves + let num_leaves = + [Felt::from(self.chain_peaks.num_leaves() as u64), ZERO, ZERO, ZERO]; + + // peaks + let num_peaks = self.chain_peaks.peaks().len(); + let num_padding_peaks = MMR_MIN_NUM_PEAKS - num_peaks; + let padding_peaks = vec![Digest::default(); min(num_padding_peaks, 0)]; + + let all_peaks_including_padding = + self.chain_peaks.peaks().iter().chain(padding_peaks.iter()); + + // fill out map data + let mut map_data: Vec = Vec::new(); + map_data.extend(num_leaves); + map_data.extend(all_peaks_including_padding.flat_map(|peak| peak.iter())); + + map_data + }; + + AdviceInputs::default() + .with_merkle_store(merkle_store) + .with_map([(self.chain_peaks.hash_peaks().into(), map_data)]) + }; + + Ok((advice_inputs, stack_inputs)) + } +} + +#[derive(Debug, PartialEq, Eq)] +pub(super) struct AccountUpdate { + pub initial_state_hash: Digest, + pub final_state_hash: Digest, + pub proof: MerklePath, +} diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 14d1f84c3..8e2d5442f 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -1,23 +1,11 @@ -use std::{ - cmp::min, - collections::{BTreeMap, BTreeSet}, - time::{SystemTime, UNIX_EPOCH}, -}; - -use miden_air::ExecutionOptions; -use miden_node_proto::domain::BlockInputs; -use miden_objects::{ - accounts::AccountId, - assembly::Assembler, - crypto::merkle::{EmptySubtreeRoots, MerkleStore, MmrPeaks}, - BlockHeader, Digest, Felt, ZERO, -}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use miden_air::{ExecutionOptions, Felt}; +use miden_objects::{assembly::Assembler, BlockHeader, Digest}; use miden_stdlib::StdLibrary; -use miden_vm::{ - crypto::MerklePath, execute, AdviceInputs, DefaultHost, MemAdviceProvider, Program, StackInputs, -}; +use miden_vm::{execute, DefaultHost, MemAdviceProvider, Program}; -use crate::{batch_builder, SharedTxBatch}; +use self::block_witness::BlockWitness; use super::{errors::BlockProverError, BuildBlockError}; @@ -27,14 +15,7 @@ pub const ACCOUNT_ROOT_WORD_IDX: usize = 0; /// The index of the word at which the note root is stored on the output stack. pub const NOTE_ROOT_WORD_IDX: usize = 4; -/// The depth at which we insert roots from the batches. -pub(crate) const CREATED_NOTES_TREE_INSERTION_DEPTH: u8 = 8; - -/// The depth of the created notes tree in the block. -pub(crate) const CREATED_NOTES_TREE_DEPTH: u8 = - CREATED_NOTES_TREE_INSERTION_DEPTH + batch_builder::CREATED_NOTES_SMT_DEPTH; - -pub(crate) const MMR_MIN_NUM_PEAKS: usize = 16; +pub mod block_witness; #[cfg(test)] mod tests; @@ -253,248 +234,3 @@ impl BlockProver { // BLOCK WITNESS // ================================================================================================= - -/// Provides inputs to the `BlockKernel` so that it can generate the new header -#[derive(Debug, PartialEq)] -pub(super) struct BlockWitness { - updated_accounts: BTreeMap, - /// (batch_index, created_notes_root) for batches that contain notes - batch_created_notes_roots: Vec<(usize, Digest)>, - chain_peaks: MmrPeaks, - prev_header: BlockHeader, -} - -impl BlockWitness { - pub(super) fn new( - block_inputs: BlockInputs, - batches: Vec, - ) -> Result { - Self::validate_inputs(&block_inputs, &batches)?; - - let updated_accounts = { - let mut account_initial_states: BTreeMap = - batches.iter().flat_map(|batch| batch.account_initial_states()).collect(); - - let mut account_merkle_proofs: BTreeMap = block_inputs - .account_states - .into_iter() - .map(|record| (record.account_id, record.proof)) - .collect(); - - batches - .iter() - .flat_map(|batch| batch.updated_accounts()) - .map(|(account_id, final_state_hash)| { - let initial_state_hash = account_initial_states - .remove(&account_id) - .expect("already validated that key exists"); - let proof = account_merkle_proofs - .remove(&account_id) - .expect("already validated that key exists"); - - ( - account_id, - AccountUpdate { - initial_state_hash, - final_state_hash, - proof, - }, - ) - }) - .collect() - }; - - let batch_created_notes_roots = batches - .iter() - .enumerate() - .filter_map(|(batch_index, batch)| { - if batch.created_notes().next().is_none() { - None - } else { - Some((batch_index, batch.created_notes_root())) - } - }) - .collect(); - - Ok(Self { - updated_accounts, - batch_created_notes_roots, - chain_peaks: block_inputs.chain_peaks, - prev_header: block_inputs.block_header, - }) - } - - fn validate_inputs( - block_inputs: &BlockInputs, - batches: &[SharedTxBatch], - ) -> Result<(), BuildBlockError> { - // TODO: - // - Block height returned for each nullifier is 0. - - // Validate that there aren't too many batches in the block. - if batches.len() > 2usize.pow(CREATED_NOTES_TREE_INSERTION_DEPTH.into()) { - return Err(BuildBlockError::TooManyBatchesInBlock(batches.len())); - } - - Self::validate_account_states(block_inputs, batches)?; - - Ok(()) - } - - /// Validate that initial account states coming from the batches are the same as the account - /// states returned from the store - fn validate_account_states( - block_inputs: &BlockInputs, - batches: &[SharedTxBatch], - ) -> Result<(), BuildBlockError> { - let batches_initial_states: BTreeMap = - batches.iter().flat_map(|batch| batch.account_initial_states()).collect(); - - let accounts_in_batches: BTreeSet = - batches_initial_states.keys().cloned().collect(); - let accounts_in_store: BTreeSet = block_inputs - .account_states - .iter() - .map(|record| &record.account_id) - .cloned() - .collect(); - - if accounts_in_batches == accounts_in_store { - let accounts_with_different_hashes: Vec = block_inputs - .account_states - .iter() - .filter_map(|record| { - let hash_in_store = record.account_hash; - let hash_in_batches = batches_initial_states - .get(&record.account_id) - .expect("we already verified that account id is contained in batches"); - - if hash_in_store == *hash_in_batches { - None - } else { - Some(record.account_id) - } - }) - .collect(); - - if accounts_with_different_hashes.is_empty() { - Ok(()) - } else { - Err(BuildBlockError::InconsistentAccountStates(accounts_with_different_hashes)) - } - } else { - // The batches and store don't modify the same set of accounts - let union: BTreeSet = - accounts_in_batches.union(&accounts_in_store).cloned().collect(); - let intersection: BTreeSet = - accounts_in_batches.intersection(&accounts_in_store).cloned().collect(); - - let difference: Vec = union.difference(&intersection).cloned().collect(); - - Err(BuildBlockError::InconsistentAccountIds(difference)) - } - } - - fn into_parts(self) -> Result<(AdviceInputs, StackInputs), BlockProverError> { - let stack_inputs = { - // Note: `StackInputs::new()` reverses the input vector, so we need to construct the stack - // from the bottom to the top - let mut stack_inputs = Vec::new(); - - // Chain MMR stack inputs - { - stack_inputs.extend(self.prev_header.hash()); - stack_inputs.extend(self.chain_peaks.hash_peaks()); - } - - // Notes stack inputs - { - let num_created_notes_roots = self.batch_created_notes_roots.len(); - for (batch_index, batch_created_notes_root) in self.batch_created_notes_roots { - stack_inputs.extend(batch_created_notes_root); - - let batch_index = u64::try_from(batch_index) - .expect("can't be more than 2^64 - 1 notes created"); - stack_inputs.push(Felt::from(batch_index)); - } - - let empty_root = EmptySubtreeRoots::entry(CREATED_NOTES_TREE_DEPTH, 0); - stack_inputs.extend(*empty_root); - stack_inputs.push(Felt::from( - u64::try_from(num_created_notes_roots) - .expect("can't be more than 2^64 - 1 notes created"), - )); - } - - // Account stack inputs - let mut num_accounts_updated: u64 = 0; - for (idx, (&account_id, account_update)) in self.updated_accounts.iter().enumerate() { - stack_inputs.push(account_id.into()); - stack_inputs.extend(account_update.final_state_hash); - - let idx = u64::try_from(idx).expect("can't be more than 2^64 - 1 accounts"); - num_accounts_updated = idx + 1; - } - - // append initial account root - stack_inputs.extend(self.prev_header.account_root()); - - // append number of accounts updated - stack_inputs.push(num_accounts_updated.into()); - - StackInputs::new(stack_inputs) - }; - - let advice_inputs = { - let mut merkle_store = MerkleStore::default(); - merkle_store - .add_merkle_paths(self.updated_accounts.into_iter().map( - |( - account_id, - AccountUpdate { - initial_state_hash, - final_state_hash: _, - proof, - }, - )| { (u64::from(account_id), initial_state_hash, proof) }, - )) - .map_err(BlockProverError::InvalidMerklePaths)?; - - // advice map data is expected to be: - // [ NUM_LEAVES, peak_0, ..., peak{n-1}, ] - let map_data = { - // num leaves - let num_leaves = - [Felt::from(self.chain_peaks.num_leaves() as u64), ZERO, ZERO, ZERO]; - - // peaks - let num_peaks = self.chain_peaks.peaks().len(); - let num_padding_peaks = MMR_MIN_NUM_PEAKS - num_peaks; - let padding_peaks = vec![Digest::default(); min(num_padding_peaks, 0)]; - - let all_peaks_including_padding = - self.chain_peaks.peaks().iter().chain(padding_peaks.iter()); - - // fill out map data - let mut map_data: Vec = Vec::new(); - map_data.extend(num_leaves); - map_data.extend(all_peaks_including_padding.flat_map(|peak| peak.iter())); - - map_data - }; - - AdviceInputs::default() - .with_merkle_store(merkle_store) - .with_map([(self.chain_peaks.hash_peaks().into(), map_data)]) - }; - - Ok((advice_inputs, stack_inputs)) - } -} - -#[derive(Debug, PartialEq, Eq)] -pub(super) struct AccountUpdate { - pub initial_state_hash: Digest, - pub final_state_hash: Digest, - pub proof: MerklePath, -} diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index 506e62bf5..bcb266e46 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -1,18 +1,21 @@ -use std::sync::Arc; +use std::{collections::BTreeSet, sync::Arc}; use miden_air::FieldElement; use miden_mock::mock::block::mock_block_header; -use miden_node_proto::domain::AccountInputRecord; +use miden_node_proto::domain::{AccountInputRecord, BlockInputs}; use miden_objects::{ - crypto::merkle::MmrPeaks, + accounts::AccountId, + crypto::merkle::{EmptySubtreeRoots, MmrPeaks}, notes::{NoteEnvelope, NoteMetadata}, }; -use miden_vm::crypto::SimpleSmt; +use miden_vm::crypto::{MerklePath, SimpleSmt}; use crate::{ batch_builder::TransactionBatch, + block_builder::prover::block_witness::CREATED_NOTES_TREE_DEPTH, store::Store, test_utils::{DummyProvenTxGenerator, MockStoreSuccess}, + SharedTxBatch, }; use super::*; From 3ed5ac86be2450cba7732725a5b0fe426195ffd9 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 29 Nov 2023 10:17:05 -0500 Subject: [PATCH 124/166] rename error variant --- block-producer/src/block_builder/errors.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/block-producer/src/block_builder/errors.rs b/block-producer/src/block_builder/errors.rs index f478d5481..9df194899 100644 --- a/block-producer/src/block_builder/errors.rs +++ b/block-producer/src/block_builder/errors.rs @@ -8,8 +8,8 @@ use super::prover::block_witness::CREATED_NOTES_TREE_INSERTION_DEPTH; #[derive(Debug, Error, PartialEq, Eq)] pub enum BuildBlockError { - #[error("failed to update account root: {0}")] - AccountRootUpdateFailed(#[from] BlockProverError), + #[error("failed to compute new block: {0}")] + BlockProverFailed(#[from] BlockProverError), #[error("failed to apply block: {0}")] ApplyBlockFailed(#[from] ApplyBlockError), #[error("failed to get block inputs from store: {0}")] From 5f7dc8ccaf1b96cd848d976c7be5aa97d13669c1 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 29 Nov 2023 10:26:32 -0500 Subject: [PATCH 125/166] remove old comment --- block-producer/src/block_builder/prover/mod.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 8e2d5442f..ee70fe52f 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -231,6 +231,3 @@ impl BlockProver { Ok((new_account_root.into(), new_note_root.into())) } } - -// BLOCK WITNESS -// ================================================================================================= From c9244790bd63a7aa945dbc47a4f2fdd933965335 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 29 Nov 2023 10:30:37 -0500 Subject: [PATCH 126/166] tests pass --- block-producer/src/block_builder/prover/block_witness.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/block-producer/src/block_builder/prover/block_witness.rs b/block-producer/src/block_builder/prover/block_witness.rs index ab39aa64e..ff8b55976 100644 --- a/block-producer/src/block_builder/prover/block_witness.rs +++ b/block-producer/src/block_builder/prover/block_witness.rs @@ -1,5 +1,5 @@ use std::{ - cmp::min, + cmp::max, collections::{BTreeMap, BTreeSet}, }; @@ -242,7 +242,7 @@ impl BlockWitness { // peaks let num_peaks = self.chain_peaks.peaks().len(); let num_padding_peaks = MMR_MIN_NUM_PEAKS - num_peaks; - let padding_peaks = vec![Digest::default(); min(num_padding_peaks, 0)]; + let padding_peaks = vec![Digest::default(); max(num_padding_peaks, 0)]; let all_peaks_including_padding = self.chain_peaks.peaks().iter().chain(padding_peaks.iter()); From 3e241b3f9ebb8d2198d2b917164d2634d0fc531d Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 29 Nov 2023 10:43:55 -0500 Subject: [PATCH 127/166] simplify padding peaks logic --- block-producer/src/block_builder/prover/block_witness.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/block-producer/src/block_builder/prover/block_witness.rs b/block-producer/src/block_builder/prover/block_witness.rs index ff8b55976..eb7f45cf3 100644 --- a/block-producer/src/block_builder/prover/block_witness.rs +++ b/block-producer/src/block_builder/prover/block_witness.rs @@ -240,9 +240,11 @@ impl BlockWitness { [Felt::from(self.chain_peaks.num_leaves() as u64), ZERO, ZERO, ZERO]; // peaks - let num_peaks = self.chain_peaks.peaks().len(); - let num_padding_peaks = MMR_MIN_NUM_PEAKS - num_peaks; - let padding_peaks = vec![Digest::default(); max(num_padding_peaks, 0)]; + let padding_peaks = { + let num_padding_peaks = max(MMR_MIN_NUM_PEAKS, self.chain_peaks.peaks().len()); + + vec![Digest::default(); num_padding_peaks] + }; let all_peaks_including_padding = self.chain_peaks.peaks().iter().chain(padding_peaks.iter()); From c157205138b5b9e58edb40d06fdf84cbc829ffea Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 29 Nov 2023 13:23:32 -0500 Subject: [PATCH 128/166] prove: pull out chain MMR root from stack outputs --- block-producer/src/block_builder/prover/mod.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index ee70fe52f..dee7eb7b5 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -15,6 +15,9 @@ pub const ACCOUNT_ROOT_WORD_IDX: usize = 0; /// The index of the word at which the note root is stored on the output stack. pub const NOTE_ROOT_WORD_IDX: usize = 4; +/// The index of the word at which the note root is stored on the output stack. +pub const CHAIN_MMR_ROOT_WORD_IDX: usize = 8; + pub mod block_witness; #[cfg(test)] @@ -177,9 +180,8 @@ impl BlockProver { let block_num = witness.prev_header.block_num(); let version = witness.prev_header.version(); - let (account_root, note_root) = self.compute_roots(witness)?; + let (account_root, note_root, chain_root) = self.compute_roots(witness)?; - let chain_root = Digest::default(); let nullifier_root = Digest::default(); let batch_root = Digest::default(); let proof_hash = Digest::default(); @@ -206,7 +208,7 @@ impl BlockProver { fn compute_roots( &self, witness: BlockWitness, - ) -> Result<(Digest, Digest), BlockProverError> { + ) -> Result<(Digest, Digest, Digest), BlockProverError> { let (advice_inputs, stack_inputs) = witness.into_parts()?; let host = { let advice_provider = MemAdviceProvider::from(advice_inputs); @@ -228,6 +230,11 @@ impl BlockProver { .get_stack_word(NOTE_ROOT_WORD_IDX) .ok_or(BlockProverError::InvalidRootOutput("note".to_string()))?; - Ok((new_account_root.into(), new_note_root.into())) + let new_chain_mmr_root = execution_output + .stack_outputs() + .get_stack_word(CHAIN_MMR_ROOT_WORD_IDX) + .ok_or(BlockProverError::InvalidRootOutput("chain mmr".to_string()))?; + + Ok((new_account_root.into(), new_note_root.into(), new_chain_mmr_root.into())) } } From c191e8e76b9bc444c6a0b27d8c950df24048b9f4 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 29 Nov 2023 14:12:14 -0500 Subject: [PATCH 129/166] chain mmr root tests comment --- block-producer/src/block_builder/prover/tests.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index bcb266e46..fc7fb2fce 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -471,3 +471,10 @@ async fn test_compute_note_root_success() { // --------------------------------------------------------------------------------------------- assert_eq!(block_header.note_root(), notes_smt.root()); } + +// CHAIN MMR ROOT TESTS +// ================================================================================================= + +// - add header to empty MMR, and check that we get the expected commitment +// - add header to non-empty MMR (1 peak), and check that we get the expected commitment +// - add header to MMR with 17 peaks From 8e294ea70eef7ffd9b59a1a18000ee8adb162c3c Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 29 Nov 2023 15:27:31 -0500 Subject: [PATCH 130/166] MockStoreSuccessBuilder --- block-producer/src/test_utils/store.rs | 52 ++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/block-producer/src/test_utils/store.rs b/block-producer/src/test_utils/store.rs index 8da745ee5..5081b2e3c 100644 --- a/block-producer/src/test_utils/store.rs +++ b/block-producer/src/test_utils/store.rs @@ -12,6 +12,58 @@ use crate::{ use super::*; +const ACCOUNT_SMT_DEPTH: u8 = 64; + +/// Builds a [`MockStoreSuccess`] +#[derive(Debug, Default)] +pub struct MockStoreSuccessBuilder { + accounts: Option, + consumed_nullifiers: Option>, +} + +impl MockStoreSuccessBuilder { + pub fn new() -> Self { + Self::default() + } + + pub fn initial_accounts( + mut self, + accounts: impl Iterator, + ) -> Self { + let accounts_smt = { + let accounts = + accounts.into_iter().map(|(account_id, hash)| (account_id.into(), hash.into())); + + SimpleSmt::with_leaves(ACCOUNT_SMT_DEPTH, accounts).unwrap() + }; + + self.accounts = Some(accounts_smt); + + self + } + + pub fn initial_nullifiers( + mut self, + consumed_nullifiers: BTreeSet, + ) -> Self { + self.consumed_nullifiers = Some(consumed_nullifiers); + + self + } + + pub fn build(self) -> MockStoreSuccess { + MockStoreSuccess { + accounts: Arc::new(RwLock::new( + self.accounts.unwrap_or(SimpleSmt::new(ACCOUNT_SMT_DEPTH).unwrap()), + )), + consumed_nullifiers: Arc::new(RwLock::new( + self.consumed_nullifiers.unwrap_or_default(), + )), + num_apply_block_called: Arc::new(RwLock::new(0)), + } + } +} + pub struct MockStoreSuccess { /// Map account id -> account hash accounts: Arc>, From b7e3672a9ea8cd363a48f20f1401a7c0620159c1 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 29 Nov 2023 15:39:50 -0500 Subject: [PATCH 131/166] use MockStoreSuccessBuilder --- .../src/block_builder/prover/tests.rs | 42 +++++----- block-producer/src/block_builder/tests.rs | 22 +++--- .../src/state_view/tests/apply_block.rs | 45 ++++++----- .../src/state_view/tests/verify_tx.rs | 77 +++++++++++-------- block-producer/src/test_utils/mod.rs | 2 +- block-producer/src/test_utils/store.rs | 17 ---- 6 files changed, 106 insertions(+), 99 deletions(-) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index fc7fb2fce..c9ea48b57 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeSet, sync::Arc}; +use std::sync::Arc; use miden_air::FieldElement; use miden_mock::mock::block::mock_block_header; @@ -14,7 +14,7 @@ use crate::{ batch_builder::TransactionBatch, block_builder::prover::block_witness::CREATED_NOTES_TREE_DEPTH, store::Store, - test_utils::{DummyProvenTxGenerator, MockStoreSuccess}, + test_utils::{DummyProvenTxGenerator, MockStoreSuccessBuilder}, SharedTxBatch, }; @@ -210,13 +210,15 @@ async fn test_compute_account_root_success() { // Set up store's account SMT // --------------------------------------------------------------------------------------------- - let store = MockStoreSuccess::new( - account_ids - .iter() - .zip(account_initial_states.iter()) - .map(|(&account_id, &account_hash)| (account_id, account_hash.into())), - BTreeSet::new(), - ); + let store = MockStoreSuccessBuilder::new() + .initial_accounts( + account_ids + .iter() + .zip(account_initial_states.iter()) + .map(|(&account_id, &account_hash)| (account_id, account_hash.into())), + ) + .build(); + // Block prover // --------------------------------------------------------------------------------------------- @@ -290,13 +292,15 @@ async fn test_compute_account_root_empty_batches() { // Set up store's account SMT // --------------------------------------------------------------------------------------------- - let store = MockStoreSuccess::new( - account_ids - .iter() - .zip(account_initial_states.iter()) - .map(|(&account_id, &account_hash)| (account_id, account_hash.into())), - BTreeSet::new(), - ); + let store = MockStoreSuccessBuilder::new() + .initial_accounts( + account_ids + .iter() + .zip(account_initial_states.iter()) + .map(|(&account_id, &account_hash)| (account_id, account_hash.into())), + ) + .build(); + // Block prover // --------------------------------------------------------------------------------------------- @@ -325,7 +329,7 @@ async fn test_compute_note_root_empty_batches_success() { // Set up store // --------------------------------------------------------------------------------------------- - let store = MockStoreSuccess::new(std::iter::empty(), BTreeSet::new()); + let store = MockStoreSuccessBuilder::new().build(); // Block prover // --------------------------------------------------------------------------------------------- @@ -354,7 +358,7 @@ async fn test_compute_note_root_empty_notes_success() { // Set up store // --------------------------------------------------------------------------------------------- - let store = MockStoreSuccess::new(std::iter::empty(), BTreeSet::new()); + let store = MockStoreSuccessBuilder::new().build(); // Block prover // --------------------------------------------------------------------------------------------- @@ -409,7 +413,7 @@ async fn test_compute_note_root_success() { // Set up store // --------------------------------------------------------------------------------------------- - let store = MockStoreSuccess::new(std::iter::empty(), BTreeSet::new()); + let store = MockStoreSuccessBuilder::new().build(); // Block prover // --------------------------------------------------------------------------------------------- diff --git a/block-producer/src/block_builder/tests.rs b/block-producer/src/block_builder/tests.rs index c21860d94..ef528364c 100644 --- a/block-producer/src/block_builder/tests.rs +++ b/block-producer/src/block_builder/tests.rs @@ -1,5 +1,3 @@ -use std::collections::BTreeSet; - // block builder tests (higher level) // 1. `apply_block()` is called use super::*; @@ -8,7 +6,7 @@ use miden_air::Felt; use crate::{ batch_builder::TransactionBatch, - test_utils::{DummyProvenTxGenerator, MockStoreFailure, MockStoreSuccess}, + test_utils::{DummyProvenTxGenerator, MockStoreFailure, MockStoreSuccessBuilder}, }; /// Tests that `build_block()` succeeds when the transaction batches are not empty @@ -18,10 +16,11 @@ async fn test_apply_block_called_nonempty_batches() { let account_id = AccountId::new_unchecked(42u64.into()); let account_initial_hash: Digest = [Felt::from(1u64), Felt::from(1u64), Felt::from(1u64), Felt::from(1u64)].into(); - let store = Arc::new(MockStoreSuccess::new( - std::iter::once((account_id, account_initial_hash)), - BTreeSet::new(), - )); + let store = Arc::new( + MockStoreSuccessBuilder::new() + .initial_accounts(std::iter::once((account_id, account_initial_hash))) + .build(), + ); let block_builder = DefaultBlockBuilder::new(store.clone()); @@ -52,10 +51,11 @@ async fn test_apply_block_called_empty_batches() { let account_id = AccountId::new_unchecked(42u64.into()); let account_hash: Digest = [Felt::from(1u64), Felt::from(1u64), Felt::from(1u64), Felt::from(1u64)].into(); - let store = Arc::new(MockStoreSuccess::new( - std::iter::once((account_id, account_hash)), - BTreeSet::new(), - )); + let store = Arc::new( + MockStoreSuccessBuilder::new() + .initial_accounts(std::iter::once((account_id, account_hash))) + .build(), + ); let block_builder = DefaultBlockBuilder::new(store.clone()); diff --git a/block-producer/src/state_view/tests/apply_block.rs b/block-producer/src/state_view/tests/apply_block.rs index b39ee8180..cc8f2da50 100644 --- a/block-producer/src/state_view/tests/apply_block.rs +++ b/block-producer/src/state_view/tests/apply_block.rs @@ -6,7 +6,7 @@ use std::iter; -use crate::test_utils::MockStoreSuccess; +use crate::test_utils::MockStoreSuccessBuilder; use super::*; @@ -16,10 +16,11 @@ async fn test_apply_block_ab1() { let tx_gen = DummyProvenTxGenerator::new(); let account: MockPrivateAccount<3> = MockPrivateAccount::from(0); - let store = Arc::new(MockStoreSuccess::new( - iter::once((account.id, account.states[0])), - BTreeSet::new(), - )); + let store = Arc::new( + MockStoreSuccessBuilder::new() + .initial_accounts(iter::once((account.id, account.states[0]))) + .build(), + ); let tx = tx_gen.dummy_proven_tx_with_params( account.id, @@ -50,13 +51,16 @@ async fn test_apply_block_ab2() { let (txs, accounts): (Vec<_>, Vec<_>) = get_txs_and_accounts(&tx_gen, 3).unzip(); - let store = Arc::new(MockStoreSuccess::new( - accounts - .clone() - .into_iter() - .map(|mock_account| (mock_account.id, mock_account.states[0])), - BTreeSet::new(), - )); + let store = Arc::new( + MockStoreSuccessBuilder::new() + .initial_accounts( + accounts + .clone() + .into_iter() + .map(|mock_account| (mock_account.id, mock_account.states[0])), + ) + .build(), + ); let state_view = DefaulStateView::new(store.clone()); @@ -88,13 +92,16 @@ async fn test_apply_block_ab3() { let (txs, accounts): (Vec<_>, Vec<_>) = get_txs_and_accounts(&tx_gen, 3).unzip(); - let store = Arc::new(MockStoreSuccess::new( - accounts - .clone() - .into_iter() - .map(|mock_account| (mock_account.id, mock_account.states[0])), - BTreeSet::new(), - )); + let store = Arc::new( + MockStoreSuccessBuilder::new() + .initial_accounts( + accounts + .clone() + .into_iter() + .map(|mock_account| (mock_account.id, mock_account.states[0])), + ) + .build(), + ); let state_view = DefaulStateView::new(store.clone()); diff --git a/block-producer/src/state_view/tests/verify_tx.rs b/block-producer/src/state_view/tests/verify_tx.rs index 0f1bafd2d..74a580173 100644 --- a/block-producer/src/state_view/tests/verify_tx.rs +++ b/block-producer/src/state_view/tests/verify_tx.rs @@ -14,7 +14,7 @@ use std::iter; use tokio::task::JoinSet; -use crate::test_utils::MockStoreSuccess; +use crate::test_utils::MockStoreSuccessBuilder; use super::*; @@ -26,12 +26,15 @@ async fn test_verify_tx_happy_path() { let (txs, accounts): (Vec, Vec) = get_txs_and_accounts(&tx_gen, 3).unzip(); - let store = Arc::new(MockStoreSuccess::new( - accounts - .into_iter() - .map(|mock_account| (mock_account.id, mock_account.states[0])), - BTreeSet::new(), - )); + let store = Arc::new( + MockStoreSuccessBuilder::new() + .initial_accounts( + accounts + .into_iter() + .map(|mock_account| (mock_account.id, mock_account.states[0])), + ) + .build(), + ); let state_view = DefaulStateView::new(store); @@ -50,12 +53,15 @@ async fn test_verify_tx_happy_path_concurrent() { let (txs, accounts): (Vec, Vec) = get_txs_and_accounts(&tx_gen, 3).unzip(); - let store = Arc::new(MockStoreSuccess::new( - accounts - .into_iter() - .map(|mock_account| (mock_account.id, mock_account.states[0])), - BTreeSet::new(), - )); + let store = Arc::new( + MockStoreSuccessBuilder::new() + .initial_accounts( + accounts + .into_iter() + .map(|mock_account| (mock_account.id, mock_account.states[0])), + ) + .build(), + ); let state_view = Arc::new(DefaulStateView::new(store)); @@ -78,10 +84,11 @@ async fn test_verify_tx_vt1() { let account = MockPrivateAccount::<3>::from(0); - let store = Arc::new(MockStoreSuccess::new( - iter::once((account.id, account.states[0])), - BTreeSet::new(), - )); + let store = Arc::new( + MockStoreSuccessBuilder::new() + .initial_accounts(iter::once((account.id, account.states[0]))) + .build(), + ); // The transaction's initial account hash uses `account.states[1]`, where the store expects // `account.states[0]` @@ -114,7 +121,7 @@ async fn test_verify_tx_vt2() { let account_not_in_store: MockPrivateAccount<3> = MockPrivateAccount::from(0); // Notice: account is not added to the store - let store = Arc::new(MockStoreSuccess::new(iter::empty(), BTreeSet::new())); + let store = Arc::new(MockStoreSuccessBuilder::new().build()); let tx = tx_gen.dummy_proven_tx_with_params( account_not_in_store.id, @@ -147,10 +154,12 @@ async fn test_verify_tx_vt3() { let consumed_note_in_store = consumed_note_by_index(0); // Notice: `consumed_note_in_store` is added to the store - let store = Arc::new(MockStoreSuccess::new( - iter::once((account.id, account.states[0])), - BTreeSet::from_iter(iter::once(consumed_note_in_store.nullifier())), - )); + let store = Arc::new( + MockStoreSuccessBuilder::new() + .initial_accounts(iter::once((account.id, account.states[0]))) + .initial_nullifiers(BTreeSet::from_iter(iter::once(consumed_note_in_store.nullifier()))) + .build(), + ); let tx = tx_gen.dummy_proven_tx_with_params( account.id, @@ -179,10 +188,11 @@ async fn test_verify_tx_vt4() { let account: MockPrivateAccount<3> = MockPrivateAccount::from(0); - let store = Arc::new(MockStoreSuccess::new( - iter::once((account.id, account.states[0])), - BTreeSet::new(), - )); + let store = Arc::new( + MockStoreSuccessBuilder::new() + .initial_accounts(iter::once((account.id, account.states[0]))) + .build(), + ); let tx1 = tx_gen.dummy_proven_tx_with_params( account.id, @@ -224,12 +234,15 @@ async fn test_verify_tx_vt5() { let consumed_note_in_both_txs = consumed_note_by_index(0); // Notice: `consumed_note_in_both_txs` is NOT in the store - let store = Arc::new(MockStoreSuccess::new( - vec![account_1, account_2] - .into_iter() - .map(|account| (account.id, account.states[0])), - BTreeSet::new(), - )); + let store = Arc::new( + MockStoreSuccessBuilder::new() + .initial_accounts( + vec![account_1, account_2] + .into_iter() + .map(|account| (account.id, account.states[0])), + ) + .build(), + ); let tx1 = tx_gen.dummy_proven_tx_with_params( account_1.id, diff --git a/block-producer/src/test_utils/mod.rs b/block-producer/src/test_utils/mod.rs index 9cdb64614..ca4125502 100644 --- a/block-producer/src/test_utils/mod.rs +++ b/block-producer/src/test_utils/mod.rs @@ -8,7 +8,7 @@ mod proven_tx; pub use proven_tx::DummyProvenTxGenerator; mod store; -pub use store::{MockStoreFailure, MockStoreSuccess}; +pub use store::{MockStoreFailure, MockStoreSuccess, MockStoreSuccessBuilder}; mod account; pub use account::MockPrivateAccount; diff --git a/block-producer/src/test_utils/store.rs b/block-producer/src/test_utils/store.rs index 5081b2e3c..ecc870779 100644 --- a/block-producer/src/test_utils/store.rs +++ b/block-producer/src/test_utils/store.rs @@ -76,23 +76,6 @@ pub struct MockStoreSuccess { } impl MockStoreSuccess { - /// Initializes the known accounts from provided mock accounts, where the account hash in the - /// store is the first state in `MockAccount.states`. - pub fn new( - accounts: impl Iterator, - consumed_nullifiers: BTreeSet, - ) -> Self { - let accounts = - accounts.into_iter().map(|(account_id, hash)| (account_id.into(), hash.into())); - let store_accounts = SimpleSmt::with_leaves(64, accounts).unwrap(); - - Self { - accounts: Arc::new(RwLock::new(store_accounts)), - consumed_nullifiers: Arc::new(RwLock::new(consumed_nullifiers)), - num_apply_block_called: Arc::new(RwLock::new(0)), - } - } - /// Update some accounts in the store pub async fn update_accounts( &self, From 3e59baba54c45c3833401511b4ccc881103481b8 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Wed, 29 Nov 2023 15:52:44 -0500 Subject: [PATCH 132/166] add MMR to `MockStoreSuccess` --- block-producer/src/test_utils/store.rs | 35 +++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/block-producer/src/test_utils/store.rs b/block-producer/src/test_utils/store.rs index ecc870779..d6c09b0e0 100644 --- a/block-producer/src/test_utils/store.rs +++ b/block-producer/src/test_utils/store.rs @@ -1,7 +1,10 @@ use async_trait::async_trait; use miden_air::{Felt, FieldElement}; use miden_node_proto::domain::{AccountInputRecord, BlockInputs}; -use miden_objects::{crypto::merkle::MmrPeaks, BlockHeader, EMPTY_WORD}; +use miden_objects::{ + crypto::merkle::{Mmr, MmrPeaks}, + BlockHeader, EMPTY_WORD, +}; use miden_vm::crypto::SimpleSmt; use crate::{ @@ -19,6 +22,7 @@ const ACCOUNT_SMT_DEPTH: u8 = 64; pub struct MockStoreSuccessBuilder { accounts: Option, consumed_nullifiers: Option>, + chain_mmr: Option, } impl MockStoreSuccessBuilder { @@ -51,6 +55,12 @@ impl MockStoreSuccessBuilder { self } + pub fn initial_chain_mmr(mut self, chain_mmr: Mmr) -> Self { + self.chain_mmr = Some(chain_mmr); + + self + } + pub fn build(self) -> MockStoreSuccess { MockStoreSuccess { accounts: Arc::new(RwLock::new( @@ -59,6 +69,7 @@ impl MockStoreSuccessBuilder { consumed_nullifiers: Arc::new(RwLock::new( self.consumed_nullifiers.unwrap_or_default(), )), + chain_mmr: Arc::new(RwLock::new(Mmr::default())), num_apply_block_called: Arc::new(RwLock::new(0)), } } @@ -71,6 +82,8 @@ pub struct MockStoreSuccess { /// Stores the nullifiers of the notes that were consumed consumed_nullifiers: Arc>>, + chain_mmr: Arc>, + /// The number of times `apply_block()` was called pub num_apply_block_called: Arc>, } @@ -106,14 +119,24 @@ impl ApplyBlock for MockStoreSuccess { let mut locked_accounts = self.accounts.write().await; let mut locked_consumed_nullifiers = self.consumed_nullifiers.write().await; + // update accounts for &(account_id, account_hash) in block.updated_accounts.iter() { locked_accounts.update_leaf(account_id.into(), account_hash.into()).unwrap(); } + // update nullifiers let mut new_nullifiers: BTreeSet = block.produced_nullifiers.iter().cloned().collect(); locked_consumed_nullifiers.append(&mut new_nullifiers); + // update chain mmr with new block header hash + { + let mut chain_mmr = self.chain_mmr.write().await; + + chain_mmr.add(block.header.hash()); + } + + // update num_apply_block_called *self.num_apply_block_called.write().await += 1; Ok(()) @@ -158,8 +181,12 @@ impl Store for MockStoreSuccess { ) -> Result { let block_header = { let prev_hash: Digest = Digest::default(); - let chain_root: Digest = Digest::default(); - let acct_root: Digest = self.account_root().await; + let chain_root = { + let chain_mmr = self.chain_mmr.read().await; + + chain_mmr.peaks(chain_mmr.forest()).unwrap().hash_peaks() + }; + let account_root: Digest = self.account_root().await; let nullifier_root: Digest = Digest::default(); let note_root: Digest = Digest::default(); let batch_root: Digest = Digest::default(); @@ -169,7 +196,7 @@ impl Store for MockStoreSuccess { prev_hash, Felt::ZERO, chain_root, - acct_root, + account_root, nullifier_root, note_root, batch_root, From 12f7a97f0873217068acfca3493e4cba4ef9111f Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 30 Nov 2023 11:59:49 -0500 Subject: [PATCH 133/166] MockStoreSuccessBuilder: use chain_mmr --- .../src/block_builder/prover/tests.rs | 51 +++++++++++++++++++ block-producer/src/test_utils/store.rs | 7 ++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index c9ea48b57..2bf89a25a 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -482,3 +482,54 @@ async fn test_compute_note_root_success() { // - add header to empty MMR, and check that we get the expected commitment // - add header to non-empty MMR (1 peak), and check that we get the expected commitment // - add header to MMR with 17 peaks + +/// Test that the current account root is returned if the batches are empty +#[tokio::test] +async fn test_compute_chain_mmr_root_empty_mmr() { + // Set up account states + // --------------------------------------------------------------------------------------------- + let account_ids = [ + AccountId::new_unchecked(Felt::from(0b0000_0000_0000_0000u64)), + AccountId::new_unchecked(Felt::from(0b1111_0000_0000_0000u64)), + AccountId::new_unchecked(Felt::from(0b1111_1111_0000_0000u64)), + AccountId::new_unchecked(Felt::from(0b1111_1111_1111_0000u64)), + AccountId::new_unchecked(Felt::from(0b1111_1111_1111_1111u64)), + ]; + + let account_initial_states = [ + [Felt::from(1u64), Felt::from(1u64), Felt::from(1u64), Felt::from(1u64)], + [Felt::from(2u64), Felt::from(2u64), Felt::from(2u64), Felt::from(2u64)], + [Felt::from(3u64), Felt::from(3u64), Felt::from(3u64), Felt::from(3u64)], + [Felt::from(4u64), Felt::from(4u64), Felt::from(4u64), Felt::from(4u64)], + [Felt::from(5u64), Felt::from(5u64), Felt::from(5u64), Felt::from(5u64)], + ]; + + // Set up store's account SMT + // --------------------------------------------------------------------------------------------- + + let store = MockStoreSuccessBuilder::new() + .initial_accounts( + account_ids + .iter() + .zip(account_initial_states.iter()) + .map(|(&account_id, &account_hash)| (account_id, account_hash.into())), + ) + .build(); + + // Block prover + // --------------------------------------------------------------------------------------------- + + // Block inputs is initialized with all the accounts and their initial state + let block_inputs_from_store: BlockInputs = + store.get_block_inputs(std::iter::empty(), std::iter::empty()).await.unwrap(); + + let batches = Vec::new(); + let block_witness = BlockWitness::new(block_inputs_from_store, batches).unwrap(); + + let block_prover = BlockProver::new(); + let block_header = block_prover.prove(block_witness).unwrap(); + + // Compare roots + // --------------------------------------------------------------------------------------------- + assert_eq!(block_header.account_root(), store.account_root().await); +} diff --git a/block-producer/src/test_utils/store.rs b/block-producer/src/test_utils/store.rs index d6c09b0e0..334c17619 100644 --- a/block-producer/src/test_utils/store.rs +++ b/block-producer/src/test_utils/store.rs @@ -55,7 +55,10 @@ impl MockStoreSuccessBuilder { self } - pub fn initial_chain_mmr(mut self, chain_mmr: Mmr) -> Self { + pub fn initial_chain_mmr( + mut self, + chain_mmr: Mmr, + ) -> Self { self.chain_mmr = Some(chain_mmr); self @@ -69,7 +72,7 @@ impl MockStoreSuccessBuilder { consumed_nullifiers: Arc::new(RwLock::new( self.consumed_nullifiers.unwrap_or_default(), )), - chain_mmr: Arc::new(RwLock::new(Mmr::default())), + chain_mmr: Arc::new(RwLock::new(self.chain_mmr.unwrap_or_default())), num_apply_block_called: Arc::new(RwLock::new(0)), } } From aa57f4de06cccceead9216f05223f5046591a0e1 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 30 Nov 2023 12:09:21 -0500 Subject: [PATCH 134/166] MockStoreSuccess: last_block_header --- block-producer/src/test_utils/store.rs | 34 +++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/block-producer/src/test_utils/store.rs b/block-producer/src/test_utils/store.rs index 334c17619..42700118f 100644 --- a/block-producer/src/test_utils/store.rs +++ b/block-producer/src/test_utils/store.rs @@ -3,7 +3,7 @@ use miden_air::{Felt, FieldElement}; use miden_node_proto::domain::{AccountInputRecord, BlockInputs}; use miden_objects::{ crypto::merkle::{Mmr, MmrPeaks}, - BlockHeader, EMPTY_WORD, + BlockHeader, EMPTY_WORD, ZERO, }; use miden_vm::crypto::SimpleSmt; @@ -23,6 +23,7 @@ pub struct MockStoreSuccessBuilder { accounts: Option, consumed_nullifiers: Option>, chain_mmr: Option, + block_header: Option, } impl MockStoreSuccessBuilder { @@ -64,7 +65,31 @@ impl MockStoreSuccessBuilder { self } + pub fn initial_block_header( + mut self, + block_header: BlockHeader, + ) -> Self { + self.block_header = Some(block_header); + + self + } + pub fn build(self) -> MockStoreSuccess { + let default_block_header = || { + BlockHeader::new( + Digest::default(), + Felt::ZERO, + Digest::default(), + Digest::default(), + Digest::default(), + Digest::default(), + Digest::default(), + Digest::default(), + Felt::ZERO, + Felt::ONE, + ) + }; + MockStoreSuccess { accounts: Arc::new(RwLock::new( self.accounts.unwrap_or(SimpleSmt::new(ACCOUNT_SMT_DEPTH).unwrap()), @@ -73,6 +98,9 @@ impl MockStoreSuccessBuilder { self.consumed_nullifiers.unwrap_or_default(), )), chain_mmr: Arc::new(RwLock::new(self.chain_mmr.unwrap_or_default())), + last_block_header: Arc::new(RwLock::new( + self.block_header.unwrap_or_else(default_block_header), + )), num_apply_block_called: Arc::new(RwLock::new(0)), } } @@ -85,8 +113,12 @@ pub struct MockStoreSuccess { /// Stores the nullifiers of the notes that were consumed consumed_nullifiers: Arc>>, + // Stores the chain MMR chain_mmr: Arc>, + // Stores the header of the last applied block + last_block_header: Arc>, + /// The number of times `apply_block()` was called pub num_apply_block_called: Arc>, } From 6c1c0f223e2fbeb9121389559b48c6c0ad621623 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 30 Nov 2023 12:29:07 -0500 Subject: [PATCH 135/166] MockStore: create proper head at build --- block-producer/src/test_utils/store.rs | 85 ++++++++------------------ 1 file changed, 25 insertions(+), 60 deletions(-) diff --git a/block-producer/src/test_utils/store.rs b/block-producer/src/test_utils/store.rs index 42700118f..28be494cd 100644 --- a/block-producer/src/test_utils/store.rs +++ b/block-producer/src/test_utils/store.rs @@ -3,7 +3,7 @@ use miden_air::{Felt, FieldElement}; use miden_node_proto::domain::{AccountInputRecord, BlockInputs}; use miden_objects::{ crypto::merkle::{Mmr, MmrPeaks}, - BlockHeader, EMPTY_WORD, ZERO, + BlockHeader, EMPTY_WORD, }; use miden_vm::crypto::SimpleSmt; @@ -23,7 +23,6 @@ pub struct MockStoreSuccessBuilder { accounts: Option, consumed_nullifiers: Option>, chain_mmr: Option, - block_header: Option, } impl MockStoreSuccessBuilder { @@ -65,42 +64,30 @@ impl MockStoreSuccessBuilder { self } - pub fn initial_block_header( - mut self, - block_header: BlockHeader, - ) -> Self { - self.block_header = Some(block_header); - - self - } - pub fn build(self) -> MockStoreSuccess { - let default_block_header = || { - BlockHeader::new( - Digest::default(), - Felt::ZERO, - Digest::default(), - Digest::default(), - Digest::default(), - Digest::default(), - Digest::default(), - Digest::default(), - Felt::ZERO, - Felt::ONE, - ) - }; + let accounts_smt = self.accounts.unwrap_or(SimpleSmt::new(ACCOUNT_SMT_DEPTH).unwrap()); + let chain_mmr = self.chain_mmr.unwrap_or_default(); + + let initial_block_header = BlockHeader::new( + Digest::default(), + Felt::ZERO, + chain_mmr.peaks(chain_mmr.forest()).unwrap().hash_peaks(), + accounts_smt.root(), + Digest::default(), + Digest::default(), + Digest::default(), + Digest::default(), + Felt::ZERO, + Felt::ONE, + ); MockStoreSuccess { - accounts: Arc::new(RwLock::new( - self.accounts.unwrap_or(SimpleSmt::new(ACCOUNT_SMT_DEPTH).unwrap()), - )), + accounts: Arc::new(RwLock::new(accounts_smt)), consumed_nullifiers: Arc::new(RwLock::new( self.consumed_nullifiers.unwrap_or_default(), )), - chain_mmr: Arc::new(RwLock::new(self.chain_mmr.unwrap_or_default())), - last_block_header: Arc::new(RwLock::new( - self.block_header.unwrap_or_else(default_block_header), - )), + chain_mmr: Arc::new(RwLock::new(chain_mmr)), + last_block_header: Arc::new(RwLock::new(initial_block_header)), num_apply_block_called: Arc::new(RwLock::new(0)), } } @@ -125,6 +112,7 @@ pub struct MockStoreSuccess { impl MockStoreSuccess { /// Update some accounts in the store + /// TODO: Remove this, and instead create a function to create a block from account updates pub async fn update_accounts( &self, updated_accounts: impl Iterator, @@ -158,6 +146,7 @@ impl ApplyBlock for MockStoreSuccess { for &(account_id, account_hash) in block.updated_accounts.iter() { locked_accounts.update_leaf(account_id.into(), account_hash.into()).unwrap(); } + debug_assert_eq!(locked_accounts.root(), block.header.account_root()); // update nullifiers let mut new_nullifiers: BTreeSet = @@ -171,6 +160,9 @@ impl ApplyBlock for MockStoreSuccess { chain_mmr.add(block.header.hash()); } + // update last block header + *self.last_block_header.write().await = block.header; + // update num_apply_block_called *self.num_apply_block_called.write().await += 1; @@ -214,33 +206,6 @@ impl Store for MockStoreSuccess { updated_accounts: impl Iterator + Send, _produced_nullifiers: impl Iterator + Send, ) -> Result { - let block_header = { - let prev_hash: Digest = Digest::default(); - let chain_root = { - let chain_mmr = self.chain_mmr.read().await; - - chain_mmr.peaks(chain_mmr.forest()).unwrap().hash_peaks() - }; - let account_root: Digest = self.account_root().await; - let nullifier_root: Digest = Digest::default(); - let note_root: Digest = Digest::default(); - let batch_root: Digest = Digest::default(); - let proof_hash: Digest = Digest::default(); - - BlockHeader::new( - prev_hash, - Felt::ZERO, - chain_root, - account_root, - nullifier_root, - note_root, - batch_root, - proof_hash, - Felt::ZERO, - Felt::ONE, - ) - }; - let chain_peaks = MmrPeaks::new(0, Vec::new()).unwrap(); let account_states = { @@ -261,7 +226,7 @@ impl Store for MockStoreSuccess { }; Ok(BlockInputs { - block_header, + block_header: self.last_block_header.read().await.clone(), chain_peaks, account_states, // TODO: return a proper nullifiers iterator From ad45c3378fef79c5dd3e9d394a3703f6080a107f Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 30 Nov 2023 13:51:04 -0500 Subject: [PATCH 136/166] create MockBlockBuilder --- block-producer/src/test_utils/block.rs | 87 ++++++++++++++++++++++++++ block-producer/src/test_utils/mod.rs | 2 + block-producer/src/test_utils/store.rs | 8 +-- 3 files changed, 93 insertions(+), 4 deletions(-) create mode 100644 block-producer/src/test_utils/block.rs diff --git a/block-producer/src/test_utils/block.rs b/block-producer/src/test_utils/block.rs new file mode 100644 index 000000000..0145d59eb --- /dev/null +++ b/block-producer/src/test_utils/block.rs @@ -0,0 +1,87 @@ +use miden_air::{Felt, FieldElement}; +use miden_objects::{accounts::AccountId, crypto::merkle::Mmr, BlockHeader, Digest}; +use miden_vm::crypto::SimpleSmt; + +use crate::block::Block; + +use super::MockStoreSuccess; + +#[derive(Debug)] +pub struct MockBlockBuilder { + store_accounts: SimpleSmt, + store_chain_mmr: Mmr, + last_block_header: BlockHeader, + + updated_accounts: Option>, + created_notes: Option>, + produced_nullifiers: Option>, +} + +impl MockBlockBuilder { + pub async fn new(store: &MockStoreSuccess) -> Self { + Self { + store_accounts: store.accounts.read().await.clone(), + store_chain_mmr: store.chain_mmr.read().await.clone(), + last_block_header: store.last_block_header.read().await.clone(), + + updated_accounts: None, + created_notes: None, + produced_nullifiers: None, + } + } + + pub fn account_updates( + mut self, + updated_accounts: Vec<(AccountId, Digest)>, + ) -> Self { + for (account_id, new_account_state) in updated_accounts.iter() { + self.store_accounts + .update_leaf(u64::from(*account_id), new_account_state.into()) + .unwrap(); + } + + self.updated_accounts = Some(updated_accounts); + + self + } + + pub fn created_notes( + mut self, + created_notes: Vec, + ) -> Self { + self.created_notes = Some(created_notes); + + self + } + + pub fn produced_nullifiers( + mut self, + produced_nullifiers: Vec, + ) -> Self { + self.produced_nullifiers = Some(produced_nullifiers); + + self + } + + pub fn build(self) -> Block { + let header = BlockHeader::new( + self.last_block_header.hash(), + self.last_block_header.block_num() + Felt::ONE, + self.store_chain_mmr.peaks(self.store_chain_mmr.forest()).unwrap().hash_peaks(), + self.store_accounts.root(), + Digest::default(), + Digest::default(), + Digest::default(), + Digest::default(), + Felt::ZERO, + Felt::ONE, + ); + + Block { + header, + updated_accounts: self.updated_accounts.unwrap_or_default(), + created_notes: self.created_notes.unwrap_or_default(), + produced_nullifiers: self.produced_nullifiers.unwrap_or_default(), + } + } +} diff --git a/block-producer/src/test_utils/mod.rs b/block-producer/src/test_utils/mod.rs index ca4125502..d21bf6bc1 100644 --- a/block-producer/src/test_utils/mod.rs +++ b/block-producer/src/test_utils/mod.rs @@ -12,3 +12,5 @@ pub use store::{MockStoreFailure, MockStoreSuccess, MockStoreSuccessBuilder}; mod account; pub use account::MockPrivateAccount; + +pub mod block; diff --git a/block-producer/src/test_utils/store.rs b/block-producer/src/test_utils/store.rs index 28be494cd..6bf35a893 100644 --- a/block-producer/src/test_utils/store.rs +++ b/block-producer/src/test_utils/store.rs @@ -95,16 +95,16 @@ impl MockStoreSuccessBuilder { pub struct MockStoreSuccess { /// Map account id -> account hash - accounts: Arc>, + pub accounts: Arc>, /// Stores the nullifiers of the notes that were consumed - consumed_nullifiers: Arc>>, + pub consumed_nullifiers: Arc>>, // Stores the chain MMR - chain_mmr: Arc>, + pub chain_mmr: Arc>, // Stores the header of the last applied block - last_block_header: Arc>, + pub last_block_header: Arc>, /// The number of times `apply_block()` was called pub num_apply_block_called: Arc>, From 7e79aea846a32eb8261abce4e79c577f11bc20a4 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 30 Nov 2023 13:56:56 -0500 Subject: [PATCH 137/166] remove update_accounts() method --- block-producer/src/block_builder/prover/tests.rs | 16 ++++++++++------ block-producer/src/test_utils/block.rs | 2 +- block-producer/src/test_utils/store.rs | 16 +--------------- 3 files changed, 12 insertions(+), 22 deletions(-) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index 2bf89a25a..db99051d6 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -13,8 +13,8 @@ use miden_vm::crypto::{MerklePath, SimpleSmt}; use crate::{ batch_builder::TransactionBatch, block_builder::prover::block_witness::CREATED_NOTES_TREE_DEPTH, - store::Store, - test_utils::{DummyProvenTxGenerator, MockStoreSuccessBuilder}, + store::{ApplyBlock, Store}, + test_utils::{block::MockBlockBuilder, DummyProvenTxGenerator, MockStoreSuccessBuilder}, SharedTxBatch, }; @@ -254,14 +254,18 @@ async fn test_compute_account_root_success() { // Update SMT by hand to get new root // --------------------------------------------------------------------------------------------- - store - .update_accounts( + let block = MockBlockBuilder::new(&store) + .await + .account_updates( account_ids .iter() .zip(account_final_states.iter()) - .map(|(&account_id, &account_hash)| (account_id, account_hash.into())), + .map(|(&account_id, &account_hash)| (account_id, account_hash.into())) + .collect(), ) - .await; + .build(); + + store.apply_block(Arc::new(block)).await.unwrap(); // Compare roots // --------------------------------------------------------------------------------------------- diff --git a/block-producer/src/test_utils/block.rs b/block-producer/src/test_utils/block.rs index 0145d59eb..e57219a95 100644 --- a/block-producer/src/test_utils/block.rs +++ b/block-producer/src/test_utils/block.rs @@ -22,7 +22,7 @@ impl MockBlockBuilder { Self { store_accounts: store.accounts.read().await.clone(), store_chain_mmr: store.chain_mmr.read().await.clone(), - last_block_header: store.last_block_header.read().await.clone(), + last_block_header: *store.last_block_header.read().await, updated_accounts: None, created_notes: None, diff --git a/block-producer/src/test_utils/store.rs b/block-producer/src/test_utils/store.rs index 6bf35a893..b513780ea 100644 --- a/block-producer/src/test_utils/store.rs +++ b/block-producer/src/test_utils/store.rs @@ -111,20 +111,6 @@ pub struct MockStoreSuccess { } impl MockStoreSuccess { - /// Update some accounts in the store - /// TODO: Remove this, and instead create a function to create a block from account updates - pub async fn update_accounts( - &self, - updated_accounts: impl Iterator, - ) { - let mut locked_accounts = self.accounts.write().await; - for (account_id, new_account_state) in updated_accounts { - locked_accounts - .update_leaf(account_id.into(), new_account_state.into()) - .unwrap(); - } - } - pub async fn account_root(&self) -> Digest { let locked_accounts = self.accounts.read().await; @@ -226,7 +212,7 @@ impl Store for MockStoreSuccess { }; Ok(BlockInputs { - block_header: self.last_block_header.read().await.clone(), + block_header: *self.last_block_header.read().await, chain_peaks, account_states, // TODO: return a proper nullifiers iterator From d8c8ddbeb23813856e7cd996a7e8ceb5a01cbbf0 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 30 Nov 2023 14:03:29 -0500 Subject: [PATCH 138/166] remove get_dummy_block() --- .../src/state_view/tests/apply_block.rs | 46 ++++++++++++++----- block-producer/src/state_view/tests/mod.rs | 32 +------------ 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/block-producer/src/state_view/tests/apply_block.rs b/block-producer/src/state_view/tests/apply_block.rs index cc8f2da50..f1e48052f 100644 --- a/block-producer/src/state_view/tests/apply_block.rs +++ b/block-producer/src/state_view/tests/apply_block.rs @@ -6,7 +6,7 @@ use std::iter; -use crate::test_utils::MockStoreSuccessBuilder; +use crate::test_utils::{block::MockBlockBuilder, MockStoreSuccessBuilder}; use super::*; @@ -36,9 +36,16 @@ async fn test_apply_block_ab1() { let verify_tx_res = state_view.verify_tx(tx.into()).await; assert!(verify_tx_res.is_ok()); - let block = Arc::new(get_dummy_block(vec![account], Vec::new())); + let block = MockBlockBuilder::new(&store) + .await + .account_updates( + std::iter::once(account) + .map(|mock_account| (mock_account.id, mock_account.states[1])) + .collect(), + ) + .build(); - let apply_block_res = state_view.apply_block(block).await; + let apply_block_res = state_view.apply_block(Arc::new(block)).await; assert!(apply_block_res.is_ok()); assert_eq!(*store.num_apply_block_called.read().await, 1); @@ -71,11 +78,19 @@ async fn test_apply_block_ab2() { } // All except the first account will go into the block. - let accounts_in_block = accounts.iter().skip(1).cloned().collect(); - - let block = Arc::new(get_dummy_block(accounts_in_block, Vec::new())); - - let apply_block_res = state_view.apply_block(block).await; + let accounts_in_block: Vec = accounts.iter().skip(1).cloned().collect(); + + let block = MockBlockBuilder::new(&store) + .await + .account_updates( + accounts_in_block + .into_iter() + .map(|mock_account| (mock_account.id, mock_account.states[1])) + .collect(), + ) + .build(); + + let apply_block_res = state_view.apply_block(Arc::new(block)).await; assert!(apply_block_res.is_ok()); let accounts_still_in_flight = state_view.accounts_in_flight.read().await; @@ -111,9 +126,18 @@ async fn test_apply_block_ab3() { assert!(verify_tx_res.is_ok()); } - let block = Arc::new(get_dummy_block(accounts.clone(), Vec::new())); - - let apply_block_res = state_view.apply_block(block).await; + let block = MockBlockBuilder::new(&store) + .await + .account_updates( + accounts + .clone() + .into_iter() + .map(|mock_account| (mock_account.id, mock_account.states[1])) + .collect(), + ) + .build(); + + let apply_block_res = state_view.apply_block(Arc::new(block)).await; assert!(apply_block_res.is_ok()); // Craft a new transaction which tries to consume the same note that was consumed in in the diff --git a/block-producer/src/state_view/tests/mod.rs b/block-producer/src/state_view/tests/mod.rs index 6ecda9fae..73b97f0c5 100644 --- a/block-producer/src/state_view/tests/mod.rs +++ b/block-producer/src/state_view/tests/mod.rs @@ -1,6 +1,6 @@ use super::*; -use miden_objects::{transaction::ConsumedNoteInfo, BlockHeader, Felt, Hasher}; +use miden_objects::{transaction::ConsumedNoteInfo, Hasher}; use crate::test_utils::{DummyProvenTxGenerator, MockPrivateAccount}; @@ -33,33 +33,3 @@ pub fn get_txs_and_accounts( (Arc::new(tx), account) }) } - -pub fn get_dummy_block( - updated_accounts: Vec, - new_nullifiers: Vec, -) -> Block { - let header = BlockHeader::new( - Digest::default(), - Felt::new(42), - Digest::default(), - Digest::default(), - Digest::default(), - Digest::default(), - Digest::default(), - Digest::default(), - Felt::new(0), - Felt::new(42), - ); - - let updated_accounts = updated_accounts - .into_iter() - .map(|mock_account| (mock_account.id, mock_account.states[1])) - .collect(); - - Block { - header, - updated_accounts, - created_notes: Vec::new(), - produced_nullifiers: new_nullifiers, - } -} From 05385788412c6f80faea430f1137437f2b4b50d4 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 30 Nov 2023 14:26:37 -0500 Subject: [PATCH 139/166] prove: compute proper block num --- block-producer/src/block_builder/prover/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index dee7eb7b5..883fa14f1 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -1,6 +1,6 @@ use std::time::{SystemTime, UNIX_EPOCH}; -use miden_air::{ExecutionOptions, Felt}; +use miden_air::{ExecutionOptions, Felt, FieldElement}; use miden_objects::{assembly::Assembler, BlockHeader, Digest}; use miden_stdlib::StdLibrary; use miden_vm::{execute, DefaultHost, MemAdviceProvider, Program}; @@ -177,7 +177,7 @@ impl BlockProver { witness: BlockWitness, ) -> Result { let prev_hash = witness.prev_header.prev_hash(); - let block_num = witness.prev_header.block_num(); + let block_num = witness.prev_header.block_num() + Felt::ONE; let version = witness.prev_header.version(); let (account_root, note_root, chain_root) = self.compute_roots(witness)?; From 2920a57876860729a09fbfec41ea87c57e4156fc Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 30 Nov 2023 14:28:04 -0500 Subject: [PATCH 140/166] test_compute_chain_mmr_root_empty_mmr --- .../src/block_builder/prover/tests.rs | 58 ++++++------------- 1 file changed, 18 insertions(+), 40 deletions(-) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index db99051d6..e2e4c0d06 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -487,53 +487,31 @@ async fn test_compute_note_root_success() { // - add header to non-empty MMR (1 peak), and check that we get the expected commitment // - add header to MMR with 17 peaks -/// Test that the current account root is returned if the batches are empty +/// Test that the chain mmr root is as expected if the batches are empty #[tokio::test] async fn test_compute_chain_mmr_root_empty_mmr() { - // Set up account states - // --------------------------------------------------------------------------------------------- - let account_ids = [ - AccountId::new_unchecked(Felt::from(0b0000_0000_0000_0000u64)), - AccountId::new_unchecked(Felt::from(0b1111_0000_0000_0000u64)), - AccountId::new_unchecked(Felt::from(0b1111_1111_0000_0000u64)), - AccountId::new_unchecked(Felt::from(0b1111_1111_1111_0000u64)), - AccountId::new_unchecked(Felt::from(0b1111_1111_1111_1111u64)), - ]; + let store = MockStoreSuccessBuilder::new().build(); - let account_initial_states = [ - [Felt::from(1u64), Felt::from(1u64), Felt::from(1u64), Felt::from(1u64)], - [Felt::from(2u64), Felt::from(2u64), Felt::from(2u64), Felt::from(2u64)], - [Felt::from(3u64), Felt::from(3u64), Felt::from(3u64), Felt::from(3u64)], - [Felt::from(4u64), Felt::from(4u64), Felt::from(4u64), Felt::from(4u64)], - [Felt::from(5u64), Felt::from(5u64), Felt::from(5u64), Felt::from(5u64)], - ]; + // Compute actual chain MMR root, from applying an empty block + let actual_chain_mmr_root = { + let block_inputs_from_store: BlockInputs = + store.get_block_inputs(std::iter::empty(), std::iter::empty()).await.unwrap(); - // Set up store's account SMT - // --------------------------------------------------------------------------------------------- + let batches = Vec::new(); + let block_witness = BlockWitness::new(block_inputs_from_store, batches).unwrap(); - let store = MockStoreSuccessBuilder::new() - .initial_accounts( - account_ids - .iter() - .zip(account_initial_states.iter()) - .map(|(&account_id, &account_hash)| (account_id, account_hash.into())), - ) - .build(); + let block_prover = BlockProver::new(); + let block_header = block_prover.prove(block_witness).unwrap(); - // Block prover - // --------------------------------------------------------------------------------------------- - - // Block inputs is initialized with all the accounts and their initial state - let block_inputs_from_store: BlockInputs = - store.get_block_inputs(std::iter::empty(), std::iter::empty()).await.unwrap(); + block_header.chain_root() + }; - let batches = Vec::new(); - let block_witness = BlockWitness::new(block_inputs_from_store, batches).unwrap(); + // Compute expected chain MMR root + let expected_chain_mmr_root = { + let expected_block = MockBlockBuilder::new(&store).await.build(); - let block_prover = BlockProver::new(); - let block_header = block_prover.prove(block_witness).unwrap(); + expected_block.header.chain_root() + }; - // Compare roots - // --------------------------------------------------------------------------------------------- - assert_eq!(block_header.account_root(), store.account_root().await); + assert_eq!(actual_chain_mmr_root, expected_chain_mmr_root); } From 4e24eb43f9b4bed35926bb4be4cdfd8e40683ec9 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 30 Nov 2023 14:35:26 -0500 Subject: [PATCH 141/166] fix block prev hash --- block-producer/src/block_builder/prover/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 883fa14f1..2c5dd40fe 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -176,7 +176,7 @@ impl BlockProver { &self, witness: BlockWitness, ) -> Result { - let prev_hash = witness.prev_header.prev_hash(); + let prev_hash = witness.prev_header.hash(); let block_num = witness.prev_header.block_num() + Felt::ONE; let version = witness.prev_header.version(); From 707a5662923ce86dee6a770dea0bed4bdb93cd02 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 30 Nov 2023 14:44:34 -0500 Subject: [PATCH 142/166] fixme --- block-producer/src/block_builder/prover/tests.rs | 6 ++---- block-producer/src/test_utils/store.rs | 1 + 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index e2e4c0d06..fd2aff0e8 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -13,7 +13,7 @@ use miden_vm::crypto::{MerklePath, SimpleSmt}; use crate::{ batch_builder::TransactionBatch, block_builder::prover::block_witness::CREATED_NOTES_TREE_DEPTH, - store::{ApplyBlock, Store}, + store::Store, test_utils::{block::MockBlockBuilder, DummyProvenTxGenerator, MockStoreSuccessBuilder}, SharedTxBatch, }; @@ -265,11 +265,9 @@ async fn test_compute_account_root_success() { ) .build(); - store.apply_block(Arc::new(block)).await.unwrap(); - // Compare roots // --------------------------------------------------------------------------------------------- - assert_eq!(block_header.account_root(), store.account_root().await); + assert_eq!(block_header.account_root(), block.header.account_root()); } /// Test that the current account root is returned if the batches are empty diff --git a/block-producer/src/test_utils/store.rs b/block-producer/src/test_utils/store.rs index b513780ea..054e01e4e 100644 --- a/block-producer/src/test_utils/store.rs +++ b/block-producer/src/test_utils/store.rs @@ -74,6 +74,7 @@ impl MockStoreSuccessBuilder { chain_mmr.peaks(chain_mmr.forest()).unwrap().hash_peaks(), accounts_smt.root(), Digest::default(), + // FIXME: FILL IN CORRECT VALUE Digest::default(), Digest::default(), Digest::default(), From 5e0e68f16e9f9073ec005005ad8da5fc647ca97b Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Thu, 30 Nov 2023 16:28:57 -0500 Subject: [PATCH 143/166] add comment --- block-producer/src/block_builder/prover/block_witness.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/block-producer/src/block_builder/prover/block_witness.rs b/block-producer/src/block_builder/prover/block_witness.rs index eb7f45cf3..03b082e1b 100644 --- a/block-producer/src/block_builder/prover/block_witness.rs +++ b/block-producer/src/block_builder/prover/block_witness.rs @@ -234,6 +234,9 @@ impl BlockWitness { // advice map data is expected to be: // [ NUM_LEAVES, peak_0, ..., peak{n-1}, ] + // + // TODO: use `self.chain_peaks.to_advice_inputs(&mut advice_inputs)` instead of this block + // when https://github.com/0xPolygonMiden/miden-base/pull/333 is merged let map_data = { // num leaves let num_leaves = From 4f36d4add11af5470f57b45f07df8b9d3524602f Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 1 Dec 2023 07:34:43 -0500 Subject: [PATCH 144/166] use ONE and ZERO --- block-producer/src/block_builder/prover/mod.rs | 6 +++--- block-producer/src/block_builder/prover/tests.rs | 14 +++++++------- block-producer/src/test_utils/block.rs | 9 ++++----- block-producer/src/test_utils/store.rs | 9 ++++----- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 2c5dd40fe..550377fea 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -1,7 +1,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; -use miden_air::{ExecutionOptions, Felt, FieldElement}; -use miden_objects::{assembly::Assembler, BlockHeader, Digest}; +use miden_air::{ExecutionOptions, Felt}; +use miden_objects::{assembly::Assembler, BlockHeader, Digest, ONE}; use miden_stdlib::StdLibrary; use miden_vm::{execute, DefaultHost, MemAdviceProvider, Program}; @@ -177,7 +177,7 @@ impl BlockProver { witness: BlockWitness, ) -> Result { let prev_hash = witness.prev_header.hash(); - let block_num = witness.prev_header.block_num() + Felt::ONE; + let block_num = witness.prev_header.block_num() + ONE; let version = witness.prev_header.version(); let (account_root, note_root, chain_root) = self.compute_roots(witness)?; diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index fd2aff0e8..bd75e0307 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -1,12 +1,12 @@ use std::sync::Arc; -use miden_air::FieldElement; use miden_mock::mock::block::mock_block_header; use miden_node_proto::domain::{AccountInputRecord, BlockInputs}; use miden_objects::{ accounts::AccountId, crypto::merkle::{EmptySubtreeRoots, MmrPeaks}, notes::{NoteEnvelope, NoteMetadata}, + ZERO, }; use miden_vm::crypto::{MerklePath, SimpleSmt}; @@ -30,12 +30,12 @@ use super::*; #[test] fn test_block_witness_validation_inconsistent_account_ids() { let tx_gen = DummyProvenTxGenerator::new(); - let account_id_1 = AccountId::new_unchecked(Felt::ZERO); - let account_id_2 = AccountId::new_unchecked(Felt::ONE); + let account_id_1 = AccountId::new_unchecked(ZERO); + let account_id_2 = AccountId::new_unchecked(ONE); let account_id_3 = AccountId::new_unchecked(Felt::new(42)); let block_inputs_from_store: BlockInputs = { - let block_header = mock_block_header(Felt::ZERO, None, None, &[]); + let block_header = mock_block_header(ZERO, None, None, &[]); let chain_peaks = MmrPeaks::new(0, Vec::new()).unwrap(); let account_states = vec![ @@ -102,8 +102,8 @@ fn test_block_witness_validation_inconsistent_account_ids() { #[test] fn test_block_witness_validation_inconsistent_account_hashes() { let tx_gen = DummyProvenTxGenerator::new(); - let account_id_1 = AccountId::new_unchecked(Felt::ZERO); - let account_id_2 = AccountId::new_unchecked(Felt::ONE); + let account_id_1 = AccountId::new_unchecked(ZERO); + let account_id_2 = AccountId::new_unchecked(ONE); let account_1_hash_store = Digest::new([Felt::from(1u64), Felt::from(2u64), Felt::from(3u64), Felt::from(4u64)]); @@ -111,7 +111,7 @@ fn test_block_witness_validation_inconsistent_account_hashes() { Digest::new([Felt::from(4u64), Felt::from(3u64), Felt::from(2u64), Felt::from(1u64)]); let block_inputs_from_store: BlockInputs = { - let block_header = mock_block_header(Felt::ZERO, None, None, &[]); + let block_header = mock_block_header(ZERO, None, None, &[]); let chain_peaks = MmrPeaks::new(0, Vec::new()).unwrap(); let account_states = vec![ diff --git a/block-producer/src/test_utils/block.rs b/block-producer/src/test_utils/block.rs index e57219a95..9d6502d10 100644 --- a/block-producer/src/test_utils/block.rs +++ b/block-producer/src/test_utils/block.rs @@ -1,5 +1,4 @@ -use miden_air::{Felt, FieldElement}; -use miden_objects::{accounts::AccountId, crypto::merkle::Mmr, BlockHeader, Digest}; +use miden_objects::{accounts::AccountId, crypto::merkle::Mmr, BlockHeader, Digest, ONE, ZERO}; use miden_vm::crypto::SimpleSmt; use crate::block::Block; @@ -66,15 +65,15 @@ impl MockBlockBuilder { pub fn build(self) -> Block { let header = BlockHeader::new( self.last_block_header.hash(), - self.last_block_header.block_num() + Felt::ONE, + self.last_block_header.block_num() + ONE, self.store_chain_mmr.peaks(self.store_chain_mmr.forest()).unwrap().hash_peaks(), self.store_accounts.root(), Digest::default(), Digest::default(), Digest::default(), Digest::default(), - Felt::ZERO, - Felt::ONE, + ZERO, + ONE, ); Block { diff --git a/block-producer/src/test_utils/store.rs b/block-producer/src/test_utils/store.rs index 054e01e4e..5d2dcb8be 100644 --- a/block-producer/src/test_utils/store.rs +++ b/block-producer/src/test_utils/store.rs @@ -1,9 +1,8 @@ use async_trait::async_trait; -use miden_air::{Felt, FieldElement}; use miden_node_proto::domain::{AccountInputRecord, BlockInputs}; use miden_objects::{ crypto::merkle::{Mmr, MmrPeaks}, - BlockHeader, EMPTY_WORD, + BlockHeader, EMPTY_WORD, ONE, ZERO, }; use miden_vm::crypto::SimpleSmt; @@ -70,7 +69,7 @@ impl MockStoreSuccessBuilder { let initial_block_header = BlockHeader::new( Digest::default(), - Felt::ZERO, + ZERO, chain_mmr.peaks(chain_mmr.forest()).unwrap().hash_peaks(), accounts_smt.root(), Digest::default(), @@ -78,8 +77,8 @@ impl MockStoreSuccessBuilder { Digest::default(), Digest::default(), Digest::default(), - Felt::ZERO, - Felt::ONE, + ZERO, + ONE, ); MockStoreSuccess { From c269dd758a5612db24bee36241d7dc073f337dde Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 1 Dec 2023 07:37:29 -0500 Subject: [PATCH 145/166] add sections --- .../src/block_builder/prover/block_witness.rs | 152 ++++++++++-------- 1 file changed, 81 insertions(+), 71 deletions(-) diff --git a/block-producer/src/block_builder/prover/block_witness.rs b/block-producer/src/block_builder/prover/block_witness.rs index 03b082e1b..c225765c5 100644 --- a/block-producer/src/block_builder/prover/block_witness.rs +++ b/block-producer/src/block_builder/prover/block_witness.rs @@ -17,6 +17,9 @@ use crate::{ SharedTxBatch, }; +// CONSTANTS +// ================================================================================================= + /// The depth at which we insert roots from the batches. pub(crate) const CREATED_NOTES_TREE_INSERTION_DEPTH: u8 = 8; @@ -26,6 +29,9 @@ pub(crate) const CREATED_NOTES_TREE_DEPTH: u8 = pub(crate) const MMR_MIN_NUM_PEAKS: usize = 16; +// BLOCK WITNESS +// ================================================================================================= + /// Provides inputs to the `BlockKernel` so that it can generate the new header #[derive(Debug, PartialEq)] pub struct BlockWitness { @@ -96,77 +102,6 @@ impl BlockWitness { }) } - fn validate_inputs( - block_inputs: &BlockInputs, - batches: &[SharedTxBatch], - ) -> Result<(), BuildBlockError> { - // TODO: - // - Block height returned for each nullifier is 0. - - // Validate that there aren't too many batches in the block. - if batches.len() > 2usize.pow(CREATED_NOTES_TREE_INSERTION_DEPTH.into()) { - return Err(BuildBlockError::TooManyBatchesInBlock(batches.len())); - } - - Self::validate_account_states(block_inputs, batches)?; - - Ok(()) - } - - /// Validate that initial account states coming from the batches are the same as the account - /// states returned from the store - fn validate_account_states( - block_inputs: &BlockInputs, - batches: &[SharedTxBatch], - ) -> Result<(), BuildBlockError> { - let batches_initial_states: BTreeMap = - batches.iter().flat_map(|batch| batch.account_initial_states()).collect(); - - let accounts_in_batches: BTreeSet = - batches_initial_states.keys().cloned().collect(); - let accounts_in_store: BTreeSet = block_inputs - .account_states - .iter() - .map(|record| &record.account_id) - .cloned() - .collect(); - - if accounts_in_batches == accounts_in_store { - let accounts_with_different_hashes: Vec = block_inputs - .account_states - .iter() - .filter_map(|record| { - let hash_in_store = record.account_hash; - let hash_in_batches = batches_initial_states - .get(&record.account_id) - .expect("we already verified that account id is contained in batches"); - - if hash_in_store == *hash_in_batches { - None - } else { - Some(record.account_id) - } - }) - .collect(); - - if accounts_with_different_hashes.is_empty() { - Ok(()) - } else { - Err(BuildBlockError::InconsistentAccountStates(accounts_with_different_hashes)) - } - } else { - // The batches and store don't modify the same set of accounts - let union: BTreeSet = - accounts_in_batches.union(&accounts_in_store).cloned().collect(); - let intersection: BTreeSet = - accounts_in_batches.intersection(&accounts_in_store).cloned().collect(); - - let difference: Vec = union.difference(&intersection).cloned().collect(); - - Err(BuildBlockError::InconsistentAccountIds(difference)) - } - } - pub(super) fn into_parts(self) -> Result<(AdviceInputs, StackInputs), BlockProverError> { let stack_inputs = { // Note: `StackInputs::new()` reverses the input vector, so we need to construct the stack @@ -267,6 +202,81 @@ impl BlockWitness { Ok((advice_inputs, stack_inputs)) } + + // HELPERS + // --------------------------------------------------------------------------------------------- + + fn validate_inputs( + block_inputs: &BlockInputs, + batches: &[SharedTxBatch], + ) -> Result<(), BuildBlockError> { + // TODO: + // - Block height returned for each nullifier is 0. + + // Validate that there aren't too many batches in the block. + if batches.len() > 2usize.pow(CREATED_NOTES_TREE_INSERTION_DEPTH.into()) { + return Err(BuildBlockError::TooManyBatchesInBlock(batches.len())); + } + + Self::validate_account_states(block_inputs, batches)?; + + Ok(()) + } + + /// Validate that initial account states coming from the batches are the same as the account + /// states returned from the store + fn validate_account_states( + block_inputs: &BlockInputs, + batches: &[SharedTxBatch], + ) -> Result<(), BuildBlockError> { + let batches_initial_states: BTreeMap = + batches.iter().flat_map(|batch| batch.account_initial_states()).collect(); + + let accounts_in_batches: BTreeSet = + batches_initial_states.keys().cloned().collect(); + let accounts_in_store: BTreeSet = block_inputs + .account_states + .iter() + .map(|record| &record.account_id) + .cloned() + .collect(); + + if accounts_in_batches == accounts_in_store { + let accounts_with_different_hashes: Vec = block_inputs + .account_states + .iter() + .filter_map(|record| { + let hash_in_store = record.account_hash; + let hash_in_batches = batches_initial_states + .get(&record.account_id) + .expect("we already verified that account id is contained in batches"); + + if hash_in_store == *hash_in_batches { + None + } else { + Some(record.account_id) + } + }) + .collect(); + + if accounts_with_different_hashes.is_empty() { + Ok(()) + } else { + Err(BuildBlockError::InconsistentAccountStates(accounts_with_different_hashes)) + } + } else { + // The batches and store don't modify the same set of accounts + let union: BTreeSet = + accounts_in_batches.union(&accounts_in_store).cloned().collect(); + let intersection: BTreeSet = + accounts_in_batches.intersection(&accounts_in_store).cloned().collect(); + + let difference: Vec = union.difference(&intersection).cloned().collect(); + + Err(BuildBlockError::InconsistentAccountIds(difference)) + } + } + } #[derive(Debug, PartialEq, Eq)] From bd2a09e1ef5ba8fb19d1b42548a24aa2c378c0fb Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 1 Dec 2023 07:38:06 -0500 Subject: [PATCH 146/166] into_program_inputs --- block-producer/src/block_builder/prover/block_witness.rs | 5 +++-- block-producer/src/block_builder/prover/mod.rs | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/block-producer/src/block_builder/prover/block_witness.rs b/block-producer/src/block_builder/prover/block_witness.rs index c225765c5..8cb8312a9 100644 --- a/block-producer/src/block_builder/prover/block_witness.rs +++ b/block-producer/src/block_builder/prover/block_witness.rs @@ -102,7 +102,9 @@ impl BlockWitness { }) } - pub(super) fn into_parts(self) -> Result<(AdviceInputs, StackInputs), BlockProverError> { + pub(super) fn into_program_inputs( + self + ) -> Result<(AdviceInputs, StackInputs), BlockProverError> { let stack_inputs = { // Note: `StackInputs::new()` reverses the input vector, so we need to construct the stack // from the bottom to the top @@ -276,7 +278,6 @@ impl BlockWitness { Err(BuildBlockError::InconsistentAccountIds(difference)) } } - } #[derive(Debug, PartialEq, Eq)] diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 550377fea..bbd5d7f7a 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -209,7 +209,7 @@ impl BlockProver { &self, witness: BlockWitness, ) -> Result<(Digest, Digest, Digest), BlockProverError> { - let (advice_inputs, stack_inputs) = witness.into_parts()?; + let (advice_inputs, stack_inputs) = witness.into_program_inputs()?; let host = { let advice_provider = MemAdviceProvider::from(advice_inputs); From b8f22657961e7521ff83138ffc3372cfe69edd20 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 1 Dec 2023 07:45:03 -0500 Subject: [PATCH 147/166] use to_advice_inputs --- .../src/block_builder/prover/block_witness.rs | 44 +++---------------- 1 file changed, 6 insertions(+), 38 deletions(-) diff --git a/block-producer/src/block_builder/prover/block_witness.rs b/block-producer/src/block_builder/prover/block_witness.rs index 8cb8312a9..bd30696d9 100644 --- a/block-producer/src/block_builder/prover/block_witness.rs +++ b/block-producer/src/block_builder/prover/block_witness.rs @@ -1,13 +1,10 @@ -use std::{ - cmp::max, - collections::{BTreeMap, BTreeSet}, -}; +use std::collections::{BTreeMap, BTreeSet}; use miden_node_proto::domain::BlockInputs; use miden_objects::{ accounts::AccountId, crypto::merkle::{EmptySubtreeRoots, MerkleStore, MmrPeaks}, - BlockHeader, Digest, Felt, ZERO, + BlockHeader, Digest, Felt, ToAdviceInputs, }; use miden_vm::{crypto::MerklePath, AdviceInputs, StackInputs}; @@ -27,8 +24,6 @@ pub(crate) const CREATED_NOTES_TREE_INSERTION_DEPTH: u8 = 8; pub(crate) const CREATED_NOTES_TREE_DEPTH: u8 = CREATED_NOTES_TREE_INSERTION_DEPTH + batch_builder::CREATED_NOTES_SMT_DEPTH; -pub(crate) const MMR_MIN_NUM_PEAKS: usize = 16; - // BLOCK WITNESS // ================================================================================================= @@ -169,37 +164,10 @@ impl BlockWitness { )) .map_err(BlockProverError::InvalidMerklePaths)?; - // advice map data is expected to be: - // [ NUM_LEAVES, peak_0, ..., peak{n-1}, ] - // - // TODO: use `self.chain_peaks.to_advice_inputs(&mut advice_inputs)` instead of this block - // when https://github.com/0xPolygonMiden/miden-base/pull/333 is merged - let map_data = { - // num leaves - let num_leaves = - [Felt::from(self.chain_peaks.num_leaves() as u64), ZERO, ZERO, ZERO]; - - // peaks - let padding_peaks = { - let num_padding_peaks = max(MMR_MIN_NUM_PEAKS, self.chain_peaks.peaks().len()); - - vec![Digest::default(); num_padding_peaks] - }; - - let all_peaks_including_padding = - self.chain_peaks.peaks().iter().chain(padding_peaks.iter()); - - // fill out map data - let mut map_data: Vec = Vec::new(); - map_data.extend(num_leaves); - map_data.extend(all_peaks_including_padding.flat_map(|peak| peak.iter())); - - map_data - }; - - AdviceInputs::default() - .with_merkle_store(merkle_store) - .with_map([(self.chain_peaks.hash_peaks().into(), map_data)]) + let mut advice_inputs = AdviceInputs::default().with_merkle_store(merkle_store); + self.chain_peaks.to_advice_inputs(&mut advice_inputs); + + advice_inputs }; Ok((advice_inputs, stack_inputs)) From 9f1e01cefd44400538cce57f6906308ec42a360e Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 1 Dec 2023 11:54:43 -0500 Subject: [PATCH 148/166] MockPrivateAccount: support u32 indices --- block-producer/src/state_view/tests/mod.rs | 14 +++++++++++--- block-producer/src/test_utils/account.rs | 9 ++++----- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/block-producer/src/state_view/tests/mod.rs b/block-producer/src/state_view/tests/mod.rs index 73b97f0c5..d795a0e2b 100644 --- a/block-producer/src/state_view/tests/mod.rs +++ b/block-producer/src/state_view/tests/mod.rs @@ -10,15 +10,23 @@ mod verify_tx; // HELPERS // ------------------------------------------------------------------------------------------------- -pub fn consumed_note_by_index(index: u8) -> ConsumedNoteInfo { - ConsumedNoteInfo::new(Hasher::hash(&[index]), Hasher::hash(&[index, index])) +pub fn consumed_note_by_index(index: u32) -> ConsumedNoteInfo { + ConsumedNoteInfo::new( + Hasher::hash(&index.to_be_bytes()), + Hasher::hash( + &[index.to_be_bytes(), index.to_be_bytes()] + .into_iter() + .flatten() + .collect::>(), + ), + ) } /// Returns `num` transactions, and the corresponding account they modify. /// The transactions each consume a single different note pub fn get_txs_and_accounts( tx_gen: &DummyProvenTxGenerator, - num: u8, + num: u32, ) -> impl Iterator + '_ { (0..num).map(|index| { let account = MockPrivateAccount::from(index); diff --git a/block-producer/src/test_utils/account.rs b/block-producer/src/test_utils/account.rs index a54430683..d5355dffb 100644 --- a/block-producer/src/test_utils/account.rs +++ b/block-producer/src/test_utils/account.rs @@ -37,12 +37,11 @@ impl MockPrivateAccount { } } -impl From for MockPrivateAccount { +impl From for MockPrivateAccount { /// Each index gives rise to a different account ID - fn from(index: u8) -> Self { - let mut init_seed: [u8; 32] = [0; 32]; - init_seed[0] = index; + fn from(index: u32) -> Self { + let init_seed: Vec<_> = index.to_be_bytes().into_iter().chain([0u8; 28]).collect(); - Self::new(init_seed) + Self::new(init_seed.try_into().unwrap()) } } From 7f681fbd852db6801e4d1ff74a3d9d55479e0445 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 1 Dec 2023 12:01:03 -0500 Subject: [PATCH 149/166] MockProvenTxBuilder --- block-producer/Cargo.toml | 8 ++- block-producer/src/test_utils/mod.rs | 2 +- block-producer/src/test_utils/proven_tx.rs | 74 +++++++++++++++++++++- 3 files changed, 78 insertions(+), 6 deletions(-) diff --git a/block-producer/Cargo.toml b/block-producer/Cargo.toml index cb88c54d4..2e48a4593 100644 --- a/block-producer/Cargo.toml +++ b/block-producer/Cargo.toml @@ -10,13 +10,15 @@ edition = "2021" rust-version = "1.73" [dev-dependencies] +miden-crypto = { workspace = true } miden-mock = { package = "miden-mock", git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "main", default-features = false } +once_cell = { version = "1.18" } winterfell = "0.7" [dependencies] anyhow = { version = "1.0" } async-trait = { version = "0.1" } -clap = { version = "4.3" , features = ["derive"] } +clap = { version = "4.3", features = ["derive"] } figment = { version = "0.10", features = ["toml", "env"] } itertools = { version = "0.12" } miden-air = { package = "miden-air", git = "https://github.com/0xPolygonMiden/miden-vm.git", branch = "next", default-features = false } @@ -25,10 +27,10 @@ miden-node-utils = { path = "../utils" } miden_objects = { workspace = true } miden_stdlib = { package = "miden-stdlib", git = "https://github.com/0xPolygonMiden/miden-vm.git", branch = "next", default-features = false } miden_vm = { package = "miden-vm", git = "https://github.com/0xPolygonMiden/miden-vm.git", branch = "next", default-features = false } -serde = { version = "1.0" , features = ["derive"] } +serde = { version = "1.0", features = ["derive"] } thiserror = { workspace = true } tokio = { version = "1.29", features = ["rt-multi-thread", "net", "macros", "sync", "time"] } toml = { version = "0.8" } tonic = { version = "0.10" } tracing = { version = "0.1" } -tracing-subscriber = { version = "0.3" , features = ["fmt"] } +tracing-subscriber = { version = "0.3", features = ["fmt"] } diff --git a/block-producer/src/test_utils/mod.rs b/block-producer/src/test_utils/mod.rs index d21bf6bc1..db39be9e8 100644 --- a/block-producer/src/test_utils/mod.rs +++ b/block-producer/src/test_utils/mod.rs @@ -5,7 +5,7 @@ use tokio::sync::RwLock; use miden_objects::{accounts::AccountId, Digest}; mod proven_tx; -pub use proven_tx::DummyProvenTxGenerator; +pub use proven_tx::{DummyProvenTxGenerator, MockProvenTxBuilder}; mod store; pub use store::{MockStoreFailure, MockStoreSuccess, MockStoreSuccessBuilder}; diff --git a/block-producer/src/test_utils/proven_tx.rs b/block-producer/src/test_utils/proven_tx.rs index b4c672745..d30b704b9 100644 --- a/block-producer/src/test_utils/proven_tx.rs +++ b/block-producer/src/test_utils/proven_tx.rs @@ -1,13 +1,17 @@ //! FibSmall taken from the `fib_small` example in `winterfell` +use std::sync::{Arc, Mutex}; + use miden_air::{ExecutionProof, HashFunction}; +use miden_crypto::hash::rpo::Rpo256; use miden_mock::constants::ACCOUNT_ID_REGULAR_ACCOUNT_UPDATABLE_CODE_ON_CHAIN; use miden_objects::{ accounts::AccountId, - notes::NoteEnvelope, + notes::{NoteEnvelope, NoteMetadata}, transaction::{ConsumedNoteInfo, ProvenTransaction}, - Digest, + Digest, ONE, ZERO, }; +use once_cell::sync::Lazy; use winterfell::{ crypto::{hashers::Blake3_192, DefaultRandomCoin}, math::fields::f64::BaseElement, @@ -19,6 +23,72 @@ use winterfell::{ TransitionConstraintDegree, }; +use super::MockPrivateAccount; + +/// Keeps track how many accounts were created as a source of randomness +static NUM_ACCOUNTS_CREATED: Lazy>> = Lazy::new(|| Arc::new(Mutex::new(0))); + +/// Keeps track how many accounts were created as a source of randomness +static NUM_NOTES_CREATED: Lazy>> = Lazy::new(|| Arc::new(Mutex::new(0))); + +pub struct MockProvenTxBuilder { + mock_account: MockPrivateAccount, + notes_created: Option>, +} + +impl MockProvenTxBuilder { + pub fn new() -> Self { + let account_index: u32 = { + let mut locked_num_accounts_created = NUM_ACCOUNTS_CREATED.lock().unwrap(); + + let account_index = *locked_num_accounts_created; + + *locked_num_accounts_created += 1; + + account_index + }; + Self { + mock_account: account_index.into(), + notes_created: None, + } + } + + pub fn num_notes_created( + mut self, + num_notes_created_in_tx: u64, + ) -> Self { + let mut locked_num_notes_created = NUM_NOTES_CREATED.lock().unwrap(); + + let notes_created: Vec<_> = (*locked_num_notes_created + ..(*locked_num_notes_created + num_notes_created_in_tx)) + .map(|note_index| { + let note_hash = Rpo256::hash(¬e_index.to_be_bytes()); + + NoteEnvelope::new(note_hash, NoteMetadata::new(self.mock_account.id, ONE, ZERO)) + }) + .collect(); + + // update state + self.notes_created = Some(notes_created); + *locked_num_notes_created += num_notes_created_in_tx; + + self + } + + pub fn build(self) -> ProvenTransaction { + ProvenTransaction::new( + self.mock_account.id, + self.mock_account.states[0], + self.mock_account.states[1], + Vec::new(), + self.notes_created.unwrap_or_default(), + None, + Digest::default(), + ExecutionProof::new(StarkProof::new_dummy(), HashFunction::Blake3_192), + ) + } +} + /// We need to generate a new `ProvenTransaction` every time because it doesn't /// derive `Clone`. Doing it this way allows us to compute the `StarkProof` /// once, and clone it for each new `ProvenTransaction`. From 7c8f245b21c4ca27e70511535998e6c7fe49daba Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 1 Dec 2023 12:50:36 -0500 Subject: [PATCH 150/166] batch mod --- block-producer/src/test_utils/batch.rs | 0 block-producer/src/test_utils/mod.rs | 2 ++ 2 files changed, 2 insertions(+) create mode 100644 block-producer/src/test_utils/batch.rs diff --git a/block-producer/src/test_utils/batch.rs b/block-producer/src/test_utils/batch.rs new file mode 100644 index 000000000..e69de29bb diff --git a/block-producer/src/test_utils/mod.rs b/block-producer/src/test_utils/mod.rs index db39be9e8..8ddf42a7c 100644 --- a/block-producer/src/test_utils/mod.rs +++ b/block-producer/src/test_utils/mod.rs @@ -14,3 +14,5 @@ mod account; pub use account::MockPrivateAccount; pub mod block; + +pub mod batch; From 5f233f87cee5d5cb047bbb493c5161c064806acd Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 1 Dec 2023 13:12:05 -0500 Subject: [PATCH 151/166] TransactionBatchConstructor --- block-producer/src/test_utils/batch.rs | 33 ++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/block-producer/src/test_utils/batch.rs b/block-producer/src/test_utils/batch.rs index e69de29bb..c66bf0181 100644 --- a/block-producer/src/test_utils/batch.rs +++ b/block-producer/src/test_utils/batch.rs @@ -0,0 +1,33 @@ +use std::sync::Arc; + +use crate::{batch_builder::TransactionBatch, test_utils::MockProvenTxBuilder}; + +pub trait TransactionBatchConstructor { + /// Returns a `TransactionBatch` with `notes_per_tx.len()` transactions, where the i'th + /// transaction has `notes_per_tx[i]` notes created + fn from_notes_created(notes_per_tx: &[u64]) -> Self; + + /// Returns a `TransactionBatch` which contains `num_txs_in_batch` transactions + fn from_txs(num_txs_in_batch: u64) -> Self; +} + +impl TransactionBatchConstructor for TransactionBatch { + fn from_notes_created(notes_per_tx: &[u64]) -> Self { + let txs: Vec<_> = notes_per_tx + .iter() + .map(|&num_notes| MockProvenTxBuilder::new().num_notes_created(num_notes).build()) + .map(Arc::new) + .collect(); + + Self::new(txs).unwrap() + } + + fn from_txs(num_txs_in_batch: u64) -> Self { + let txs: Vec<_> = (0..num_txs_in_batch) + .map(|_| MockProvenTxBuilder::new().build()) + .map(Arc::new) + .collect(); + + Self::new(txs).unwrap() + } +} From 3e660cec3e7d5a79781f474df559eeaafffd117a Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 1 Dec 2023 13:45:24 -0500 Subject: [PATCH 152/166] build_expected_block() --- block-producer/src/test_utils/block.rs | 67 +++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/block-producer/src/test_utils/block.rs b/block-producer/src/test_utils/block.rs index 9d6502d10..21e890ff7 100644 --- a/block-producer/src/test_utils/block.rs +++ b/block-producer/src/test_utils/block.rs @@ -1,10 +1,75 @@ use miden_objects::{accounts::AccountId, crypto::merkle::Mmr, BlockHeader, Digest, ONE, ZERO}; use miden_vm::crypto::SimpleSmt; -use crate::block::Block; +use crate::{batch_builder::TransactionBatch, block::Block}; use super::MockStoreSuccess; +/// Constructs the block we expect to be built given the store state, and a set of transaction +/// batches to be applied +pub async fn build_expected_block( + store: &MockStoreSuccess, + batches: Vec, +) -> Block { + let last_block_header = *store.last_block_header.read().await; + + // Compute new account root + let updated_accounts: Vec<(AccountId, Digest)> = + batches.iter().flat_map(|batch| batch.updated_accounts()).collect(); + let new_account_root = { + let mut store_accounts = store.accounts.read().await.clone(); + for (account_id, new_account_state) in updated_accounts.iter() { + store_accounts + .update_leaf(u64::from(*account_id), new_account_state.into()) + .unwrap(); + } + + store_accounts.root() + }; + + // Compute created notes root + // FIXME: compute the right root. Needs + // https://github.com/0xPolygonMiden/crypto/issues/220#issuecomment-1823911017 + let created_notes: Vec = + batches.iter().flat_map(|batch| batch.created_notes()).collect(); + let new_created_notes_root = Digest::default(); + + // Compute new chain MMR root + let new_chain_mmr_root = { + let mut store_chain_mmr = store.chain_mmr.read().await.clone(); + + store_chain_mmr.add(last_block_header.hash()); + + store_chain_mmr.peaks(store_chain_mmr.forest()).unwrap().hash_peaks() + }; + + // Build header + let header = BlockHeader::new( + last_block_header.hash(), + last_block_header.block_num() + ONE, + new_chain_mmr_root, + new_account_root, + // FIXME: FILL IN CORRECT NULLIFIER ROOT + Digest::default(), + // FIXME: FILL IN CORRECT CREATED NOTES ROOT + new_created_notes_root, + Digest::default(), + Digest::default(), + ZERO, + ONE, + ); + + let produced_nullifiers: Vec = + batches.iter().flat_map(|batch| batch.produced_nullifiers()).collect(); + + Block { + header, + updated_accounts, + created_notes, + produced_nullifiers, + } +} + #[derive(Debug)] pub struct MockBlockBuilder { store_accounts: SimpleSmt, From 97a18baf744d3bb849ce90f9122548f715365ebb Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 1 Dec 2023 14:11:28 -0500 Subject: [PATCH 153/166] build_actual_block_header --- block-producer/src/block_builder/mod.rs | 2 +- .../src/block_builder/prover/mod.rs | 2 +- .../src/block_builder/prover/tests.rs | 13 +++---- block-producer/src/test_utils/block.rs | 38 ++++++++++++++++++- 4 files changed, 44 insertions(+), 11 deletions(-) diff --git a/block-producer/src/block_builder/mod.rs b/block-producer/src/block_builder/mod.rs index 34fdfe2ca..d6d4e2431 100644 --- a/block-producer/src/block_builder/mod.rs +++ b/block-producer/src/block_builder/mod.rs @@ -7,7 +7,7 @@ use crate::{block::Block, store::Store, SharedTxBatch}; pub mod errors; -mod prover; +pub(crate) mod prover; use self::{ errors::BuildBlockError, prover::{block_witness::BlockWitness, BlockProver}, diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index bbd5d7f7a..9bcdd7fb0 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -150,7 +150,7 @@ end "; #[derive(Debug)] -pub(super) struct BlockProver { +pub(crate) struct BlockProver { kernel: Program, } diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index bd75e0307..f07f7fcf7 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -14,7 +14,10 @@ use crate::{ batch_builder::TransactionBatch, block_builder::prover::block_witness::CREATED_NOTES_TREE_DEPTH, store::Store, - test_utils::{block::MockBlockBuilder, DummyProvenTxGenerator, MockStoreSuccessBuilder}, + test_utils::{ + block::{build_expected_block, MockBlockBuilder}, + DummyProvenTxGenerator, MockStoreSuccessBuilder, + }, SharedTxBatch, }; @@ -505,11 +508,7 @@ async fn test_compute_chain_mmr_root_empty_mmr() { }; // Compute expected chain MMR root - let expected_chain_mmr_root = { - let expected_block = MockBlockBuilder::new(&store).await.build(); + let expected_block = build_expected_block(&store, &[]).await; - expected_block.header.chain_root() - }; - - assert_eq!(actual_chain_mmr_root, expected_chain_mmr_root); + assert_eq!(actual_chain_mmr_root, expected_block.header.chain_root()); } diff --git a/block-producer/src/test_utils/block.rs b/block-producer/src/test_utils/block.rs index 21e890ff7..3923170db 100644 --- a/block-producer/src/test_utils/block.rs +++ b/block-producer/src/test_utils/block.rs @@ -1,7 +1,15 @@ +use std::sync::Arc; + +use miden_node_proto::domain::BlockInputs; use miden_objects::{accounts::AccountId, crypto::merkle::Mmr, BlockHeader, Digest, ONE, ZERO}; use miden_vm::crypto::SimpleSmt; -use crate::{batch_builder::TransactionBatch, block::Block}; +use crate::{ + batch_builder::TransactionBatch, + block::Block, + block_builder::prover::{block_witness::BlockWitness, BlockProver}, + store::Store, +}; use super::MockStoreSuccess; @@ -9,7 +17,7 @@ use super::MockStoreSuccess; /// batches to be applied pub async fn build_expected_block( store: &MockStoreSuccess, - batches: Vec, + batches: &[TransactionBatch], ) -> Block { let last_block_header = *store.last_block_header.read().await; @@ -70,6 +78,32 @@ pub async fn build_expected_block( } } +/// Builds the "actual" block header; i.e. the block header built using the Miden VM, used in the +/// node +pub async fn build_actual_block_header( + store: &MockStoreSuccess, + batches: Vec, +) -> BlockHeader { + let updated_accounts: Vec<(AccountId, Digest)> = + batches.iter().flat_map(|batch| batch.updated_accounts()).collect(); + let produced_nullifiers: Vec = + batches.iter().flat_map(|batch| batch.produced_nullifiers()).collect(); + + let block_inputs_from_store: BlockInputs = store + .get_block_inputs( + updated_accounts.iter().map(|(account_id, _)| account_id), + produced_nullifiers.iter(), + ) + .await + .unwrap(); + + let block_witness = + BlockWitness::new(block_inputs_from_store, batches.into_iter().map(Arc::new).collect()) + .unwrap(); + + BlockProver::new().prove(block_witness).unwrap() +} + #[derive(Debug)] pub struct MockBlockBuilder { store_accounts: SimpleSmt, From c1d2875a21340340d890adf4382795564856037e Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 1 Dec 2023 14:13:05 -0500 Subject: [PATCH 154/166] build_expected_block_header --- .../src/block_builder/prover/tests.rs | 6 +++--- block-producer/src/test_utils/block.rs | 20 ++++--------------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index f07f7fcf7..8667ee7de 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -15,7 +15,7 @@ use crate::{ block_builder::prover::block_witness::CREATED_NOTES_TREE_DEPTH, store::Store, test_utils::{ - block::{build_expected_block, MockBlockBuilder}, + block::{build_expected_block_header, MockBlockBuilder}, DummyProvenTxGenerator, MockStoreSuccessBuilder, }, SharedTxBatch, @@ -508,7 +508,7 @@ async fn test_compute_chain_mmr_root_empty_mmr() { }; // Compute expected chain MMR root - let expected_block = build_expected_block(&store, &[]).await; + let expected_block_header = build_expected_block_header(&store, &[]).await; - assert_eq!(actual_chain_mmr_root, expected_block.header.chain_root()); + assert_eq!(actual_chain_mmr_root, expected_block_header.chain_root()); } diff --git a/block-producer/src/test_utils/block.rs b/block-producer/src/test_utils/block.rs index 3923170db..9d43260a9 100644 --- a/block-producer/src/test_utils/block.rs +++ b/block-producer/src/test_utils/block.rs @@ -15,10 +15,10 @@ use super::MockStoreSuccess; /// Constructs the block we expect to be built given the store state, and a set of transaction /// batches to be applied -pub async fn build_expected_block( +pub async fn build_expected_block_header( store: &MockStoreSuccess, batches: &[TransactionBatch], -) -> Block { +) -> BlockHeader { let last_block_header = *store.last_block_header.read().await; // Compute new account root @@ -38,8 +38,6 @@ pub async fn build_expected_block( // Compute created notes root // FIXME: compute the right root. Needs // https://github.com/0xPolygonMiden/crypto/issues/220#issuecomment-1823911017 - let created_notes: Vec = - batches.iter().flat_map(|batch| batch.created_notes()).collect(); let new_created_notes_root = Digest::default(); // Compute new chain MMR root @@ -52,7 +50,7 @@ pub async fn build_expected_block( }; // Build header - let header = BlockHeader::new( + BlockHeader::new( last_block_header.hash(), last_block_header.block_num() + ONE, new_chain_mmr_root, @@ -65,17 +63,7 @@ pub async fn build_expected_block( Digest::default(), ZERO, ONE, - ); - - let produced_nullifiers: Vec = - batches.iter().flat_map(|batch| batch.produced_nullifiers()).collect(); - - Block { - header, - updated_accounts, - created_notes, - produced_nullifiers, - } + ) } /// Builds the "actual" block header; i.e. the block header built using the Miden VM, used in the From 5df94af9d474799255eb9db650bdeaddb34d5ef6 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 1 Dec 2023 14:14:44 -0500 Subject: [PATCH 155/166] test_compute_chain_mmr_root_empty_mmr --- .../src/block_builder/prover/tests.rs | 21 +++---------------- 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index 8667ee7de..4c32a2ba0 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -15,7 +15,7 @@ use crate::{ block_builder::prover::block_witness::CREATED_NOTES_TREE_DEPTH, store::Store, test_utils::{ - block::{build_expected_block_header, MockBlockBuilder}, + block::{build_actual_block_header, build_expected_block_header, MockBlockBuilder}, DummyProvenTxGenerator, MockStoreSuccessBuilder, }, SharedTxBatch, @@ -484,7 +484,6 @@ async fn test_compute_note_root_success() { // CHAIN MMR ROOT TESTS // ================================================================================================= -// - add header to empty MMR, and check that we get the expected commitment // - add header to non-empty MMR (1 peak), and check that we get the expected commitment // - add header to MMR with 17 peaks @@ -493,22 +492,8 @@ async fn test_compute_note_root_success() { async fn test_compute_chain_mmr_root_empty_mmr() { let store = MockStoreSuccessBuilder::new().build(); - // Compute actual chain MMR root, from applying an empty block - let actual_chain_mmr_root = { - let block_inputs_from_store: BlockInputs = - store.get_block_inputs(std::iter::empty(), std::iter::empty()).await.unwrap(); - - let batches = Vec::new(); - let block_witness = BlockWitness::new(block_inputs_from_store, batches).unwrap(); - - let block_prover = BlockProver::new(); - let block_header = block_prover.prove(block_witness).unwrap(); - - block_header.chain_root() - }; - - // Compute expected chain MMR root let expected_block_header = build_expected_block_header(&store, &[]).await; + let actual_block_header = build_actual_block_header(&store, Vec::new()).await; - assert_eq!(actual_chain_mmr_root, expected_block_header.chain_root()); + assert_eq!(actual_block_header.chain_root(), expected_block_header.chain_root()); } From 96d180e1b01dc33f0a74017d0a0edd542aa52bee Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 1 Dec 2023 14:17:37 -0500 Subject: [PATCH 156/166] test_compute_chain_mmr_root_mmr_1_peak --- .../src/block_builder/prover/tests.rs | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index 4c32a2ba0..627a6ba34 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -1,5 +1,6 @@ use std::sync::Arc; +use miden_crypto::merkle::Mmr; use miden_mock::mock::block::mock_block_header; use miden_node_proto::domain::{AccountInputRecord, BlockInputs}; use miden_objects::{ @@ -484,7 +485,6 @@ async fn test_compute_note_root_success() { // CHAIN MMR ROOT TESTS // ================================================================================================= -// - add header to non-empty MMR (1 peak), and check that we get the expected commitment // - add header to MMR with 17 peaks /// Test that the chain mmr root is as expected if the batches are empty @@ -497,3 +497,21 @@ async fn test_compute_chain_mmr_root_empty_mmr() { assert_eq!(actual_block_header.chain_root(), expected_block_header.chain_root()); } + +/// add header to non-empty MMR (1 peak), and check that we get the expected commitment +#[tokio::test] +async fn test_compute_chain_mmr_root_mmr_1_peak() { + let initial_chain_mmr = { + let mut mmr = Mmr::new(); + mmr.add(Digest::default()); + + mmr + }; + + let store = MockStoreSuccessBuilder::new().initial_chain_mmr(initial_chain_mmr).build(); + + let expected_block_header = build_expected_block_header(&store, &[]).await; + let actual_block_header = build_actual_block_header(&store, Vec::new()).await; + + assert_eq!(actual_block_header.chain_root(), expected_block_header.chain_root()); +} From 08349d4b625264d768409219d31422c8d71e2a17 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 1 Dec 2023 14:58:16 -0500 Subject: [PATCH 157/166] fix store mmr --- block-producer/src/test_utils/store.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/block-producer/src/test_utils/store.rs b/block-producer/src/test_utils/store.rs index 5d2dcb8be..a36f00756 100644 --- a/block-producer/src/test_utils/store.rs +++ b/block-producer/src/test_utils/store.rs @@ -1,9 +1,6 @@ use async_trait::async_trait; use miden_node_proto::domain::{AccountInputRecord, BlockInputs}; -use miden_objects::{ - crypto::merkle::{Mmr, MmrPeaks}, - BlockHeader, EMPTY_WORD, ONE, ZERO, -}; +use miden_objects::{crypto::merkle::Mmr, BlockHeader, EMPTY_WORD, ONE, ZERO}; use miden_vm::crypto::SimpleSmt; use crate::{ @@ -192,7 +189,10 @@ impl Store for MockStoreSuccess { updated_accounts: impl Iterator + Send, _produced_nullifiers: impl Iterator + Send, ) -> Result { - let chain_peaks = MmrPeaks::new(0, Vec::new()).unwrap(); + let chain_peaks = { + let locked_chain_mmr = self.chain_mmr.read().await; + locked_chain_mmr.peaks(locked_chain_mmr.forest()).unwrap() + }; let account_states = { let locked_accounts = self.accounts.read().await; From 087917b53878f5cc6a00439af7d90df0c63a3dff Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 1 Dec 2023 15:05:49 -0500 Subject: [PATCH 158/166] test_compute_chain_mmr_root_mmr_17_peaks --- .../src/block_builder/prover/tests.rs | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index 627a6ba34..87fed67a6 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -485,8 +485,6 @@ async fn test_compute_note_root_success() { // CHAIN MMR ROOT TESTS // ================================================================================================= -// - add header to MMR with 17 peaks - /// Test that the chain mmr root is as expected if the batches are empty #[tokio::test] async fn test_compute_chain_mmr_root_empty_mmr() { @@ -515,3 +513,25 @@ async fn test_compute_chain_mmr_root_mmr_1_peak() { assert_eq!(actual_block_header.chain_root(), expected_block_header.chain_root()); } + +/// add header to an MMR with 17 peaks, and check that we get the expected commitment +#[tokio::test] +async fn test_compute_chain_mmr_root_mmr_17_peaks() { + let initial_chain_mmr = { + let mut mmr = Mmr::new(); + for _ in 0..(2_u32.pow(17)-1) { + mmr.add(Digest::default()); + } + + assert_eq!(mmr.peaks(mmr.forest()).unwrap().peaks().len(), 17); + + mmr + }; + + let store = MockStoreSuccessBuilder::new().initial_chain_mmr(initial_chain_mmr).build(); + + let expected_block_header = build_expected_block_header(&store, &[]).await; + let actual_block_header = build_actual_block_header(&store, Vec::new()).await; + + assert_eq!(actual_block_header.chain_root(), expected_block_header.chain_root()); +} From 7f5ba308b36c4be8a6166a2937ee5231c8bf3952 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 1 Dec 2023 15:16:43 -0500 Subject: [PATCH 159/166] test_compute_chain_mmr_root_mmr_1_peak_with_batches --- .../src/block_builder/prover/tests.rs | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index 87fed67a6..6b728541a 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -16,6 +16,7 @@ use crate::{ block_builder::prover::block_witness::CREATED_NOTES_TREE_DEPTH, store::Store, test_utils::{ + batch::TransactionBatchConstructor, block::{build_actual_block_header, build_expected_block_header, MockBlockBuilder}, DummyProvenTxGenerator, MockStoreSuccessBuilder, }, @@ -514,12 +515,33 @@ async fn test_compute_chain_mmr_root_mmr_1_peak() { assert_eq!(actual_block_header.chain_root(), expected_block_header.chain_root()); } +/// add header to non-empty MMR (1 peak), and check that we get the expected commitment. +/// This version of the test adds a batch to the block. +#[tokio::test] +async fn test_compute_chain_mmr_root_mmr_1_peak_with_batches() { + let initial_chain_mmr = { + let mut mmr = Mmr::new(); + mmr.add(Digest::default()); + + mmr + }; + + let store = MockStoreSuccessBuilder::new().initial_chain_mmr(initial_chain_mmr).build(); + + let batches = vec![TransactionBatch::from_txs(5)]; + + let expected_block_header = build_expected_block_header(&store, &batches).await; + let actual_block_header = build_actual_block_header(&store, batches).await; + + assert_eq!(actual_block_header.chain_root(), expected_block_header.chain_root()); +} + /// add header to an MMR with 17 peaks, and check that we get the expected commitment #[tokio::test] async fn test_compute_chain_mmr_root_mmr_17_peaks() { let initial_chain_mmr = { let mut mmr = Mmr::new(); - for _ in 0..(2_u32.pow(17)-1) { + for _ in 0..(2_u32.pow(17) - 1) { mmr.add(Digest::default()); } From 5b5e32b726e5c73851c0efb47b85a79d149bfd83 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 1 Dec 2023 15:19:39 -0500 Subject: [PATCH 160/166] remove test --- .../src/block_builder/prover/tests.rs | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index 6b728541a..b53952820 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -515,27 +515,6 @@ async fn test_compute_chain_mmr_root_mmr_1_peak() { assert_eq!(actual_block_header.chain_root(), expected_block_header.chain_root()); } -/// add header to non-empty MMR (1 peak), and check that we get the expected commitment. -/// This version of the test adds a batch to the block. -#[tokio::test] -async fn test_compute_chain_mmr_root_mmr_1_peak_with_batches() { - let initial_chain_mmr = { - let mut mmr = Mmr::new(); - mmr.add(Digest::default()); - - mmr - }; - - let store = MockStoreSuccessBuilder::new().initial_chain_mmr(initial_chain_mmr).build(); - - let batches = vec![TransactionBatch::from_txs(5)]; - - let expected_block_header = build_expected_block_header(&store, &batches).await; - let actual_block_header = build_actual_block_header(&store, batches).await; - - assert_eq!(actual_block_header.chain_root(), expected_block_header.chain_root()); -} - /// add header to an MMR with 17 peaks, and check that we get the expected commitment #[tokio::test] async fn test_compute_chain_mmr_root_mmr_17_peaks() { From f4da6f692032e708d370a11a1e520bdb0c38400a Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Fri, 1 Dec 2023 15:25:31 -0500 Subject: [PATCH 161/166] add comment --- block-producer/src/test_utils/store.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/block-producer/src/test_utils/store.rs b/block-producer/src/test_utils/store.rs index a36f00756..28bbcbd7f 100644 --- a/block-producer/src/test_utils/store.rs +++ b/block-producer/src/test_utils/store.rs @@ -22,6 +22,8 @@ pub struct MockStoreSuccessBuilder { } impl MockStoreSuccessBuilder { + /// FIXME: the store always needs to be properly initialized with initial accounts + /// see https://github.com/0xPolygonMiden/miden-node/issues/79 pub fn new() -> Self { Self::default() } From 12d2cbf444fd357b86f828a23b3f47ceb0a90c23 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Mon, 4 Dec 2023 09:00:25 -0500 Subject: [PATCH 162/166] add period --- block-producer/src/block_builder/prover/block_witness.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block-producer/src/block_builder/prover/block_witness.rs b/block-producer/src/block_builder/prover/block_witness.rs index bd30696d9..32409715b 100644 --- a/block-producer/src/block_builder/prover/block_witness.rs +++ b/block-producer/src/block_builder/prover/block_witness.rs @@ -27,7 +27,7 @@ pub(crate) const CREATED_NOTES_TREE_DEPTH: u8 = // BLOCK WITNESS // ================================================================================================= -/// Provides inputs to the `BlockKernel` so that it can generate the new header +/// Provides inputs to the `BlockKernel` so that it can generate the new header. #[derive(Debug, PartialEq)] pub struct BlockWitness { pub(super) updated_accounts: BTreeMap, From b50196132a5590b648864b6c31250ee98ec307b8 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Mon, 4 Dec 2023 09:05:28 -0500 Subject: [PATCH 163/166] MAX_BATCHES_PER_BLOCK constant --- block-producer/src/block_builder/prover/block_witness.rs | 6 ++++-- block-producer/src/block_builder/prover/tests.rs | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/block-producer/src/block_builder/prover/block_witness.rs b/block-producer/src/block_builder/prover/block_witness.rs index 32409715b..a1c0ff20c 100644 --- a/block-producer/src/block_builder/prover/block_witness.rs +++ b/block-producer/src/block_builder/prover/block_witness.rs @@ -24,6 +24,9 @@ pub(crate) const CREATED_NOTES_TREE_INSERTION_DEPTH: u8 = 8; pub(crate) const CREATED_NOTES_TREE_DEPTH: u8 = CREATED_NOTES_TREE_INSERTION_DEPTH + batch_builder::CREATED_NOTES_SMT_DEPTH; +pub(crate) const MAX_BATCHES_PER_BLOCK: usize = + 2_usize.pow(CREATED_NOTES_TREE_INSERTION_DEPTH as u32); + // BLOCK WITNESS // ================================================================================================= @@ -183,8 +186,7 @@ impl BlockWitness { // TODO: // - Block height returned for each nullifier is 0. - // Validate that there aren't too many batches in the block. - if batches.len() > 2usize.pow(CREATED_NOTES_TREE_INSERTION_DEPTH.into()) { + if batches.len() > MAX_BATCHES_PER_BLOCK { return Err(BuildBlockError::TooManyBatchesInBlock(batches.len())); } diff --git a/block-producer/src/block_builder/prover/tests.rs b/block-producer/src/block_builder/prover/tests.rs index b53952820..8bf11727b 100644 --- a/block-producer/src/block_builder/prover/tests.rs +++ b/block-producer/src/block_builder/prover/tests.rs @@ -16,7 +16,6 @@ use crate::{ block_builder::prover::block_witness::CREATED_NOTES_TREE_DEPTH, store::Store, test_utils::{ - batch::TransactionBatchConstructor, block::{build_actual_block_header, build_expected_block_header, MockBlockBuilder}, DummyProvenTxGenerator, MockStoreSuccessBuilder, }, From 945a98b9dbe41732194711eaa3151cc76c1b20b8 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Mon, 4 Dec 2023 09:06:44 -0500 Subject: [PATCH 164/166] fix comment in masm --- block-producer/src/block_builder/prover/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block-producer/src/block_builder/prover/mod.rs b/block-producer/src/block_builder/prover/mod.rs index 9bcdd7fb0..eb5908095 100644 --- a/block-producer/src/block_builder/prover/mod.rs +++ b/block-producer/src/block_builder/prover/mod.rs @@ -141,7 +141,7 @@ begin # => [ ] exec.compute_chain_mmr_root - # => [ ] + # => [ CHAIN_MMR_ROOT ] # Load output on stack padw mem_loadw.1 padw mem_loadw.0 From 7df7ed27f9f09aa3462c777200283aa25a518556 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Mon, 4 Dec 2023 09:11:33 -0500 Subject: [PATCH 165/166] use BTreeMap instead of Vec --- block-producer/src/block_builder/prover/block_witness.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block-producer/src/block_builder/prover/block_witness.rs b/block-producer/src/block_builder/prover/block_witness.rs index a1c0ff20c..626f22982 100644 --- a/block-producer/src/block_builder/prover/block_witness.rs +++ b/block-producer/src/block_builder/prover/block_witness.rs @@ -35,7 +35,7 @@ pub(crate) const MAX_BATCHES_PER_BLOCK: usize = pub struct BlockWitness { pub(super) updated_accounts: BTreeMap, /// (batch_index, created_notes_root) for batches that contain notes - pub(super) batch_created_notes_roots: Vec<(usize, Digest)>, + pub(super) batch_created_notes_roots: BTreeMap, pub(super) chain_peaks: MmrPeaks, pub(super) prev_header: BlockHeader, } From eee1d0a6f7023b4af29652e62279c3b7f3421467 Mon Sep 17 00:00:00 2001 From: Philippe Laferriere Date: Mon, 4 Dec 2023 09:11:49 -0500 Subject: [PATCH 166/166] clippy test --- block-producer/src/test_utils/proven_tx.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/block-producer/src/test_utils/proven_tx.rs b/block-producer/src/test_utils/proven_tx.rs index d30b704b9..41786fba2 100644 --- a/block-producer/src/test_utils/proven_tx.rs +++ b/block-producer/src/test_utils/proven_tx.rs @@ -89,6 +89,12 @@ impl MockProvenTxBuilder { } } +impl Default for MockProvenTxBuilder { + fn default() -> Self { + Self::new() + } +} + /// We need to generate a new `ProvenTransaction` every time because it doesn't /// derive `Clone`. Doing it this way allows us to compute the `StarkProof` /// once, and clone it for each new `ProvenTransaction`.