From b72be502de9d3e6731b14f88aef03494591755c5 Mon Sep 17 00:00:00 2001 From: SantiagoPittella Date: Thu, 13 Feb 2025 12:08:26 -0300 Subject: [PATCH] wip: remote batch prover --- Cargo.lock | 118 ++++++++++++++++-- Cargo.toml | 9 +- bin/node/src/config.rs | 20 ++- crates/block-producer/Cargo.toml | 1 + .../block-producer/src/batch_builder/mod.rs | 49 ++++++-- crates/block-producer/src/config.rs | 14 ++- crates/block-producer/src/server.rs | 16 ++- crates/block-producer/src/test_utils/batch.rs | 5 + 8 files changed, 200 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 90826a365..65e9b71de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -837,7 +837,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -1787,7 +1787,7 @@ dependencies = [ [[package]] name = "miden-lib" version = "0.8.0" -source = "git+https://github.com/0xPolygonMiden/miden-base?branch=next#66cf1bc8744cf739aa3ef726300c389796047394" +source = "git+https://github.com/0xPolygonMiden/miden-base?branch=santiagopittella-test-mutex#decf833e272fb7a8050001b8770be8ba84fe8263" dependencies = [ "miden-assembly", "miden-objects", @@ -1821,7 +1821,7 @@ dependencies = [ "supports-hyperlinks", "supports-unicode", "syn", - "terminal_size", + "terminal_size 0.3.0", "textwrap", "thiserror 2.0.11", "trybuild", @@ -1876,6 +1876,7 @@ dependencies = [ "miden-node-utils", "miden-objects", "miden-processor", + "miden-proving-service-client", "miden-stdlib", "miden-tx", "miden-tx-batch-prover", @@ -1983,7 +1984,7 @@ dependencies = [ [[package]] name = "miden-objects" version = "0.8.0" -source = "git+https://github.com/0xPolygonMiden/miden-base?branch=next#66cf1bc8744cf739aa3ef726300c389796047394" +source = "git+https://github.com/0xPolygonMiden/miden-base?branch=santiagopittella-test-mutex#decf833e272fb7a8050001b8770be8ba84fe8263" dependencies = [ "getrandom 0.2.15", "miden-assembly", @@ -2026,6 +2027,26 @@ dependencies = [ "winter-prover", ] +[[package]] +name = "miden-proving-service-client" +version = "0.8.0" +source = "git+https://github.com/0xPolygonMiden/miden-base?branch=santiagopittella-test-mutex#decf833e272fb7a8050001b8770be8ba84fe8263" +dependencies = [ + "async-trait", + "miden-objects", + "miden-tx", + "miden-tx-batch-prover", + "miette", + "prost", + "prost-build", + "protox", + "thiserror 2.0.11", + "tokio", + "tonic", + "tonic-build", + "tonic-web-wasm-client", +] + [[package]] name = "miden-rpc-proto" version = "0.8.0" @@ -2042,7 +2063,7 @@ dependencies = [ [[package]] name = "miden-tx" version = "0.8.0" -source = "git+https://github.com/0xPolygonMiden/miden-base?branch=next#66cf1bc8744cf739aa3ef726300c389796047394" +source = "git+https://github.com/0xPolygonMiden/miden-base?branch=santiagopittella-test-mutex#decf833e272fb7a8050001b8770be8ba84fe8263" dependencies = [ "async-trait", "miden-lib", @@ -2059,7 +2080,7 @@ dependencies = [ [[package]] name = "miden-tx-batch-prover" version = "0.8.0" -source = "git+https://github.com/0xPolygonMiden/miden-base?branch=next#66cf1bc8744cf739aa3ef726300c389796047394" +source = "git+https://github.com/0xPolygonMiden/miden-base?branch=santiagopittella-test-mutex#decf833e272fb7a8050001b8770be8ba84fe8263" dependencies = [ "miden-core", "miden-crypto", @@ -2088,8 +2109,16 @@ version = "7.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a955165f87b37fd1862df2a59547ac542c77ef6d17c666f619d1ad22dd89484" dependencies = [ + "backtrace", + "backtrace-ext", "cfg-if", "miette-derive", + "owo-colors", + "supports-color", + "supports-hyperlinks", + "supports-unicode", + "terminal_size 0.4.1", + "textwrap", "thiserror 1.0.69", "unicode-width 0.1.14", ] @@ -2913,7 +2942,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -3317,7 +3346,7 @@ dependencies = [ "getrandom 0.3.1", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -3350,6 +3379,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "terminal_size" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5352447f921fda68cf61b4101566c0bdb5104eff6804d0678e5227580ab6a4e9" +dependencies = [ + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "textwrap" version = "0.16.1" @@ -3625,6 +3664,31 @@ dependencies = [ "tracing", ] +[[package]] +name = "tonic-web-wasm-client" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c191f6613df48874158b6af303313eadf25d1b7a534216b62a1f049d77cd2711" +dependencies = [ + "base64", + "byteorder", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "httparse", + "js-sys", + "pin-project", + "thiserror 1.0.69", + "tonic", + "tower-service", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + [[package]] name = "tower" version = "0.4.13" @@ -4062,6 +4126,19 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.100" @@ -4094,6 +4171,29 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "web-time" version = "1.1.0" @@ -4126,7 +4226,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4e3a13cfe..b39c965d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,18 +28,19 @@ version = "0.8.0" assert_matches = { version = "1.5" } itertools = { version = "0.14" } miden-air = { version = "0.12" } -miden-lib = { git = "https://github.com/0xPolygonMiden/miden-base", branch = "next" } +miden-lib = { git = "https://github.com/0xPolygonMiden/miden-base", branch = "santiagopittella-test-mutex" } miden-node-block-producer = { path = "crates/block-producer", version = "0.8" } miden-node-proto = { path = "crates/proto", version = "0.8" } miden-node-rpc = { path = "crates/rpc", version = "0.8" } miden-node-store = { path = "crates/store", version = "0.8" } miden-node-test-macro = { path = "crates/test-macro" } miden-node-utils = { path = "crates/utils", version = "0.8" } -miden-objects = { git = "https://github.com/0xPolygonMiden/miden-base", branch = "next" } +miden-objects = { git = "https://github.com/0xPolygonMiden/miden-base", branch = "santiagopittella-test-mutex" } miden-processor = { version = "0.12" } miden-stdlib = { version = "0.12", default-features = false } -miden-tx = { git = "https://github.com/0xPolygonMiden/miden-base", branch = "next" } -miden-tx-batch-prover = { git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "next" } +miden-tx = { git = "https://github.com/0xPolygonMiden/miden-base", branch = "santiagopittella-test-mutex" } +miden-tx-batch-prover = { git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "santiagopittella-test-mutex" } +miden-proving-service-client = { git = "https://github.com/0xPolygonMiden/miden-base.git", branch = "santiagopittella-test-mutex", features = ["batch-prover"]} prost = { version = "0.13" } rand = { version = "0.8" } thiserror = { version = "2.0", default-features = false } diff --git a/bin/node/src/config.rs b/bin/node/src/config.rs index 4fef3981f..f457911a0 100644 --- a/bin/node/src/config.rs +++ b/bin/node/src/config.rs @@ -27,6 +27,7 @@ struct NormalizedRpcConfig { struct NormalizedBlockProducerConfig { endpoint: Url, verify_tx_proofs: bool, + remote_batch_prover: Option, } impl Default for NormalizedRpcConfig { @@ -44,9 +45,17 @@ impl Default for NormalizedRpcConfig { impl Default for NormalizedBlockProducerConfig { fn default() -> Self { // Ensure we stay in sync with the original defaults. - let BlockProducerConfig { endpoint, store_url: _, verify_tx_proofs } = - BlockProducerConfig::default(); - Self { endpoint, verify_tx_proofs } + let BlockProducerConfig { + endpoint, + store_url: _, + verify_tx_proofs, + remote_batch_prover, + } = BlockProducerConfig::default(); + Self { + endpoint, + verify_tx_proofs, + remote_batch_prover, + } } } @@ -58,6 +67,7 @@ impl NodeConfig { endpoint: block_producer.endpoint, store_url: store.endpoint.clone(), verify_tx_proofs: block_producer.verify_tx_proofs, + remote_batch_prover: block_producer.remote_batch_prover, }; let rpc = RpcConfig { @@ -92,6 +102,7 @@ mod tests { [block_producer] endpoint = "http://127.0.0.1:8080" verify_tx_proofs = true + remote_batch_prover = "http://127.0.0.1:8081" [rpc] endpoint = "http://127.0.0.1:8080" @@ -111,7 +122,8 @@ mod tests { NodeConfig { block_producer: NormalizedBlockProducerConfig { endpoint: Url::parse("http://127.0.0.1:8080").unwrap(), - verify_tx_proofs: true + verify_tx_proofs: true, + remote_batch_prover: Some(Url::parse("http://127.0.0.1:8081").unwrap()), }, rpc: NormalizedRpcConfig { endpoint: Url::parse("http://127.0.0.1:8080").unwrap(), diff --git a/crates/block-producer/Cargo.toml b/crates/block-producer/Cargo.toml index dabf8d7bd..747d230f7 100644 --- a/crates/block-producer/Cargo.toml +++ b/crates/block-producer/Cargo.toml @@ -29,6 +29,7 @@ miden-processor = { workspace = true } miden-stdlib = { workspace = true } miden-tx = { workspace = true } miden-tx-batch-prover = { workspace = true } +miden-proving-service-client = { workspace = true, features = ["batch-prover"] } rand = { version = "0.8" } serde = { version = "1.0", features = ["derive"] } thiserror = { workspace = true } diff --git a/crates/block-producer/src/batch_builder/mod.rs b/crates/block-producer/src/batch_builder/mod.rs index 550dcb736..b6fdf641a 100644 --- a/crates/block-producer/src/batch_builder/mod.rs +++ b/crates/block-producer/src/batch_builder/mod.rs @@ -1,4 +1,4 @@ -use std::{num::NonZeroUsize, ops::Range, time::Duration}; +use std::{num::NonZeroUsize, ops::Range, sync::Arc, time::Duration}; use miden_node_proto::domain::batch::BatchInputs; use miden_node_utils::formatting::format_array; @@ -6,9 +6,10 @@ use miden_objects::{ batch::{BatchId, ProposedBatch, ProvenBatch}, MIN_PROOF_SECURITY_LEVEL, }; +use miden_proving_service_client::remote_prover::batch_prover::RemoteBatchProver; use miden_tx_batch_prover::LocalBatchProver; use rand::Rng; -use tokio::{task::JoinSet, time}; +use tokio::{sync::Mutex, task::JoinSet, time}; use tracing::{debug, info, instrument, Span}; use crate::{ @@ -54,7 +55,12 @@ impl BatchBuilder { /// A pool of batch-proving workers is spawned, which are fed new batch jobs periodically. /// A batch is skipped if there are no available workers, or if there are no transactions /// available to batch. - pub async fn run(self, mempool: SharedMempool, store: StoreClient) { + pub async fn run( + self, + mempool: SharedMempool, + store: StoreClient, + remote_batch_prover: Arc>>, + ) { assert!( self.failure_rate < 1.0 && self.failure_rate.is_sign_positive(), "Failure rate must be a percentage" @@ -63,8 +69,13 @@ impl BatchBuilder { let mut interval = tokio::time::interval(self.batch_interval); interval.set_missed_tick_behavior(time::MissedTickBehavior::Delay); - let mut worker_pool = - WorkerPool::new(self.workers, self.simulated_proof_time, self.failure_rate, store); + let mut worker_pool = WorkerPool::new( + self.workers, + self.simulated_proof_time, + self.failure_rate, + store, + remote_batch_prover, + ); loop { tokio::select! { @@ -123,6 +134,7 @@ struct WorkerPool { /// impact beyond ergonomics. task_map: Vec<(tokio::task::Id, BatchId)>, store: StoreClient, + remote_batch_prover: Arc>>, } impl WorkerPool { @@ -131,6 +143,7 @@ impl WorkerPool { simulated_proof_time: Range, failure_rate: f32, store: StoreClient, + remote_batch_prover: Arc>>, ) -> Self { Self { simulated_proof_time, @@ -139,6 +152,7 @@ impl WorkerPool { store, in_progress: JoinSet::default(), task_map: Vec::default(), + remote_batch_prover, } } @@ -215,6 +229,8 @@ impl WorkerPool { let failed = rand::thread_rng().gen::() < self.failure_rate; let store = self.store.clone(); + let remote_batch_prover = self.remote_batch_prover.clone(); + async move { tracing::debug!("Begin proving batch."); @@ -229,8 +245,9 @@ impl WorkerPool { .await .map_err(|err| (id, BuildBatchError::FetchBatchInputsFailed(err)))?; - let batch = - Self::build_batch(transactions, batch_inputs).map_err(|err| (id, err))?; + let batch = Self::build_batch(transactions, batch_inputs, remote_batch_prover) + .await + .map_err(|err| (id, err))?; tokio::time::sleep(simulated_proof_time).await; if failed { @@ -251,9 +268,10 @@ impl WorkerPool { } #[instrument(target = COMPONENT, skip_all, err, fields(batch_id))] - fn build_batch( + async fn build_batch( txs: Vec, batch_inputs: BatchInputs, + remote_batch_prover: Arc>>, ) -> Result { let num_txs = txs.len(); @@ -275,9 +293,18 @@ impl WorkerPool { Span::current().record("batch_id", proposed_batch.id().to_string()); info!(target: COMPONENT, "Proposed Batch built"); - let proven_batch = LocalBatchProver::new(MIN_PROOF_SECURITY_LEVEL) - .prove(proposed_batch) - .map_err(BuildBatchError::ProveBatchError)?; + let proven_batch = if let Some(remote_prover) = remote_batch_prover.lock().await.as_ref() { + let proven_batch = remote_prover + .prove(proposed_batch) + .await + .map_err(BuildBatchError::ProveBatchError)?; + + proven_batch + } else { + LocalBatchProver::new(MIN_PROOF_SECURITY_LEVEL) + .prove(proposed_batch) + .map_err(BuildBatchError::ProveBatchError)? + }; Span::current().record("batch_id", proven_batch.id().to_string()); info!(target: COMPONENT, "Proven Batch built"); diff --git a/crates/block-producer/src/config.rs b/crates/block-producer/src/config.rs index 0ad19741b..b0e15e98d 100644 --- a/crates/block-producer/src/config.rs +++ b/crates/block-producer/src/config.rs @@ -23,13 +23,22 @@ pub struct BlockProducerConfig { /// verification may take ~15ms/proof. This is OK when all transactions are forwarded to the /// block producer from the RPC component as transaction proofs are also verified there. pub verify_tx_proofs: bool, + + /// URL of the remote batch prover service. + /// + /// If this is set, the block producer will use the remote batch prover to prove transaction + /// batches before they are included in blocks. If this is not set, the block producer will + /// prove transaction batches itself. + pub remote_batch_prover: Option, } impl Display for BlockProducerConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.write_fmt(format_args!( - "{{ endpoint: \"{}\", store_url: \"{}\" }}", - self.endpoint, self.store_url + "{{ endpoint: \"{}\", store_url: \"{}\", remote_batch_prover: \"{}\" }}", + self.endpoint, + self.store_url, + self.remote_batch_prover.as_ref().map_or("none", |url| url.as_str()) )) } } @@ -44,6 +53,7 @@ impl Default for BlockProducerConfig { store_url: Url::parse(format!("http://127.0.0.1:{DEFAULT_STORE_PORT}").as_str()) .unwrap(), verify_tx_proofs: true, + remote_batch_prover: None, } } } diff --git a/crates/block-producer/src/server.rs b/crates/block-producer/src/server.rs index 251d0b339..577dfa18f 100644 --- a/crates/block-producer/src/server.rs +++ b/crates/block-producer/src/server.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::{collections::HashMap, sync::Arc}; use miden_node_proto::generated::{ block_producer::api_server, requests::SubmitProvenTransactionRequest, @@ -12,6 +12,7 @@ use miden_node_utils::{ use miden_objects::{ block::BlockNumber, transaction::ProvenTransaction, utils::serde::Deserializable, }; +use miden_proving_service_client::remote_prover::batch_prover::RemoteBatchProver; use tokio::{net::TcpListener, sync::Mutex}; use tokio_stream::wrappers::TcpListenerStream; use tonic::Status; @@ -44,6 +45,7 @@ pub struct BlockProducer { rpc_listener: TcpListener, store: StoreClient, chain_tip: BlockNumber, + remote_batch_prover: Arc>>, } impl BlockProducer { @@ -80,6 +82,14 @@ impl BlockProducer { info!(target: COMPONENT, "Server initialized"); + let remote_batch_prover = if let Some(endpoint) = config.remote_batch_prover { + let prover = RemoteBatchProver::new(endpoint.as_str()); + info!(target: COMPONENT, "Remote batch prover enabled"); + Arc::new(Mutex::new(Some(prover))) + } else { + Arc::new(Mutex::new(None)) + }; + Ok(Self { batch_builder: BatchBuilder::default(), block_builder: BlockBuilder::new(store.clone()), @@ -90,6 +100,7 @@ impl BlockProducer { store, rpc_listener, chain_tip, + remote_batch_prover, }) } @@ -104,6 +115,7 @@ impl BlockProducer { store, chain_tip, expiration_slack, + remote_batch_prover, } = self; let mempool = Mempool::shared( @@ -127,7 +139,7 @@ impl BlockProducer { let mempool = mempool.clone(); let store = store.clone(); async { - batch_builder.run(mempool, store).await; + batch_builder.run(mempool, store, remote_batch_prover).await; Ok(()) } }) diff --git a/crates/block-producer/src/test_utils/batch.rs b/crates/block-producer/src/test_utils/batch.rs index b4caffd28..e49c18e00 100644 --- a/crates/block-producer/src/test_utils/batch.rs +++ b/crates/block-producer/src/test_utils/batch.rs @@ -55,8 +55,13 @@ impl TransactionBatchConstructor for ProvenBatch { output_notes.extend(tx.output_notes().iter().cloned()); } + let reference_block_commitment = txs[0].block_ref(); + let reference_block_num = txs[0].block_num(); + ProvenBatch::new( BatchId::from_transactions(txs.into_iter()), + reference_block_commitment, + reference_block_num, account_updates, InputNotes::new_unchecked(input_notes), BatchNoteTree::with_contiguous_leaves(