diff --git a/aggregation_mode/Cargo.lock b/aggregation_mode/Cargo.lock index c6e6a5f40c..624978fc0d 100644 --- a/aggregation_mode/Cargo.lock +++ b/aggregation_mode/Cargo.lock @@ -98,6 +98,7 @@ dependencies = [ "log", "reqwest 0.12.15", "serde", + "serde_bytes", "serde_json", "serde_repr", "sha3 0.10.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -7858,6 +7859,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" version = "1.0.219" diff --git a/crates/Cargo.lock b/crates/Cargo.lock index 744392b748..1d03e6005a 100644 --- a/crates/Cargo.lock +++ b/crates/Cargo.lock @@ -103,7 +103,6 @@ dependencies = [ "bytes", "ciborium", "clap", - "dashmap", "dotenvy", "env_logger", "ethers", @@ -139,6 +138,7 @@ dependencies = [ "log", "reqwest 0.12.15", "serde", + "serde_bytes", "serde_json", "serde_repr", "sha3", @@ -2040,20 +2040,6 @@ dependencies = [ "syn 2.0.100", ] -[[package]] -name = "dashmap" -version = "6.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" -dependencies = [ - "cfg-if", - "crossbeam-utils", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core", -] - [[package]] name = "dashu" version = "0.4.2" @@ -6712,6 +6698,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" version = "1.0.219" diff --git a/crates/batcher/src/types/batch_queue.rs b/crates/batcher/src/types/batch_queue.rs index 7461b4ac3d..35ea3ce0c4 100644 --- a/crates/batcher/src/types/batch_queue.rs +++ b/crates/batcher/src/types/batch_queue.rs @@ -103,7 +103,7 @@ impl Ord for BatchQueueEntryPriority { // Implementation of lowest-first: let ord: std::cmp::Ordering = other.max_fee.cmp(&self.max_fee); // This means, less max_fee will go first - // We want this because we will .pop() to remove unwanted elements, low fee submissions. + // We want this because we will .pop() to remove unwanted elements, low fee submitions. if ord == std::cmp::Ordering::Equal { // Case of same max_fee: @@ -138,71 +138,57 @@ pub(crate) fn calculate_batch_size(batch_queue: &BatchQueue) -> Result Result, BatcherError> { - let mut batch_size = calculate_batch_size(batch_queue)?; - let mut rejected_entries = Vec::new(); - - // Remove entries that won't pay enough, or that makes a queue that is too big - loop { - let should_remove = if let Some((entry, _)) = batch_queue.peek() { - let batch_len = batch_queue.len(); - let fee_per_proof = calculate_fee_per_proof(batch_len, gas_price, constant_gas_cost); - - // if batch is not acceptable: - batch_size > max_batch_byte_size - || fee_per_proof > entry.nonced_verification_data.max_fee - || batch_len > max_batch_proof_qty - } else { - false - }; - - if should_remove { - // Remove this entry (it won't pay enough) and save it - let (rejected_entry, rejected_priority) = batch_queue.pop().unwrap(); - - // Update batch size - let verification_data_size = rejected_entry - .nonced_verification_data - .cbor_size_upper_bound(); + let mut finalized_batch = batch_queue; + let mut batch_size = calculate_batch_size(&finalized_batch)?; + + while let Some((entry, _)) = finalized_batch.peek() { + let batch_len = finalized_batch.len(); + let fee_per_proof = calculate_fee_per_proof(batch_len, gas_price, constant_gas_cost); + + // if batch is not acceptable: + if batch_size > max_batch_byte_size + || fee_per_proof > entry.nonced_verification_data.max_fee + || batch_len > max_batch_proof_qty + { + // Update the state for the next iteration: + // * Subtract this entry size to the size of the batch size. + // * Push the current entry to the resulting batch queue. + + let verification_data_size = entry.nonced_verification_data.cbor_size_upper_bound(); batch_size -= verification_data_size; - rejected_entries.push((rejected_entry, rejected_priority)); - } else { - // At this point, we found a viable batch - break - break; - } - } + finalized_batch.pop(); - // Check if we have a viable batch - if batch_queue.is_empty() { - // No viable batch found - put back the rejected entries - for (entry, priority) in rejected_entries { - batch_queue.push(entry, priority); + continue; } - return Err(BatcherError::BatchCostTooHigh); - } - // Extract remaining entries - let mut batch_for_posting = Vec::new(); - while let Some((entry, _)) = batch_queue.pop() { - batch_for_posting.push(entry); + // At this point, we break since we found a batch that can be submitted + break; } - // Put back the rejected entries (they stay in the queue for later) - for (entry, priority) in rejected_entries { - batch_queue.push(entry, priority); + // If `finalized_batch` is empty, this means that all the batch queue was traversed and we didn't find + // any user willing to pay fot the fee per proof. + if finalized_batch.is_empty() { + return Err(BatcherError::BatchCostTooHigh); } - Ok(batch_for_posting) + Ok(finalized_batch.clone().into_sorted_vec()) } fn calculate_fee_per_proof(batch_len: usize, gas_price: U256, constant_gas_cost: u128) -> U256 { @@ -312,9 +298,8 @@ mod test { batch_queue.push(entry_3, batch_priority_3); let gas_price = U256::from(1); - let mut test_queue = batch_queue.clone(); - let finalized_batch = extract_batch_directly( - &mut test_queue, + let finalized_batch = try_build_batch( + batch_queue.clone(), gas_price, 5000000, 50, @@ -425,9 +410,8 @@ mod test { batch_queue.push(entry_3, batch_priority_3); let gas_price = U256::from(1); - let mut test_queue = batch_queue.clone(); - let finalized_batch = extract_batch_directly( - &mut test_queue, + let finalized_batch = try_build_batch( + batch_queue.clone(), gas_price, 5000000, 50, @@ -536,9 +520,8 @@ mod test { batch_queue.push(entry_3.clone(), batch_priority_3.clone()); let gas_price = U256::from(1); - let mut test_queue = batch_queue.clone(); - let finalized_batch = extract_batch_directly( - &mut test_queue, + let finalized_batch = try_build_batch( + batch_queue.clone(), gas_price, 5000000, 2, @@ -647,9 +630,8 @@ mod test { batch_queue.push(entry_3, batch_priority_3); let gas_price = U256::from(1); - let mut test_queue = batch_queue.clone(); - let finalized_batch = extract_batch_directly( - &mut test_queue, + let finalized_batch = try_build_batch( + batch_queue.clone(), gas_price, 5000000, 50, @@ -764,9 +746,8 @@ mod test { batch_queue.push(entry_3, batch_priority_3); let gas_price = U256::from(1); - let mut test_queue = batch_queue.clone(); - let finalized_batch = extract_batch_directly( - &mut test_queue, + let finalized_batch = try_build_batch( + batch_queue.clone(), gas_price, 5000000, 50, @@ -787,77 +768,6 @@ mod test { ); } - #[test] - fn batch_finalization_algorithm_works_single_high_fee_proof() { - // Test the scenario: 1 proof with high fee that should be viable - let proof_generator_addr = Address::random(); - let payment_service_addr = Address::random(); - let sender_addr = Address::random(); - let bytes_for_verification_data = vec![42_u8; 10]; - let dummy_signature = Signature { - r: U256::from(1), - s: U256::from(2), - v: 3, - }; - let verification_data = VerificationData { - proving_system: ProvingSystemId::Risc0, - proof: bytes_for_verification_data.clone(), - pub_input: Some(bytes_for_verification_data.clone()), - verification_key: Some(bytes_for_verification_data.clone()), - vm_program_code: Some(bytes_for_verification_data), - proof_generator_addr, - }; - let chain_id = U256::from(42); - - // Single entry with very high fee - should definitely be viable - let nonce = U256::from(1); - let high_max_fee = U256::from(1_000_000_000_000_000_000u128); // Very high fee - 1 ETH - let nonced_verification_data = NoncedVerificationData::new( - verification_data, - nonce, - high_max_fee, - chain_id, - payment_service_addr, - ); - let vd_commitment: VerificationDataCommitment = nonced_verification_data.clone().into(); - let entry = BatchQueueEntry::new_for_testing( - nonced_verification_data, - vd_commitment, - dummy_signature, - sender_addr, - ); - let batch_priority = BatchQueueEntryPriority::new(high_max_fee, nonce); - - let mut batch_queue = BatchQueue::new(); - batch_queue.push(entry, batch_priority); - - let gas_price = U256::from(10_000_000_000u64); // 10 gwei gas price - let mut test_queue = batch_queue.clone(); - let finalized_batch = extract_batch_directly( - &mut test_queue, - gas_price, - 5000000, // Large byte size limit - 50, // Large proof quantity limit - DEFAULT_CONSTANT_GAS_COST, - ); - - // This should succeed and return the single proof - assert!( - finalized_batch.is_ok(), - "Should successfully extract batch with single high-fee proof" - ); - let batch = finalized_batch.unwrap(); - assert_eq!(batch.len(), 1, "Batch should contain exactly 1 proof"); - assert_eq!(batch[0].nonced_verification_data.max_fee, high_max_fee); - - // The queue should now be empty (no rejected entries to put back) - assert_eq!( - test_queue.len(), - 0, - "Queue should be empty after extracting the single viable proof" - ); - } - #[test] fn batch_finalization_algorithm_works_not_bigger_than_max_batch_proof_qty() { // The following information will be the same for each entry, it is just some dummy data to see @@ -952,9 +862,8 @@ mod test { // The max batch len is 2, so the algorithm should stop at the second entry. let max_batch_proof_qty = 2; - let mut test_queue = batch_queue.clone(); - let finalized_batch = extract_batch_directly( - &mut test_queue, + let finalized_batch = try_build_batch( + batch_queue.clone(), gas_price, 5000000, max_batch_proof_qty, @@ -973,4 +882,208 @@ mod test { max_fee_1 ); } + + #[test] + fn test_batch_size_limit_enforcement_with_real_sp1_proofs() { + use aligned_sdk::common::types::VerificationData; + use aligned_sdk::communication::serialization::cbor_serialize; + use std::fs; + + let proof_generator_addr = Address::random(); + let payment_service_addr = Address::random(); + let chain_id = U256::from(42); + let dummy_signature = Signature { + r: U256::from(1), + s: U256::from(2), + v: 3, + }; + + // Load actual SP1 proof files + let proof_path = "../../scripts/test_files/sp1/sp1_fibonacci_5_0_0.proof"; + let elf_path = "../../scripts/test_files/sp1/sp1_fibonacci_5_0_0.elf"; + let pub_input_path = "../../scripts/test_files/sp1/sp1_fibonacci_5_0_0.pub"; + + let proof_data = match fs::read(proof_path) { + Ok(data) => data, + Err(_) => return, // Skip test if files not found + }; + + let elf_data = match fs::read(elf_path) { + Ok(data) => data, + Err(_) => return, + }; + + let pub_input_data = match fs::read(pub_input_path) { + Ok(data) => data, + Err(_) => return, + }; + + let verification_data = VerificationData { + proving_system: ProvingSystemId::SP1, + proof: proof_data, + pub_input: Some(pub_input_data), + verification_key: None, + vm_program_code: Some(elf_data), + proof_generator_addr, + }; + + // Create 10 entries using the same SP1 proof data + let mut batch_queue = BatchQueue::new(); + let max_fee = U256::from(1_000_000_000_000_000u128); + + for i in 0..10 { + let sender_addr = Address::random(); + let nonce = U256::from(i + 1); + + let nonced_verification_data = NoncedVerificationData::new( + verification_data.clone(), + nonce, + max_fee, + chain_id, + payment_service_addr, + ); + + let vd_commitment: VerificationDataCommitment = nonced_verification_data.clone().into(); + let entry = BatchQueueEntry::new_for_testing( + nonced_verification_data, + vd_commitment, + dummy_signature, + sender_addr, + ); + let batch_priority = BatchQueueEntryPriority::new(max_fee, nonce); + batch_queue.push(entry, batch_priority); + } + + // Test with a 5MB batch size limit + let batch_size_limit = 5_000_000; // 5MB + let gas_price = U256::from(1); + + let finalized_batch = try_build_batch( + batch_queue.clone(), + gas_price, + batch_size_limit, + 50, // max proof qty + DEFAULT_CONSTANT_GAS_COST, + ) + .unwrap(); + + // Verify the finalized batch respects the size limit + let finalized_verification_data: Vec = finalized_batch + .iter() + .map(|entry| entry.nonced_verification_data.verification_data.clone()) + .collect(); + + let finalized_serialized = cbor_serialize(&finalized_verification_data).unwrap(); + let finalized_actual_size = finalized_serialized.len(); + + // Assert the batch respects the limit + assert!( + finalized_actual_size <= batch_size_limit, + "Finalized batch size {} exceeds limit {}", + finalized_actual_size, + batch_size_limit + ); + + // Verify some entries were included (not empty batch) + assert!(!finalized_batch.is_empty(), "Batch should not be empty"); + + // Verify not all entries were included (some should be rejected due to size limit) + assert!( + finalized_batch.len() < 10, + "Batch should not include all entries due to size limit" + ); + } + + #[test] + fn test_cbor_size_upper_bound_accuracy() { + use aligned_sdk::common::types::VerificationData; + use aligned_sdk::communication::serialization::cbor_serialize; + use std::fs; + + let proof_generator_addr = Address::random(); + let payment_service_addr = Address::random(); + let chain_id = U256::from(42); + + // Load actual SP1 proof files + let proof_path = "../../scripts/test_files/sp1/sp1_fibonacci_5_0_0.proof"; + let elf_path = "../../scripts/test_files/sp1/sp1_fibonacci_5_0_0.elf"; + let pub_input_path = "../../scripts/test_files/sp1/sp1_fibonacci_5_0_0.pub"; + + let proof_data = match fs::read(proof_path) { + Ok(data) => data, + Err(_) => return, // Skip test if files not found + }; + + let elf_data = match fs::read(elf_path) { + Ok(data) => data, + Err(_) => return, + }; + + let pub_input_data = match fs::read(pub_input_path) { + Ok(data) => data, + Err(_) => return, + }; + + let verification_data = VerificationData { + proving_system: ProvingSystemId::SP1, + proof: proof_data, + pub_input: Some(pub_input_data), + verification_key: None, + vm_program_code: Some(elf_data), + proof_generator_addr, + }; + + let nonced_verification_data = NoncedVerificationData::new( + verification_data.clone(), + U256::from(1), + U256::from(1_000_000_000_000_000u128), + chain_id, + payment_service_addr, + ); + + // Test cbor_size_upper_bound() accuracy + let estimated_size = nonced_verification_data.cbor_size_upper_bound(); + + // Compare with actual CBOR serialization of the full NoncedVerificationData + let actual_nonced_serialized = cbor_serialize(&nonced_verification_data).unwrap(); + let actual_nonced_size = actual_nonced_serialized.len(); + + // Also test the inner VerificationData for additional validation + let actual_inner_serialized = cbor_serialize(&verification_data).unwrap(); + let actual_inner_size = actual_inner_serialized.len(); + + // Verify CBOR encodes binary data efficiently (with serde_bytes fix), this misses some overhead but the proof is big enough as to not matter + + let raw_total = verification_data.proof.len() + + verification_data.vm_program_code.as_ref().unwrap().len() + + verification_data.pub_input.as_ref().unwrap().len(); + + let cbor_efficiency_ratio = actual_inner_size as f64 / raw_total as f64; + + // With serde_bytes, CBOR should be very efficient (close to 1.0x) + assert!( + cbor_efficiency_ratio < 1.01, + "CBOR serialization should be efficient with serde_bytes. Ratio: {:.3}x", + cbor_efficiency_ratio + ); + + // Verify CBOR encodes binary data efficiently with serde_bytes + // Should be close to 1.0x overhead (raw data size vs CBOR size) + + // The estimation should be an upper bound + assert!( + estimated_size >= actual_nonced_size, + "cbor_size_upper_bound() should be an upper bound. Estimated: {}, Actual: {}", + estimated_size, + actual_nonced_size + ); + + // The estimation should also be reasonable (not wildly over-estimated) + let estimation_overhead = estimated_size as f64 / actual_nonced_size as f64; + assert!( + estimation_overhead < 1.1, + "Estimation should be reasonable, not wildly over-estimated. Overhead: {:.3}x", + estimation_overhead + ); + } } diff --git a/crates/sdk/Cargo.toml b/crates/sdk/Cargo.toml index 964be4a4e9..767e2ed266 100644 --- a/crates/sdk/Cargo.toml +++ b/crates/sdk/Cargo.toml @@ -19,6 +19,7 @@ tokio = { version = "1.37.0", features = [ ] } lambdaworks-crypto = { git = "https://github.com/lambdaclass/lambdaworks.git", rev = "5f8f2cfcc8a1a22f77e8dff2d581f1166eefb80b", features = ["serde"]} serde = { version = "1.0.201", features = ["derive"] } +serde_bytes = "0.11" sha3 = { version = "0.10.8" } url = "2.5.0" hex = "0.4.3" diff --git a/crates/sdk/src/common/types.rs b/crates/sdk/src/common/types.rs index 2129ac38f1..6ff2bf42a9 100644 --- a/crates/sdk/src/common/types.rs +++ b/crates/sdk/src/common/types.rs @@ -65,9 +65,28 @@ impl Display for ProvingSystemId { #[derive(Debug, Serialize, Deserialize, Clone)] pub struct VerificationData { pub proving_system: ProvingSystemId, + #[serde(with = "serde_bytes")] pub proof: Vec, + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "deserialize_option_bytes", + serialize_with = "serialize_option_bytes" + )] pub pub_input: Option>, + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "deserialize_option_bytes", + serialize_with = "serialize_option_bytes" + )] pub verification_key: Option>, + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "deserialize_option_bytes", + serialize_with = "serialize_option_bytes" + )] pub vm_program_code: Option>, pub proof_generator_addr: Address, } @@ -504,6 +523,25 @@ impl Network { } } +// Helper functions for serializing Option> with serde_bytes +fn serialize_option_bytes(value: &Option>, serializer: S) -> Result +where + S: serde::Serializer, +{ + match value { + Some(bytes) => serde_bytes::serialize(bytes, serializer), + None => serializer.serialize_none(), + } +} + +fn deserialize_option_bytes<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: serde::Deserializer<'de>, +{ + let opt: Option = Option::deserialize(deserializer)?; + Ok(opt.map(|buf| buf.into_vec())) +} + #[cfg(test)] mod tests { use ethers::signers::LocalWallet;