feat: batcher should not be able to build batches too big (#1410)

Co-authored-by: Marcos Nicolau <[email protected]>

Author: uri-99

Makefile

@@ -410,7 +410,7 @@ batcher_send_plonk_bn254_burst: batcher/target/release/aligned
 		--vk ../../scripts/test_files/gnark_plonk_bn254_script/plonk.vk \
 		--proof_generator_addr 0x66f9664f97F2b50F62D13eA064982f936dE76657 \
 		--rpc_url $(RPC_URL) \
-		--repetitions 4 \
+		--repetitions $(BURST_SIZE) \
 		--network $(NETWORK)
 
 batcher_send_plonk_bls12_381_task: batcher/target/release/aligned

batcher/aligned-batcher/src/config/mod.rs

@@ -40,7 +40,8 @@ pub struct BatcherConfigFromYaml {
     pub block_interval: u64,
     pub transaction_wait_timeout: u64,
     pub max_proof_size: usize,
-    pub max_batch_size: usize,
+    pub max_batch_byte_size: usize,
+    pub max_batch_proof_qty: usize,
     pub pre_verification_is_enabled: bool,
     pub metrics_port: u16,
     pub telemetry_ip_port_address: String,

batcher/aligned-batcher/src/lib.rs

@@ -82,7 +82,8 @@ pub struct Batcher {
     max_block_interval: u64,
     transaction_wait_timeout: u64,
     max_proof_size: usize,
-    max_batch_size: usize,
+    max_batch_byte_size: usize,
+    max_batch_proof_qty: usize,
     last_uploaded_batch_block: Mutex<u64>,
     pre_verification_is_enabled: bool,
     non_paying_config: Option<NonPayingConfig>,
@@ -244,7 +245,8 @@ impl Batcher {
             max_block_interval: config.batcher.block_interval,
             transaction_wait_timeout: config.batcher.transaction_wait_timeout,
             max_proof_size: config.batcher.max_proof_size,
-            max_batch_size: config.batcher.max_batch_size,
+            max_batch_byte_size: config.batcher.max_batch_byte_size,
+            max_batch_proof_qty: config.batcher.max_batch_proof_qty,
             last_uploaded_batch_block: Mutex::new(last_uploaded_batch_block),
             pre_verification_is_enabled: config.batcher.pre_verification_is_enabled,
             non_paying_config,
@@ -1055,21 +1057,25 @@ impl Batcher {
         // Set the batch posting flag to true
         *batch_posting = true;
         let batch_queue_copy = batch_state_lock.batch_queue.clone();
-        let (resulting_batch_queue, finalized_batch) =
-            batch_queue::try_build_batch(batch_queue_copy, gas_price, self.max_batch_size)
-                .inspect_err(|e| {
-                    *batch_posting = false;
-                    match e {
-                        // We can't post a batch since users are not willing to pay the needed fee, wait for more proofs
-                        BatcherError::BatchCostTooHigh => {
-                            info!("No working batch found. Waiting for more proofs")
-                        }
-                        // FIXME: We should refactor this code and instead of returning None, return an error.
-                        // See issue https://github.com/yetanotherco/aligned_layer/issues/1046.
-                        e => error!("Unexpected error: {:?}", e),
-                    }
-                })
-                .ok()?;
+        let (resulting_batch_queue, finalized_batch) = batch_queue::try_build_batch(
+            batch_queue_copy,
+            gas_price,
+            self.max_batch_byte_size,
+            self.max_batch_proof_qty,
+        )
+        .inspect_err(|e| {
+            *batch_posting = false;
+            match e {
+                // We can't post a batch since users are not willing to pay the needed fee, wait for more proofs
+                BatcherError::BatchCostTooHigh => {
+                    info!("No working batch found. Waiting for more proofs")
+                }
+                // FIXME: We should refactor this code and instead of returning None, return an error.
+                // See issue https://github.com/yetanotherco/aligned_layer/issues/1046.
+                e => error!("Unexpected error: {:?}", e),
+            }
+        })
+        .ok()?;
 
         batch_state_lock.batch_queue = resulting_batch_queue;
         let updated_user_proof_count_and_min_fee =

batcher/aligned-batcher/src/types/batch_queue.rs

@@ -146,7 +146,8 @@ pub(crate) fn calculate_batch_size(batch_queue: &BatchQueue) -> Result<usize, Ba
 pub(crate) fn try_build_batch(
     batch_queue: BatchQueue,
     gas_price: U256,
-    max_batch_size: usize,
+    max_batch_byte_size: usize,
+    max_batch_proof_qty: usize,
 ) -> Result<(BatchQueue, Vec<BatchQueueEntry>), BatcherError> {
     let mut batch_queue = batch_queue;
     let mut batch_size = calculate_batch_size(&batch_queue)?;
@@ -156,7 +157,10 @@ pub(crate) fn try_build_batch(
         let batch_len = batch_queue.len();
         let fee_per_proof = calculate_fee_per_proof(batch_len, gas_price);
 
-        if batch_size > max_batch_size || fee_per_proof > entry.nonced_verification_data.max_fee {
+        if batch_size > max_batch_byte_size
+            || fee_per_proof > entry.nonced_verification_data.max_fee
+            || batch_len > max_batch_proof_qty
+        {
             // Update the state for the next iteration:
             // * Subtract this entry size to the size of the batch size.
             // * Push the current entry to the resulting batch queue.
@@ -298,7 +302,7 @@ mod test {
 
         let gas_price = U256::from(1);
         let (resulting_batch_queue, batch) =
-            try_build_batch(batch_queue, gas_price, 5000000).unwrap();
+            try_build_batch(batch_queue, gas_price, 5000000, 50).unwrap();
 
         assert!(resulting_batch_queue.is_empty());
 
@@ -401,7 +405,7 @@ mod test {
 
         let gas_price = U256::from(1);
         let (resulting_batch_queue, finalized_batch) =
-            try_build_batch(batch_queue, gas_price, 5000000).unwrap();
+            try_build_batch(batch_queue, gas_price, 5000000, 50).unwrap();
 
         // The resulting batch queue (entries from the old batch queue that were not willing to pay
         // in this batch), should be empty and hence, all entries from the batch queue should be in
@@ -512,7 +516,7 @@ mod test {
 
         let gas_price = U256::from(1);
         let (resulting_batch_queue, finalized_batch) =
-            try_build_batch(batch_queue, gas_price, 5000000).unwrap();
+            try_build_batch(batch_queue, gas_price, 5000000, 50).unwrap();
 
         // The resulting batch queue (entries from the old batch queue that were not willing to pay
         // in this batch), should be empty and hence, all entries from the batch queue should be in
@@ -529,4 +533,113 @@ mod test {
             max_fee_1
         );
     }
+
+    #[test]
+    fn batch_finalization_algorithm_works_not_bigger_than_max_batch_proof_qty() {
+        // The following information will be the same for each entry, it is just some dummy data to see
+        // algorithm working.
+
+        let proof_generator_addr = Address::random();
+        let payment_service_addr = Address::random();
+        let sender_addr = Address::random();
+        let bytes_for_verification_data = vec![42_u8; 10];
+        let dummy_signature = Signature {
+            r: U256::from(1),
+            s: U256::from(2),
+            v: 3,
+        };
+        let verification_data = VerificationData {
+            proving_system: ProvingSystemId::Risc0,
+            proof: bytes_for_verification_data.clone(),
+            pub_input: Some(bytes_for_verification_data.clone()),
+            verification_key: Some(bytes_for_verification_data.clone()),
+            vm_program_code: Some(bytes_for_verification_data),
+            proof_generator_addr,
+        };
+        let chain_id = U256::from(42);
+
+        // Here we create different entries for the batch queue.
+        // Since we are sending with the same address, the low nonces should have higher max fees.
+
+        // Entry 1
+        let nonce_1 = U256::from(1);
+        let max_fee_1 = U256::from(1_300_000_000_000_002u128);
+        let nonced_verification_data_1 = NoncedVerificationData::new(
+            verification_data.clone(),
+            nonce_1,
+            max_fee_1,
+            chain_id,
+            payment_service_addr,
+        );
+        let vd_commitment_1: VerificationDataCommitment = nonced_verification_data_1.clone().into();
+        let entry_1 = BatchQueueEntry::new_for_testing(
+            nonced_verification_data_1,
+            vd_commitment_1,
+            dummy_signature,
+            sender_addr,
+        );
+        let batch_priority_1 = BatchQueueEntryPriority::new(max_fee_1, nonce_1);
+
+        // Entry 2
+        let nonce_2 = U256::from(2);
+        let max_fee_2 = U256::from(1_300_000_000_000_001u128);
+        let nonced_verification_data_2 = NoncedVerificationData::new(
+            verification_data.clone(),
+            nonce_2,
+            max_fee_2,
+            chain_id,
+            payment_service_addr,
+        );
+        let vd_commitment_2: VerificationDataCommitment = nonced_verification_data_2.clone().into();
+        let entry_2 = BatchQueueEntry::new_for_testing(
+            nonced_verification_data_2,
+            vd_commitment_2,
+            dummy_signature,
+            sender_addr,
+        );
+        let batch_priority_2 = BatchQueueEntryPriority::new(max_fee_2, nonce_2);
+
+        // Entry 3
+        let nonce_3 = U256::from(3);
+        let max_fee_3 = U256::from(1_300_000_000_000_000u128);
+        let nonced_verification_data_3 = NoncedVerificationData::new(
+            verification_data.clone(),
+            nonce_3,
+            max_fee_3,
+            chain_id,
+            payment_service_addr,
+        );
+        let vd_commitment_3: VerificationDataCommitment = nonced_verification_data_3.clone().into();
+        let entry_3 = BatchQueueEntry::new_for_testing(
+            nonced_verification_data_3,
+            vd_commitment_3,
+            dummy_signature,
+            sender_addr,
+        );
+        let batch_priority_3 = BatchQueueEntryPriority::new(max_fee_3, nonce_3);
+
+        let mut batch_queue = BatchQueue::new();
+        batch_queue.push(entry_1, batch_priority_1);
+        batch_queue.push(entry_2, batch_priority_2);
+        batch_queue.push(entry_3, batch_priority_3);
+
+        let gas_price = U256::from(1);
+
+        // The max batch len is 2, so the algorithm should stop at the second entry.
+        let max_batch_proof_qty = 2;
+
+        let (resulting_batch_queue, finalized_batch) =
+            try_build_batch(batch_queue, gas_price, 5000000, max_batch_proof_qty).unwrap();
+
+        assert_eq!(resulting_batch_queue.len(), 1);
+        assert_eq!(finalized_batch.len(), 2);
+        assert_eq!(
+            finalized_batch[0].nonced_verification_data.max_fee,
+            max_fee_2
+        );
+        assert_eq!(
+            finalized_batch[1].nonced_verification_data.max_fee,
+            max_fee_1
+        );
+    }
 }

config-files/config-batcher-docker.yaml

@@ -20,7 +20,8 @@ batcher:
   batch_size_interval: 10
   transaction_wait_timeout: 36000 # 3 blocks
   max_proof_size: 67108864 # 64 MiB
-  max_batch_size: 268435456 # 256 MiB
+  max_batch_byte_size: 268435456 # 256 MiB
+  max_batch_proof_qty: 3000 # 3000 proofs in a batch
   eth_ws_reconnects: 99999999999999
   pre_verification_is_enabled: true
   metrics_port: 9093

config-files/config-batcher.yaml

@@ -20,7 +20,8 @@ batcher:
   batch_size_interval: 10
   transaction_wait_timeout: 36000 # 3 blocks
   max_proof_size: 67108864 # 64 MiB
-  max_batch_size: 268435456 # 256 MiB
+  max_batch_byte_size: 268435456 # 256 MiB
+  max_batch_proof_qty: 3000 # 3000 proofs in a batch
   pre_verification_is_enabled: true
   metrics_port: 9093
   telemetry_ip_port_address: localhost:4001

config-files/config-docker.yaml

@@ -24,7 +24,7 @@ batcher:
   block_interval: 3
   batch_size_interval: 10
   max_proof_size: 67108864 # 64 MiB
-  max_batch_size: 268435456 # 256 MiB
+  max_batch_byte_size: 268435456 # 256 MiB
   eth_ws_reconnects: 99999999999999
   pre_verification_is_enabled: true
 
@@ -47,7 +47,7 @@ operator:
   metadata_url: "https://yetanotherco.github.io/operator_metadata/metadata.json"
   enable_metrics: true
   metrics_ip_port_address: localhost:9092
-  max_batch_size: 268435456 # 256 MiB
+  max_batch_byte_size: 268435456 # 256 MiB
   last_processed_batch_filepath: config-files/operator.last_processed_batch.json
 # Operators variables needed for register it in EigenLayer
 el_delegation_manager_address: "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9"

config-files/config.yaml

@@ -24,7 +24,7 @@ batcher:
   block_interval: 3
   batch_size_interval: 10
   max_proof_size: 67108864 # 64 MiB
-  max_batch_size: 268435456 # 256 MiB
+  max_batch_byte_size: 268435456 # 256 MiB
   pre_verification_is_enabled: true
   metrics_port: 9093