Merge remote-tracking branch 'origin/staging' into feat/aggregation-mode-explorer

Author: MarcosNicolau

Makefile

@@ -7,7 +7,7 @@ CONFIG_FILE?=config-files/config.yaml
 export OPERATOR_ADDRESS ?= $(shell yq -r '.operator.address' $(CONFIG_FILE))
 AGG_CONFIG_FILE?=config-files/config-aggregator.yaml
 
-OPERATOR_VERSION=v0.15.0
+OPERATOR_VERSION=v0.15.1
 EIGEN_SDK_GO_VERSION_TESTNET=v0.2.0-beta.1
 EIGEN_SDK_GO_VERSION_MAINNET=v0.1.13
 
@@ -20,8 +20,8 @@ ifeq ($(OS),Darwin)
 endif
 
 ifeq ($(OS),Linux)
-	export LD_LIBRARY_PATH+=$(CURDIR)/operator/risc_zero/lib
-	OPERATOR_FFIS=$(CURDIR)/operator/risc_zero/lib
+	export LD_LIBRARY_PATH+=$(CURDIR)/operator/risc_zero/lib:$(CURDIR)/operator/sp1/lib
+	OPERATOR_FFIS=$(CURDIR)/operator/risc_zero/lib:$(CURDIR)/operator/sp1/lib
 endif
 
 ifeq ($(OS),Linux)
@@ -150,11 +150,11 @@ anvil_start_with_block_time_with_more_prefunded_accounts:
 	@echo "Starting Anvil..."
 	anvil --load-state contracts/scripts/anvil/state/alignedlayer-deployed-anvil-state.json --block-time 7 -a 2000
 
-__AGGREGATION_MODE__:
-start_proof_aggregator_local:
+__AGGREGATION_MODE__: ## ____
+start_proof_aggregator_local: ## Start the proof aggregator locally using Mock Verifier Contract
 	cargo run --manifest-path ./aggregation_mode/Cargo.toml --release -- config-files/config-proof-aggregator.yaml
 
-start_proof_aggregator_local_with_proving:
+start_proof_aggregator_local_with_proving: ## Start the proof aggregator locally using SP1 Verifier Contract
 	cargo run --manifest-path ./aggregation_mode/Cargo.toml --release --features prove -- config-files/config-proof-aggregator.yaml
 
 _AGGREGATOR_:
@@ -425,19 +425,19 @@ batcher_send_risc0_task:
 	@echo "Sending Risc0 fibonacci task to Batcher..."
 	@cd batcher/aligned/ && cargo run --release -- submit \
 		--proving_system Risc0 \
-		--proof ../../scripts/test_files/risc_zero/fibonacci_proof_generator/risc_zero_fibonacci.proof \
-        --vm_program ../../scripts/test_files/risc_zero/fibonacci_proof_generator/fibonacci_id.bin \
-        --public_input ../../scripts/test_files/risc_zero/fibonacci_proof_generator/risc_zero_fibonacci.pub \
+		--proof ../../scripts/test_files/risc_zero/fibonacci_proof_generator/risc_zero_fibonacci_2_0.proof \
+        --vm_program ../../scripts/test_files/risc_zero/fibonacci_proof_generator/fibonacci_id_2_0.bin \
+        --public_input ../../scripts/test_files/risc_zero/fibonacci_proof_generator/risc_zero_fibonacci_2_0.pub \
 		--proof_generator_addr 0x66f9664f97F2b50F62D13eA064982f936dE76657 \
 		--rpc_url $(RPC_URL) \
 		--network $(NETWORK)
 
 batcher_send_risc0_task_no_pub_input:
-	@echo "Sending Risc0 fibonacci task to Batcher..."
+	@echo "Sending Risc0 no pub input task to Batcher..."
 	@cd batcher/aligned/ && cargo run --release -- submit \
 		--proving_system Risc0 \
-		--proof ../../scripts/test_files/risc_zero/no_public_inputs/risc_zero_no_pub_input.proof \
-        --vm_program ../../scripts/test_files/risc_zero/no_public_inputs/no_pub_input_id.bin \
+		--proof ../../scripts/test_files/risc_zero/no_public_inputs/risc_zero_no_pub_input_2_0.proof \
+        --vm_program ../../scripts/test_files/risc_zero/no_public_inputs/no_pub_input_id_2_0.bin \
 		--proof_generator_addr 0x66f9664f97F2b50F62D13eA064982f936dE76657 \
 		--rpc_url $(RPC_URL) \
 		--network $(NETWORK)
@@ -446,9 +446,9 @@ batcher_send_risc0_burst:
 	@echo "Sending Risc0 fibonacci task to Batcher..."
 	@cd batcher/aligned/ && cargo run --release -- submit \
 		--proving_system Risc0 \
-		--proof ../../scripts/test_files/risc_zero/fibonacci_proof_generator/risc_zero_fibonacci.proof \
-        --vm_program ../../scripts/test_files/risc_zero/fibonacci_proof_generator/fibonacci_id.bin \
-        --public_input ../../scripts/test_files/risc_zero/fibonacci_proof_generator/risc_zero_fibonacci.pub \
+		--proof ../../scripts/test_files/risc_zero/fibonacci_proof_generator/risc_zero_fibonacci_2_0.proof \
+        --vm_program ../../scripts/test_files/risc_zero/fibonacci_proof_generator/fibonacci_id_2_0.bin \
+        --public_input ../../scripts/test_files/risc_zero/fibonacci_proof_generator/risc_zero_fibonacci_2_0.pub \
         --repetitions $(BURST_SIZE) \
 		--proof_generator_addr 0x66f9664f97F2b50F62D13eA064982f936dE76657 \
 		--rpc_url $(RPC_URL) \
@@ -952,9 +952,9 @@ docker_batcher_send_risc0_burst:
 	docker exec $(shell docker ps | grep batcher | awk '{print $$1}') aligned submit \
               --private_key $(DOCKER_PROOFS_PRIVATE_KEY) \
               --proving_system Risc0 \
-              --proof ./scripts/test_files/risc_zero/fibonacci_proof_generator/risc_zero_fibonacci.proof \
-              --vm_program ./scripts/test_files/risc_zero/fibonacci_proof_generator/fibonacci_id.bin \
-              --public_input ./scripts/test_files/risc_zero/fibonacci_proof_generator/risc_zero_fibonacci.pub \
+              --proof ./scripts/test_files/risc_zero/fibonacci_proof_generator/risc_zero_fibonacci_2_0.proof \
+              --vm_program ./scripts/test_files/risc_zero/fibonacci_proof_generator/fibonacci_id_2_0.bin \
+              --public_input ./scripts/test_files/risc_zero/fibonacci_proof_generator/risc_zero_fibonacci_2_0.pub \
               --repetitions $(DOCKER_BURST_SIZE) \
               --proof_generator_addr $(PROOF_GENERATOR_ADDRESS) \
               --rpc_url $(DOCKER_RPC_URL) \

aggregation_mode/Cargo.lock

@@ -12,7 +12,7 @@ serde_json = "1.0.117"
 serde_yaml = "0.9"
 tracing = { version = "0.1", features = ["log"] }
 tracing-subscriber = { version = "0.3.0", features = ["env-filter"] }
-alloy = { version = "0.11", features = ["default", "signer-keystore"] }
+alloy = { version = "0.11", features = ["default", "signer-keystore", "kzg"] }
 c-kzg = "1.0.3"
 bincode = "1.3.3"
 tokio = { version = "1", features = ["time"]}
@@ -27,7 +27,6 @@ sp1-build = { version = "4.1.3" }
 
 [profile.release]
 opt-level = 3
-lto = true
 
 [features]
 prove = []

aggregation_mode/Cargo.toml

@@ -0,0 +1,57 @@
+
+# Aligned aggregation mode
+
+## How to run it locally
+
+1. Deploy aligned contracts on anvil:
+
+```shell
+make deploy_aligned_contracts
+```
+
+2. Start anvil:
+
+```shell
+make anvil_start_with_block_time
+```
+
+3. Start batcher
+
+```shell
+make start_batcher_local
+```
+
+4. Send SP1 proofs:
+
+```shell
+make batcher_send_sp1_burst
+```
+
+5. Start proof aggregator:
+
+```shell
+# This will not run a real prover but a mocked one see below to run a real prover
+make start_proof_aggregator_local
+```
+
+or
+
+```shell
+make start_proof_aggregator_local_with_proving
+```
+
+Note: Proving can be quite slow without GPUs
+
+### Check the logs
+
+1. Get latest aggregated proof:
+
+```shell
+cast call 0xcbEAF3BDe82155F56486Fb5a1072cb8baAf547cc "currentAggregatedProofNumber()" --rpc-url http://localhost:8545
+```
+
+2. Get aggregated proof info:
+
+```shell
+cast call 0xcbEAF3BDe82155F56486Fb5a1072cb8baAf547cc "getAggregatedProof(uint64)(uint8,bytes32,bytes32)" <AGG_PROOF_NUMBER>  --rpc-url http://localhost:8545
+```

aggregation_mode/README.md

@@ -11,7 +11,6 @@ pub struct ECDSAConfig {
 pub struct Config {
     pub eth_rpc_url: String,
     pub eth_ws_url: String,
-    pub private_key: String,
     pub max_proofs_in_queue: u16,
     pub proof_aggregation_service_address: String,
     pub aligned_service_manager_address: String,

aggregation_mode/src/backend/config.rs

@@ -5,7 +5,8 @@ use super::{
     types::{AlignedLayerServiceManager, AlignedLayerServiceManagerContract, RPCProvider},
 };
 use crate::{
-    aggregators::{sp1_aggregator::SP1ProofWithPubValuesAndElf, AlignedProof}, backend::s3::get_aligned_batch_from_s3
+    aggregators::{sp1_aggregator::SP1ProofWithPubValuesAndElf, AlignedProof},
+    backend::s3::get_aligned_batch_from_s3,
 };
 use aligned_sdk::core::types::ProvingSystemId;
 use alloy::{
@@ -29,11 +30,11 @@ pub struct ProofsFetcher {
 
 impl ProofsFetcher {
     pub fn new(config: &Config) -> Self {
-        let rpc_url = config.eth_rpc_url.parse().expect("correct url");
+        let rpc_url = config.eth_rpc_url.parse().expect("RPC URL should be valid");
         let rpc_provider = ProviderBuilder::new().on_http(rpc_url);
         let aligned_service_manager = AlignedLayerServiceManager::new(
             Address::from_str(&config.aligned_service_manager_address)
-                .expect("Address to be correct"),
+                .expect("AlignedProofAggregationService address should be valid"),
             rpc_provider.clone(),
         );
 
@@ -88,7 +89,10 @@ impl ProofsFetcher {
                     ProvingSystemId::SP1 => {
                         let elf = p.vm_program_code?;
                         let proof_with_pub_values = bincode::deserialize(&p.proof).ok()?;
-                        let sp1_proof = SP1ProofWithPubValuesAndElf { proof_with_pub_values, elf };
+                        let sp1_proof = SP1ProofWithPubValuesAndElf {
+                            proof_with_pub_values,
+                            elf,
+                        };
 
                         Some(AlignedProof::SP1(sp1_proof))
                     }

aggregation_mode/src/backend/fetcher.rs

@@ -10,14 +10,7 @@ pub fn combine_hashes(hash_a: &[u8; 32], hash_b: &[u8; 32]) -> [u8; 32] {
 
 /// Returns (merkle_root, leaves)
 pub fn compute_proofs_merkle_root(proofs: &[AlignedProof]) -> ([u8; 32], Vec<[u8; 32]>) {
-    let leaves: Vec<[u8; 32]> = proofs
-        .chunks(2)
-        .map(|chunk| match chunk {
-            [a, b] => combine_hashes(&a.hash(), &b.hash()),
-            [a] => combine_hashes(&a.hash(), &a.hash()),
-            _ => panic!("Unexpected chunk leaves"),
-        })
-        .collect();
+    let leaves: Vec<[u8; 32]> = proofs.iter().map(|proof| proof.hash()).collect();
 
     let mut root = leaves.clone();
 

aggregation_mode/src/backend/merkle_tree.rs

@@ -4,15 +4,18 @@ mod merkle_tree;
 mod s3;
 mod types;
 
-use crate::aggregators::{lib::{AggregatedProof, ProofAggregationError}, sp1_aggregator::{aggregate_proofs, SP1AggregationInput}, AlignedProof, ZKVMEngine};
-
+use crate::aggregators::{
+    lib::{AggregatedProof, ProofAggregationError},
+    sp1_aggregator::{aggregate_proofs, SP1AggregationInput},
+    AlignedProof, ZKVMEngine,
+};
 
 use alloy::{
-    consensus::{Blob, BlobTransactionSidecar},
+    consensus::BlobTransactionSidecar,
     eips::eip4844::BYTES_PER_BLOB,
     hex,
     network::EthereumWallet,
-    primitives::{Address, FixedBytes},
+    primitives::Address,
     providers::{PendingTransactionError, ProviderBuilder},
     rpc::types::TransactionReceipt,
     signers::local::LocalSigner,
@@ -25,7 +28,6 @@ use std::str::FromStr;
 use tracing::{error, info, warn};
 use types::{AlignedProofAggregationService, AlignedProofAggregationServiceContract};
 
-
 #[derive(Debug)]
 pub enum AggregatedProofSubmissionError {
     Aggregation(ProofAggregationError),
@@ -45,17 +47,17 @@ pub struct ProofAggregator {
 
 impl ProofAggregator {
     pub fn new(config: &Config) -> Self {
-        let rpc_url = config.eth_rpc_url.parse().expect("correct url");
+        let rpc_url = config.eth_rpc_url.parse().expect("RPC URL should be valid");
         let signer = LocalSigner::decrypt_keystore(
             config.ecdsa.private_key_store_path.clone(),
             config.ecdsa.private_key_store_password.clone(),
         )
-        .expect("Correct keystore signer");
+        .expect("Keystore signer should be `cast wallet` compliant");
         let wallet = EthereumWallet::from(signer);
         let rpc_provider = ProviderBuilder::new().wallet(wallet).on_http(rpc_url);
-        let proof_aggregation_service: AlignedProofAggregationService::AlignedProofAggregationServiceInstance<(), alloy::providers::fillers::FillProvider<alloy::providers::fillers::JoinFill<alloy::providers::fillers::JoinFill<alloy::providers::Identity, alloy::providers::fillers::JoinFill<alloy::providers::fillers::GasFiller, alloy::providers::fillers::JoinFill<alloy::providers::fillers::BlobGasFiller, alloy::providers::fillers::JoinFill<alloy::providers::fillers::NonceFiller, alloy::providers::fillers::ChainIdFiller>>>>, alloy::providers::fillers::WalletFiller<EthereumWallet>>, alloy::providers::RootProvider>> = AlignedProofAggregationService::new(
+        let proof_aggregation_service = AlignedProofAggregationService::new(
             Address::from_str(&config.proof_aggregation_service_address)
-                .expect("Address to be correct"),
+                .expect("AlignedProofAggregationService address should be valid"),
             rpc_provider,
         );
         let fetcher = ProofsFetcher::new(config);
@@ -122,8 +124,7 @@ impl ProofAggregator {
                     merkle_root,
                 };
 
-                aggregate_proofs(input)
-                    .map_err(AggregatedProofSubmissionError::Aggregation)?
+                aggregate_proofs(input).map_err(AggregatedProofSubmissionError::Aggregation)?
             }
         };
         info!("Proof aggregation program finished");
@@ -184,8 +185,17 @@ impl ProofAggregator {
         let data: Vec<u8> = leaves.iter().flat_map(|arr| arr.iter().copied()).collect();
         let mut blob_data: [u8; BYTES_PER_BLOB] = [0u8; BYTES_PER_BLOB];
 
-        for (i, byte) in data.iter().enumerate() {
-            blob_data[i] = *byte;
+        // We pad the data with 0x0 byte every 31 bytes so that the field elements
+        // constructed from the bytes are less than BLS_MODULUS.
+        //
+        // See https://github.com/ethereum/consensus-specs/blob/86fb82b221474cc89387fa6436806507b3849d88/specs/deneb/polynomial-commitments.md#bytes_to_bls_field
+        let mut offset = 0;
+        for chunk in data.chunks(31) {
+            blob_data[offset] = 0x00;
+            let start = offset + 1;
+            let end = start + chunk.len();
+            blob_data[start..end].copy_from_slice(chunk);
+            offset += 32;
         }
 
         // calculate kzg commitments for blob
@@ -197,12 +207,11 @@ impl ProofAggregator {
             c_kzg::KzgProof::compute_blob_kzg_proof(&blob, &commitment.to_bytes(), settings)
                 .map_err(|_| AggregatedProofSubmissionError::BuildingBlobProof)?;
 
-        // convert to alloy types
-        let blob = Blob::from_slice(&blob_data);
-        let commitment: FixedBytes<48> = FixedBytes::from_slice(commitment.to_bytes().as_slice());
-        let proof: FixedBytes<48> = FixedBytes::from_slice(proof.to_bytes().as_slice());
-
-        let blob = BlobTransactionSidecar::new(vec![blob], vec![commitment], vec![proof]);
+        let blob = BlobTransactionSidecar::from_kzg(
+            vec![blob],
+            vec![commitment.to_bytes()],
+            vec![proof.to_bytes()],
+        );
         let blob_versioned_hash = blob
             .versioned_hash_for_blob(0)
             .ok_or(AggregatedProofSubmissionError::BuildingBlobVersionedHash)?

aggregation_mode/src/backend/mod.rs

@@ -6,15 +6,25 @@ pub enum GetBatchProofsError {
     Deserialization,
     EmptyBody,
     StatusFailed,
+    ReqwestClientFailed,
 }
 
+// needed to make S3 bucket work
+const DEFAULT_USER_AGENT: &str = "proof-aggregator/aligned-layer";
+
 pub async fn get_aligned_batch_from_s3(
     url: String,
 ) -> Result<Vec<VerificationData>, GetBatchProofsError> {
-    let response = reqwest::get(url)
+    let client = reqwest::Client::builder()
+        .user_agent(DEFAULT_USER_AGENT)
+        .build()
+        .map_err(|_| GetBatchProofsError::ReqwestClientFailed)?;
+
+    let response = client
+        .get(url)
+        .send()
         .await
         .map_err(|_| GetBatchProofsError::Fetching)?;
-
     if !response.status().is_success() {
         return Err(GetBatchProofsError::StatusFailed);
     }