chore: Fix error log in prover node. Added more test scenarios

spartan/.gitignore

@@ -6,7 +6,10 @@ tfplan
 mnemonic.tmp
 environments/*
 !environments/network-defaults.yml
-!environments/10-tps-scenario.env
+!environments/ten-tps-short-epoch.env
+!environments/ten-tps-long-epoch.env
+!environments/five-tps-short-epoch.env
+!environments/five-tps-long-epoch.env
 !environments/devnet-avm-prover.env
 !environments/devnet-next.env
 !environments/devnet.env

spartan/environments/five-tps-long-epoch.env

@@ -0,0 +1,73 @@
+NAMESPACE=${NAMESPACE:-five-tps}
+CLUSTER=aztec-gke-private
+GCP_REGION=us-west1-a
+DESTROY_NAMESPACE=true
+DESTROY_ETH_DEVNET=true
+CREATE_ETH_DEVNET=${CREATE_ETH_DEVNET:-true}
+AZTEC_EPOCH_DURATION=32
+AZTEC_SLOT_DURATION=36
+AZTEC_PROOF_SUBMISSION_WINDOW=64
+ETHEREUM_CHAIN_ID=1337
+LABS_INFRA_MNEMONIC="test test test test test test test test test test test junk"
+FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
+# CREATE_CHAOS_MESH=true
+
+CREATE_ROLLUP_CONTRACTS=true
+REDEPLOY_ROLLUP_CONTRACTS=true
+VERIFY_CONTRACTS=false
+DESTROY_AZTEC_INFRA=true
+
+AZTEC_LAG_IN_EPOCHS=1
+
+OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET
+
+VALIDATOR_REPLICAS=12
+VALIDATORS_PER_NODE=4
+PUBLISHERS_PER_VALIDATOR_KEY=2
+VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX=5000
+
+REAL_VERIFIER=false
+
+RPC_REPLICAS=12
+RPC_INGRESS_ENABLED=false
+
+FULL_NODE_REPLICAS=500
+FULL_NODE_RESOURCE_PROFILE="2-core-spot"
+
+PUBLISHERS_PER_PROVER=2
+PROVER_PUBLISHER_MNEMONIC_START_INDEX=8000
+PROVER_REPLICAS=64
+PROVER_RESOURCE_PROFILE="hi-tps"
+PROVER_AGENT_POLL_INTERVAL_MS=10000
+
+RUN_TESTS=false
+
+P2P_MAX_TX_POOL_SIZE=1000000000
+PROVER_TEST_DELAY_TYPE=fixed
+
+AZTEC_SLASHING_ROUND_SIZE_IN_EPOCHS=1
+AZTEC_SLASHING_QUORUM=20
+AZTEC_SLASHING_EXECUTION_DELAY_IN_ROUNDS=0
+AZTEC_SLASHING_OFFSET_IN_ROUNDS=1
+AZTEC_LOCAL_EJECTION_THRESHOLD=90000000000000000000
+
+SEQ_MAX_TX_PER_BLOCK=180
+SEQ_MIN_TX_PER_BLOCK=0
+
+# Override L1 tx utils bump percentages for scenario tests
+VALIDATOR_L1_PRIORITY_FEE_BUMP_PERCENTAGE=0
+VALIDATOR_L1_PRIORITY_FEE_RETRY_BUMP_PERCENTAGE=0
+PROVER_L1_PRIORITY_FEE_BUMP_PERCENTAGE=0
+PROVER_L1_PRIORITY_FEE_RETRY_BUMP_PERCENTAGE=0
+
+# Enable latency mesaruement for p2p messages
+DEBUG_P2P_INSTRUMENT_MESSAGES=true
+
+# Inject artificial delay of proof verification for all nodes
+PROVER_TEST_VERIFICATION_DELAY_MS=250
+
+# Reduce the amount of metrics produced by prover agents and full nodes
+PROVER_AGENT_INCLUDE_METRICS="aztec.circuit"
+FULL_NODE_INCLUDE_METRICS="aztec.p2p.gossip.agg_"
+LOG_LEVEL=info
+

spartan/environments/five-tps-short-epoch.env

@@ -0,0 +1,73 @@
+NAMESPACE=${NAMESPACE:-five-tps}
+CLUSTER=aztec-gke-private
+GCP_REGION=us-west1-a
+DESTROY_NAMESPACE=true
+DESTROY_ETH_DEVNET=true
+CREATE_ETH_DEVNET=${CREATE_ETH_DEVNET:-true}
+AZTEC_EPOCH_DURATION=8
+AZTEC_SLOT_DURATION=36
+AZTEC_PROOF_SUBMISSION_WINDOW=16
+ETHEREUM_CHAIN_ID=1337
+LABS_INFRA_MNEMONIC="test test test test test test test test test test test junk"
+FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
+# CREATE_CHAOS_MESH=true
+
+CREATE_ROLLUP_CONTRACTS=true
+REDEPLOY_ROLLUP_CONTRACTS=true
+VERIFY_CONTRACTS=false
+DESTROY_AZTEC_INFRA=true
+
+AZTEC_LAG_IN_EPOCHS=1
+
+OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET
+
+VALIDATOR_REPLICAS=12
+VALIDATORS_PER_NODE=4
+PUBLISHERS_PER_VALIDATOR_KEY=2
+VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX=5000
+
+REAL_VERIFIER=false
+
+RPC_REPLICAS=12
+RPC_INGRESS_ENABLED=false
+
+FULL_NODE_REPLICAS=500
+FULL_NODE_RESOURCE_PROFILE="2-core-spot"
+
+PUBLISHERS_PER_PROVER=2
+PROVER_PUBLISHER_MNEMONIC_START_INDEX=8000
+PROVER_REPLICAS=64
+PROVER_RESOURCE_PROFILE="hi-tps"
+PROVER_AGENT_POLL_INTERVAL_MS=10000
+
+RUN_TESTS=false
+
+P2P_MAX_TX_POOL_SIZE=1000000000
+PROVER_TEST_DELAY_TYPE=fixed
+
+AZTEC_SLASHING_ROUND_SIZE_IN_EPOCHS=1
+AZTEC_SLASHING_QUORUM=5
+AZTEC_SLASHING_EXECUTION_DELAY_IN_ROUNDS=0
+AZTEC_SLASHING_OFFSET_IN_ROUNDS=1
+AZTEC_LOCAL_EJECTION_THRESHOLD=90000000000000000000
+
+SEQ_MAX_TX_PER_BLOCK=180
+SEQ_MIN_TX_PER_BLOCK=0
+
+# Override L1 tx utils bump percentages for scenario tests
+VALIDATOR_L1_PRIORITY_FEE_BUMP_PERCENTAGE=0
+VALIDATOR_L1_PRIORITY_FEE_RETRY_BUMP_PERCENTAGE=0
+PROVER_L1_PRIORITY_FEE_BUMP_PERCENTAGE=0
+PROVER_L1_PRIORITY_FEE_RETRY_BUMP_PERCENTAGE=0
+
+# Enable latency mesaruement for p2p messages
+DEBUG_P2P_INSTRUMENT_MESSAGES=true
+
+# Inject artificial delay of proof verification for all nodes
+PROVER_TEST_VERIFICATION_DELAY_MS=250
+
+# Reduce the amount of metrics produced by prover agents and full nodes
+PROVER_AGENT_INCLUDE_METRICS="aztec.circuit"
+FULL_NODE_INCLUDE_METRICS="aztec.p2p.gossip.agg_"
+LOG_LEVEL=info
+

spartan/environments/10-tps-scenario.env → spartan/environments/ten-tps-long-epoch.env

@@ -1,11 +1,11 @@
-NAMESPACE=${NAMESPACE:-10-tps}
+NAMESPACE=${NAMESPACE:-ten-tps}
 CLUSTER=aztec-gke-private
 GCP_REGION=us-west1-a
 DESTROY_NAMESPACE=true
 DESTROY_ETH_DEVNET=true
 CREATE_ETH_DEVNET=${CREATE_ETH_DEVNET:-true}
 AZTEC_EPOCH_DURATION=32
-AZTEC_SLOT_DURATION=72
+AZTEC_SLOT_DURATION=36
 AZTEC_PROOF_SUBMISSION_WINDOW=64
 ETHEREUM_CHAIN_ID=1337
 LABS_INFRA_MNEMONIC="test test test test test test test test test test test junk"
@@ -51,7 +51,7 @@ AZTEC_SLASHING_EXECUTION_DELAY_IN_ROUNDS=0
 AZTEC_SLASHING_OFFSET_IN_ROUNDS=1
 AZTEC_LOCAL_EJECTION_THRESHOLD=90000000000000000000
 
-SEQ_MAX_TX_PER_BLOCK=720
+SEQ_MAX_TX_PER_BLOCK=360
 SEQ_MIN_TX_PER_BLOCK=0
 
 # Override L1 tx utils bump percentages for scenario tests

spartan/environments/ten-tps-short-epoch.env

@@ -0,0 +1,73 @@
+NAMESPACE=${NAMESPACE:-ten-tps}
+CLUSTER=aztec-gke-private
+GCP_REGION=us-west1-a
+DESTROY_NAMESPACE=true
+DESTROY_ETH_DEVNET=true
+CREATE_ETH_DEVNET=${CREATE_ETH_DEVNET:-true}
+AZTEC_EPOCH_DURATION=8
+AZTEC_SLOT_DURATION=36
+AZTEC_PROOF_SUBMISSION_WINDOW=16
+ETHEREUM_CHAIN_ID=1337
+LABS_INFRA_MNEMONIC="test test test test test test test test test test test junk"
+FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
+# CREATE_CHAOS_MESH=true
+
+CREATE_ROLLUP_CONTRACTS=true
+REDEPLOY_ROLLUP_CONTRACTS=true
+VERIFY_CONTRACTS=false
+DESTROY_AZTEC_INFRA=true
+
+AZTEC_LAG_IN_EPOCHS=1
+
+OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET
+
+VALIDATOR_REPLICAS=12
+VALIDATORS_PER_NODE=4
+PUBLISHERS_PER_VALIDATOR_KEY=2
+VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX=5000
+
+REAL_VERIFIER=false
+
+RPC_REPLICAS=12
+RPC_INGRESS_ENABLED=false
+
+FULL_NODE_REPLICAS=500
+FULL_NODE_RESOURCE_PROFILE="2-core-spot"
+
+PUBLISHERS_PER_PROVER=2
+PROVER_PUBLISHER_MNEMONIC_START_INDEX=8000
+PROVER_REPLICAS=128
+PROVER_RESOURCE_PROFILE="hi-tps"
+PROVER_AGENT_POLL_INTERVAL_MS=10000
+
+RUN_TESTS=false
+
+P2P_MAX_TX_POOL_SIZE=1000000000
+PROVER_TEST_DELAY_TYPE=fixed
+
+AZTEC_SLASHING_ROUND_SIZE_IN_EPOCHS=1
+AZTEC_SLASHING_QUORUM=5
+AZTEC_SLASHING_EXECUTION_DELAY_IN_ROUNDS=0
+AZTEC_SLASHING_OFFSET_IN_ROUNDS=1
+AZTEC_LOCAL_EJECTION_THRESHOLD=90000000000000000000
+
+SEQ_MAX_TX_PER_BLOCK=360
+SEQ_MIN_TX_PER_BLOCK=0
+
+# Override L1 tx utils bump percentages for scenario tests
+VALIDATOR_L1_PRIORITY_FEE_BUMP_PERCENTAGE=0
+VALIDATOR_L1_PRIORITY_FEE_RETRY_BUMP_PERCENTAGE=0
+PROVER_L1_PRIORITY_FEE_BUMP_PERCENTAGE=0
+PROVER_L1_PRIORITY_FEE_RETRY_BUMP_PERCENTAGE=0
+
+# Enable latency mesaruement for p2p messages
+DEBUG_P2P_INSTRUMENT_MESSAGES=true
+
+# Inject artificial delay of proof verification for all nodes
+PROVER_TEST_VERIFICATION_DELAY_MS=250
+
+# Reduce the amount of metrics produced by prover agents and full nodes
+PROVER_AGENT_INCLUDE_METRICS="aztec.circuit"
+FULL_NODE_INCLUDE_METRICS="aztec.p2p.gossip.agg_"
+LOG_LEVEL=info
+

yarn-project/prover-client/src/orchestrator/orchestrator.ts

@@ -73,6 +73,11 @@ import { TxProvingState } from './tx-proving-state.js';
 
 const logger = createLogger('prover-client:orchestrator');
 
+type WorldStateFork = {
+  fork: MerkleTreeWriteOperations;
+  cleanupPromise: Promise<void> | undefined;
+};
+
 /**
  * Implements an event driven proving scheduler to build the recursive proof tree. The idea being:
  * 1. Transactions are provided to the scheduler post simulation.
@@ -94,7 +99,7 @@ export class ProvingOrchestrator implements EpochProver {
   private provingPromise: Promise<ProvingResult> | undefined = undefined;
   private metrics: ProvingOrchestratorMetrics;
   // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
-  private dbs: Map<BlockNumber, MerkleTreeWriteOperations> = new Map();
+  private dbs: Map<BlockNumber, WorldStateFork> = new Map();
 
   constructor(
     private dbProvider: ReadonlyWorldStateAccess & ForkMerkleTreeOperations,
@@ -175,7 +180,7 @@ export class ProvingOrchestrator implements EpochProver {
     const db = await this.dbProvider.fork(lastBlockNumber);
 
     const firstBlockNumber = BlockNumber(lastBlockNumber + 1);
-    this.dbs.set(firstBlockNumber, db);
+    this.dbs.set(firstBlockNumber, { fork: db, cleanupPromise: undefined });
 
     // Get archive sibling path before any block in this checkpoint lands.
     const lastArchiveSiblingPath = await getLastSiblingPath(MerkleTreeId.ARCHIVE, db);
@@ -233,9 +238,9 @@ export class ProvingOrchestrator implements EpochProver {
     if (!this.dbs.has(blockNumber)) {
       // Fork world state at the end of the immediately previous block
       const db = await this.dbProvider.fork(BlockNumber(blockNumber - 1));
-      this.dbs.set(blockNumber, db);
+      this.dbs.set(blockNumber, { fork: db, cleanupPromise: undefined });
     }
-    const db = this.dbs.get(blockNumber)!;
+    const db = this.dbs.get(blockNumber)!.fork;
 
     // Get archive snapshot and sibling path before any txs in this block lands.
     const lastArchiveTreeSnapshot = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db);
@@ -310,7 +315,7 @@ export class ProvingOrchestrator implements EpochProver {
 
     logger.info(`Adding ${txs.length} transactions to block ${blockNumber}`);
 
-    const db = this.dbs.get(blockNumber)!;
+    const db = this.dbs.get(blockNumber)!.fork;
     const lastArchive = provingState.lastArchiveTreeSnapshot;
     const newL1ToL2MessageTreeSnapshot = provingState.newL1ToL2MessageTreeSnapshot;
     const spongeBlobState = provingState.getStartSpongeBlob().clone();
@@ -439,7 +444,7 @@ export class ProvingOrchestrator implements EpochProver {
     }
 
     // Get db for this block
-    const db = this.dbs.get(provingState.blockNumber)!;
+    const db = this.dbs.get(provingState.blockNumber)!.fork;
 
     // Update the archive tree, so we're ready to start processing the next block:
     logger.verbose(
@@ -475,7 +480,7 @@ export class ProvingOrchestrator implements EpochProver {
 
     // Get db for this block
     const blockNumber = provingState.blockNumber;
-    const db = this.dbs.get(blockNumber)!;
+    const db = this.dbs.get(blockNumber)!.fork;
 
     const newArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db);
     const syncedArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, this.dbProvider.getSnapshot(blockNumber));
@@ -551,7 +556,10 @@ export class ProvingOrchestrator implements EpochProver {
     }
 
     try {
-      await fork.close();
+      if (!fork.cleanupPromise) {
+        fork.cleanupPromise = fork.fork.close();
+      }
+      await fork.cleanupPromise;
       this.dbs.delete(blockNumber);
     } catch (err) {
       logger.error(`Error closing db for block ${blockNumber}`, err);