diff --git a/spartan/.gitignore b/spartan/.gitignore index f4db608c60b7..76a51cd32ce9 100644 --- a/spartan/.gitignore +++ b/spartan/.gitignore @@ -6,7 +6,10 @@ tfplan mnemonic.tmp environments/* !environments/network-defaults.yml -!environments/10-tps-scenario.env +!environments/ten-tps-short-epoch.env +!environments/ten-tps-long-epoch.env +!environments/five-tps-short-epoch.env +!environments/five-tps-long-epoch.env !environments/devnet-avm-prover.env !environments/devnet-next.env !environments/devnet.env diff --git a/spartan/environments/five-tps-long-epoch.env b/spartan/environments/five-tps-long-epoch.env new file mode 100644 index 000000000000..66de0301fd9c --- /dev/null +++ b/spartan/environments/five-tps-long-epoch.env @@ -0,0 +1,73 @@ +NAMESPACE=${NAMESPACE:-five-tps} +CLUSTER=aztec-gke-private +GCP_REGION=us-west1-a +DESTROY_NAMESPACE=true +DESTROY_ETH_DEVNET=true +CREATE_ETH_DEVNET=${CREATE_ETH_DEVNET:-true} +AZTEC_EPOCH_DURATION=32 +AZTEC_SLOT_DURATION=36 +AZTEC_PROOF_SUBMISSION_WINDOW=64 +ETHEREUM_CHAIN_ID=1337 +LABS_INFRA_MNEMONIC="test test test test test test test test test test test junk" +FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" +# CREATE_CHAOS_MESH=true + +CREATE_ROLLUP_CONTRACTS=true +REDEPLOY_ROLLUP_CONTRACTS=true +VERIFY_CONTRACTS=false +DESTROY_AZTEC_INFRA=true + +AZTEC_LAG_IN_EPOCHS=1 + +OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET + +VALIDATOR_REPLICAS=12 +VALIDATORS_PER_NODE=4 +PUBLISHERS_PER_VALIDATOR_KEY=2 +VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX=5000 + +REAL_VERIFIER=false + +RPC_REPLICAS=12 +RPC_INGRESS_ENABLED=false + +FULL_NODE_REPLICAS=500 +FULL_NODE_RESOURCE_PROFILE="2-core-spot" + +PUBLISHERS_PER_PROVER=2 +PROVER_PUBLISHER_MNEMONIC_START_INDEX=8000 +PROVER_REPLICAS=64 +PROVER_RESOURCE_PROFILE="hi-tps" +PROVER_AGENT_POLL_INTERVAL_MS=10000 + +RUN_TESTS=false + +P2P_MAX_TX_POOL_SIZE=1000000000 +PROVER_TEST_DELAY_TYPE=fixed + +AZTEC_SLASHING_ROUND_SIZE_IN_EPOCHS=1 +AZTEC_SLASHING_QUORUM=20 +AZTEC_SLASHING_EXECUTION_DELAY_IN_ROUNDS=0 +AZTEC_SLASHING_OFFSET_IN_ROUNDS=1 +AZTEC_LOCAL_EJECTION_THRESHOLD=90000000000000000000 + +SEQ_MAX_TX_PER_BLOCK=180 +SEQ_MIN_TX_PER_BLOCK=0 + +# Override L1 tx utils bump percentages for scenario tests +VALIDATOR_L1_PRIORITY_FEE_BUMP_PERCENTAGE=0 +VALIDATOR_L1_PRIORITY_FEE_RETRY_BUMP_PERCENTAGE=0 +PROVER_L1_PRIORITY_FEE_BUMP_PERCENTAGE=0 +PROVER_L1_PRIORITY_FEE_RETRY_BUMP_PERCENTAGE=0 + +# Enable latency mesaruement for p2p messages +DEBUG_P2P_INSTRUMENT_MESSAGES=true + +# Inject artificial delay of proof verification for all nodes +PROVER_TEST_VERIFICATION_DELAY_MS=250 + +# Reduce the amount of metrics produced by prover agents and full nodes +PROVER_AGENT_INCLUDE_METRICS="aztec.circuit" +FULL_NODE_INCLUDE_METRICS="aztec.p2p.gossip.agg_" +LOG_LEVEL=info + diff --git a/spartan/environments/five-tps-short-epoch.env b/spartan/environments/five-tps-short-epoch.env new file mode 100644 index 000000000000..c4cfd82e9e34 --- /dev/null +++ b/spartan/environments/five-tps-short-epoch.env @@ -0,0 +1,73 @@ +NAMESPACE=${NAMESPACE:-five-tps} +CLUSTER=aztec-gke-private +GCP_REGION=us-west1-a +DESTROY_NAMESPACE=true +DESTROY_ETH_DEVNET=true +CREATE_ETH_DEVNET=${CREATE_ETH_DEVNET:-true} +AZTEC_EPOCH_DURATION=8 +AZTEC_SLOT_DURATION=36 +AZTEC_PROOF_SUBMISSION_WINDOW=16 +ETHEREUM_CHAIN_ID=1337 +LABS_INFRA_MNEMONIC="test test test test test test test test test test test junk" +FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" +# CREATE_CHAOS_MESH=true + +CREATE_ROLLUP_CONTRACTS=true +REDEPLOY_ROLLUP_CONTRACTS=true +VERIFY_CONTRACTS=false +DESTROY_AZTEC_INFRA=true + +AZTEC_LAG_IN_EPOCHS=1 + +OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET + +VALIDATOR_REPLICAS=12 +VALIDATORS_PER_NODE=4 +PUBLISHERS_PER_VALIDATOR_KEY=2 +VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX=5000 + +REAL_VERIFIER=false + +RPC_REPLICAS=12 +RPC_INGRESS_ENABLED=false + +FULL_NODE_REPLICAS=500 +FULL_NODE_RESOURCE_PROFILE="2-core-spot" + +PUBLISHERS_PER_PROVER=2 +PROVER_PUBLISHER_MNEMONIC_START_INDEX=8000 +PROVER_REPLICAS=64 +PROVER_RESOURCE_PROFILE="hi-tps" +PROVER_AGENT_POLL_INTERVAL_MS=10000 + +RUN_TESTS=false + +P2P_MAX_TX_POOL_SIZE=1000000000 +PROVER_TEST_DELAY_TYPE=fixed + +AZTEC_SLASHING_ROUND_SIZE_IN_EPOCHS=1 +AZTEC_SLASHING_QUORUM=5 +AZTEC_SLASHING_EXECUTION_DELAY_IN_ROUNDS=0 +AZTEC_SLASHING_OFFSET_IN_ROUNDS=1 +AZTEC_LOCAL_EJECTION_THRESHOLD=90000000000000000000 + +SEQ_MAX_TX_PER_BLOCK=180 +SEQ_MIN_TX_PER_BLOCK=0 + +# Override L1 tx utils bump percentages for scenario tests +VALIDATOR_L1_PRIORITY_FEE_BUMP_PERCENTAGE=0 +VALIDATOR_L1_PRIORITY_FEE_RETRY_BUMP_PERCENTAGE=0 +PROVER_L1_PRIORITY_FEE_BUMP_PERCENTAGE=0 +PROVER_L1_PRIORITY_FEE_RETRY_BUMP_PERCENTAGE=0 + +# Enable latency mesaruement for p2p messages +DEBUG_P2P_INSTRUMENT_MESSAGES=true + +# Inject artificial delay of proof verification for all nodes +PROVER_TEST_VERIFICATION_DELAY_MS=250 + +# Reduce the amount of metrics produced by prover agents and full nodes +PROVER_AGENT_INCLUDE_METRICS="aztec.circuit" +FULL_NODE_INCLUDE_METRICS="aztec.p2p.gossip.agg_" +LOG_LEVEL=info + diff --git a/spartan/environments/10-tps-scenario.env b/spartan/environments/ten-tps-long-epoch.env similarity index 95% rename from spartan/environments/10-tps-scenario.env rename to spartan/environments/ten-tps-long-epoch.env index beb2cf0d0934..799e974dc6f6 100644 --- a/spartan/environments/10-tps-scenario.env +++ b/spartan/environments/ten-tps-long-epoch.env @@ -1,11 +1,11 @@ -NAMESPACE=${NAMESPACE:-10-tps} +NAMESPACE=${NAMESPACE:-ten-tps} CLUSTER=aztec-gke-private GCP_REGION=us-west1-a DESTROY_NAMESPACE=true DESTROY_ETH_DEVNET=true CREATE_ETH_DEVNET=${CREATE_ETH_DEVNET:-true} AZTEC_EPOCH_DURATION=32 -AZTEC_SLOT_DURATION=72 +AZTEC_SLOT_DURATION=36 AZTEC_PROOF_SUBMISSION_WINDOW=64 ETHEREUM_CHAIN_ID=1337 LABS_INFRA_MNEMONIC="test test test test test test test test test test test junk" @@ -51,7 +51,7 @@ AZTEC_SLASHING_EXECUTION_DELAY_IN_ROUNDS=0 AZTEC_SLASHING_OFFSET_IN_ROUNDS=1 AZTEC_LOCAL_EJECTION_THRESHOLD=90000000000000000000 -SEQ_MAX_TX_PER_BLOCK=720 +SEQ_MAX_TX_PER_BLOCK=360 SEQ_MIN_TX_PER_BLOCK=0 # Override L1 tx utils bump percentages for scenario tests diff --git a/spartan/environments/ten-tps-short-epoch.env b/spartan/environments/ten-tps-short-epoch.env new file mode 100644 index 000000000000..9eb65492dedf --- /dev/null +++ b/spartan/environments/ten-tps-short-epoch.env @@ -0,0 +1,73 @@ +NAMESPACE=${NAMESPACE:-ten-tps} +CLUSTER=aztec-gke-private +GCP_REGION=us-west1-a +DESTROY_NAMESPACE=true +DESTROY_ETH_DEVNET=true +CREATE_ETH_DEVNET=${CREATE_ETH_DEVNET:-true} +AZTEC_EPOCH_DURATION=8 +AZTEC_SLOT_DURATION=36 +AZTEC_PROOF_SUBMISSION_WINDOW=16 +ETHEREUM_CHAIN_ID=1337 +LABS_INFRA_MNEMONIC="test test test test test test test test test test test junk" +FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" +# CREATE_CHAOS_MESH=true + +CREATE_ROLLUP_CONTRACTS=true +REDEPLOY_ROLLUP_CONTRACTS=true +VERIFY_CONTRACTS=false +DESTROY_AZTEC_INFRA=true + +AZTEC_LAG_IN_EPOCHS=1 + +OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET + +VALIDATOR_REPLICAS=12 +VALIDATORS_PER_NODE=4 +PUBLISHERS_PER_VALIDATOR_KEY=2 +VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX=5000 + +REAL_VERIFIER=false + +RPC_REPLICAS=12 +RPC_INGRESS_ENABLED=false + +FULL_NODE_REPLICAS=500 +FULL_NODE_RESOURCE_PROFILE="2-core-spot" + +PUBLISHERS_PER_PROVER=2 +PROVER_PUBLISHER_MNEMONIC_START_INDEX=8000 +PROVER_REPLICAS=128 +PROVER_RESOURCE_PROFILE="hi-tps" +PROVER_AGENT_POLL_INTERVAL_MS=10000 + +RUN_TESTS=false + +P2P_MAX_TX_POOL_SIZE=1000000000 +PROVER_TEST_DELAY_TYPE=fixed + +AZTEC_SLASHING_ROUND_SIZE_IN_EPOCHS=1 +AZTEC_SLASHING_QUORUM=5 +AZTEC_SLASHING_EXECUTION_DELAY_IN_ROUNDS=0 +AZTEC_SLASHING_OFFSET_IN_ROUNDS=1 +AZTEC_LOCAL_EJECTION_THRESHOLD=90000000000000000000 + +SEQ_MAX_TX_PER_BLOCK=360 +SEQ_MIN_TX_PER_BLOCK=0 + +# Override L1 tx utils bump percentages for scenario tests +VALIDATOR_L1_PRIORITY_FEE_BUMP_PERCENTAGE=0 +VALIDATOR_L1_PRIORITY_FEE_RETRY_BUMP_PERCENTAGE=0 +PROVER_L1_PRIORITY_FEE_BUMP_PERCENTAGE=0 +PROVER_L1_PRIORITY_FEE_RETRY_BUMP_PERCENTAGE=0 + +# Enable latency mesaruement for p2p messages +DEBUG_P2P_INSTRUMENT_MESSAGES=true + +# Inject artificial delay of proof verification for all nodes +PROVER_TEST_VERIFICATION_DELAY_MS=250 + +# Reduce the amount of metrics produced by prover agents and full nodes +PROVER_AGENT_INCLUDE_METRICS="aztec.circuit" +FULL_NODE_INCLUDE_METRICS="aztec.p2p.gossip.agg_" +LOG_LEVEL=info + diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.ts index 8f8d774832a9..302d90464a47 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.ts @@ -73,6 +73,11 @@ import { TxProvingState } from './tx-proving-state.js'; const logger = createLogger('prover-client:orchestrator'); +type WorldStateFork = { + fork: MerkleTreeWriteOperations; + cleanupPromise: Promise | undefined; +}; + /** * Implements an event driven proving scheduler to build the recursive proof tree. The idea being: * 1. Transactions are provided to the scheduler post simulation. @@ -94,7 +99,7 @@ export class ProvingOrchestrator implements EpochProver { private provingPromise: Promise | undefined = undefined; private metrics: ProvingOrchestratorMetrics; // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections - private dbs: Map = new Map(); + private dbs: Map = new Map(); constructor( private dbProvider: ReadonlyWorldStateAccess & ForkMerkleTreeOperations, @@ -175,7 +180,7 @@ export class ProvingOrchestrator implements EpochProver { const db = await this.dbProvider.fork(lastBlockNumber); const firstBlockNumber = BlockNumber(lastBlockNumber + 1); - this.dbs.set(firstBlockNumber, db); + this.dbs.set(firstBlockNumber, { fork: db, cleanupPromise: undefined }); // Get archive sibling path before any block in this checkpoint lands. const lastArchiveSiblingPath = await getLastSiblingPath(MerkleTreeId.ARCHIVE, db); @@ -233,9 +238,9 @@ export class ProvingOrchestrator implements EpochProver { if (!this.dbs.has(blockNumber)) { // Fork world state at the end of the immediately previous block const db = await this.dbProvider.fork(BlockNumber(blockNumber - 1)); - this.dbs.set(blockNumber, db); + this.dbs.set(blockNumber, { fork: db, cleanupPromise: undefined }); } - const db = this.dbs.get(blockNumber)!; + const db = this.dbs.get(blockNumber)!.fork; // Get archive snapshot and sibling path before any txs in this block lands. const lastArchiveTreeSnapshot = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db); @@ -310,7 +315,7 @@ export class ProvingOrchestrator implements EpochProver { logger.info(`Adding ${txs.length} transactions to block ${blockNumber}`); - const db = this.dbs.get(blockNumber)!; + const db = this.dbs.get(blockNumber)!.fork; const lastArchive = provingState.lastArchiveTreeSnapshot; const newL1ToL2MessageTreeSnapshot = provingState.newL1ToL2MessageTreeSnapshot; const spongeBlobState = provingState.getStartSpongeBlob().clone(); @@ -439,7 +444,7 @@ export class ProvingOrchestrator implements EpochProver { } // Get db for this block - const db = this.dbs.get(provingState.blockNumber)!; + const db = this.dbs.get(provingState.blockNumber)!.fork; // Update the archive tree, so we're ready to start processing the next block: logger.verbose( @@ -475,7 +480,7 @@ export class ProvingOrchestrator implements EpochProver { // Get db for this block const blockNumber = provingState.blockNumber; - const db = this.dbs.get(blockNumber)!; + const db = this.dbs.get(blockNumber)!.fork; const newArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db); const syncedArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, this.dbProvider.getSnapshot(blockNumber)); @@ -551,7 +556,10 @@ export class ProvingOrchestrator implements EpochProver { } try { - await fork.close(); + if (!fork.cleanupPromise) { + fork.cleanupPromise = fork.fork.close(); + } + await fork.cleanupPromise; this.dbs.delete(blockNumber); } catch (err) { logger.error(`Error closing db for block ${blockNumber}`, err);