Merged new ST Prover with atomic transactions arch

Author: rpanic

packages/common/src/index.ts

@@ -23,4 +23,3 @@ export * from "./compiling/AtomicCompileHelper";
 export * from "./compiling/CompileRegistry";
 export * from "./compiling/CompilableModule";
 export * from "./compiling/services/ChildVerificationKeyService";
-export * from "./config/injectAlias";

packages/persistance/src/services/prisma/PrismaBlockStorage.ts

@@ -121,8 +121,7 @@ export class PrismaBlockStorage
                 statusMessage: tx.statusMessage,
                 txHash: tx.txHash,
 
-                stateTransitions:
-                  tx.stateTransitions as Prisma.InputJsonArray,
+                stateTransitions: tx.stateTransitions as Prisma.InputJsonArray,
                 events: tx.events as Prisma.InputJsonArray,
               };
             }),

packages/sequencer/src/protocol/production/BatchProducerModule.ts

@@ -26,9 +26,7 @@ export type StateRecord = Record<string, Field[] | undefined>;
 
 interface BatchMetadata {
   batch: SettleableBatch;
-  changes: {
-    commit: () => Promise<void>;
-  };
+  changes: CachedMerkleTreeStore;
 }
 
 const errors = {
@@ -61,60 +59,19 @@ export class BatchProducerModule extends SequencerModule {
     super();
   }
 
-  // TODO
-  private async applyStateChanges(batch: BatchMetadata) {
-    // TODO Introduce Proven and Unproven BlockHashTree stores - for rollbacks
-    await this.database.executeInTransaction(async () => {
-      await batch.stateService.mergeIntoParent();
-      await batch.merkleStore.mergeIntoParent();
-    });
-  }
-
   /**
    * Main function to call when wanting to create a new block based on the
    * transactions that are present in the mempool. This function should also
-   * be the one called by BlockTriggers
+   * be the one called by BlockTriggerss
    */
   public async createBatch(
     blocks: BlockWithResult[]
   ): Promise<SettleableBatch | undefined> {
-    log.info("Producing batch...");
-
-    const height = await this.batchStorage.getCurrentBatchHeight();
-
-    const batchWithStateDiff = await this.tryProduceBatch(blocks, height);
-
-    if (batchWithStateDiff !== undefined) {
-      const numTxs = blocks.reduce(
-        (sum, block) => sum + block.block.transactions.length,
-        0
-      );
-      log.info(
-        `Batch produced (${batchWithStateDiff.batch.blockHashes.length} blocks, ${numTxs} txs)`
-      );
-
-      // Apply state changes to current MerkleTreeStore
-      await batchWithStateDiff.changes.commit();
-
-      // TODO Add transition from unproven to proven state for stateservice
-      //  This needs proper DB-level masking
-    }
-    return batchWithStateDiff?.batch;
-  }
-
-  public async start(): Promise<void> {
-    noop();
-  }
-
-  private async tryProduceBatch(
-    blocks: BlockWithResult[],
-    height: number
-  ): Promise<BatchMetadata | undefined> {
     if (!this.productionInProgress) {
       try {
         this.productionInProgress = true;
 
-        const batch = await this.produceBatch(blocks, height);
+        const batch = await this.tryProduceBatch(blocks);
 
         this.productionInProgress = false;
 
@@ -144,6 +101,40 @@ export class BatchProducerModule extends SequencerModule {
     return undefined;
   }
 
+  private async tryProduceBatch(
+    blocks: BlockWithResult[]
+  ): Promise<SettleableBatch | undefined> {
+    log.info("Producing batch...");
+
+    const height = await this.batchStorage.getCurrentBatchHeight();
+
+    const batchWithStateDiff = await this.produceBatch(blocks, height);
+
+    if (batchWithStateDiff !== undefined) {
+      const numTxs = blocks.reduce(
+        (sum, block) => sum + block.block.transactions.length,
+        0
+      );
+      log.info(
+        `Batch produced (${batchWithStateDiff.batch.blockHashes.length} blocks, ${numTxs} txs)`
+      );
+
+      // Apply state changes to current MerkleTreeStore
+      await this.database.executeInTransaction(async () => {
+        await this.batchStorage.pushBatch(batchWithStateDiff.batch);
+        await batchWithStateDiff.changes.mergeIntoParent();
+      });
+
+      // TODO Add transition from unproven to proven state for stateservice
+      //  This needs proper DB-level masking
+    }
+    return batchWithStateDiff?.batch;
+  }
+
+  public async start(): Promise<void> {
+    noop();
+  }
+
   private async produceBatch(
     blocks: BlockWithResult[],
     height: number
@@ -187,9 +178,7 @@ export class BatchProducerModule extends SequencerModule {
     blockId: number
   ): Promise<{
     proof: Proof<BlockProverPublicInput, BlockProverPublicOutput>;
-    changes: {
-      commit: () => Promise<void>;
-    };
+    changes: CachedMerkleTreeStore;
     fromNetworkState: NetworkState;
     toNetworkState: NetworkState;
   }> {
@@ -209,13 +198,9 @@ export class BatchProducerModule extends SequencerModule {
     const fromNetworkState = blocks[0].block.networkState.before;
     const toNetworkState = blocks.at(-1)!.result.afterNetworkState;
 
-    const changes = {
-      commit: async () => await merkleTreeStore.mergeIntoParent(),
-    };
-
     return {
       proof,
-      changes,
+      changes: merkleTreeStore,
       fromNetworkState,
       toNetworkState,
     };

packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts

@@ -23,7 +23,6 @@ import {
   BlockResult,
   BlockWithResult,
 } from "../../../storage/model/Block";
-import { CachedStateService } from "../../../state/state/CachedStateService";
 import { MessageStorage } from "../../../storage/repositories/MessageStorage";
 import { Database } from "../../../storage/Database";
 
@@ -108,16 +107,18 @@ export class BlockProducerModule extends SequencerModule<BlockConfig> {
   }
 
   public async generateMetadata(block: Block): Promise<BlockResult> {
-    const { result, blockHashTreeStore, treeStore } =
+    const { result, blockHashTreeStore, treeStore, stateService } =
       await this.resultService.generateMetadataForNextBlock(
         block,
         this.unprovenMerkleStore,
-        this.blockTreeStore
+        this.blockTreeStore,
+        this.unprovenStateService
       );
 
     await this.database.executeInTransaction(async () => {
       await blockHashTreeStore.mergeIntoParent();
       await treeStore.mergeIntoParent();
+      await stateService.mergeIntoParent();
 
       await this.blockQueue.pushResult(result);
     });
@@ -210,27 +211,25 @@ export class BlockProducerModule extends SequencerModule<BlockConfig> {
       return undefined;
     }
 
-    const cachedStateService = new CachedStateService(
-      this.unprovenStateService
-    );
-
-    const block = await this.productionService.createBlock(
-      cachedStateService,
+    const blockResult = await this.productionService.createBlock(
+      this.unprovenStateService,
       txs,
       metadata,
       this.allowEmptyBlock()
     );
 
-    if (block !== undefined) {
+    if (blockResult !== undefined) {
+      const { block, stateChanges } = blockResult;
+
       await this.database.executeInTransaction(async () => {
-        await cachedStateService.mergeIntoParent();
+        await stateChanges.mergeIntoParent();
         await this.blockQueue.pushBlock(block);
       });
     }
 
     this.productionInProgress = false;
 
-    return block;
+    return blockResult?.block;
   }
 
   public async blockResultCompleteCheck() {

packages/sequencer/src/protocol/production/sequencing/BlockProductionService.ts

@@ -51,9 +51,8 @@ export class BlockProductionService {
   public async executeBeforeBlockHook(
     args: BeforeBlockHookArguments,
     inputNetworkState: NetworkState,
-    asyncStateService: AsyncStateService
+    cachedStateService: CachedStateService
   ) {
-    const cachedStateService = new CachedStateService(asyncStateService);
     this.stateServiceProvider.setCurrentStateService(cachedStateService);
 
     // Execute afterBlock hooks
@@ -73,7 +72,9 @@ export class BlockProductionService {
     );
 
     this.stateServiceProvider.popCurrentStateService();
-    await cachedStateService.mergeIntoParent();
+    await cachedStateService.applyStateTransitions(
+      executionResult.stateTransitions
+    );
 
     return executionResult;
   }
@@ -83,11 +84,19 @@ export class BlockProductionService {
    * attached that is needed for tracing
    */
   public async createBlock(
-    stateService: CachedStateService,
+    asyncStateService: AsyncStateService,
     transactions: PendingTransaction[],
     lastBlockWithResult: BlockWithResult,
     allowEmptyBlocks: boolean
-  ): Promise<Block | undefined> {
+  ): Promise<
+    | {
+        block: Block;
+        stateChanges: CachedStateService;
+      }
+    | undefined
+  > {
+    const stateService = new CachedStateService(asyncStateService);
+
     const lastResult = lastBlockWithResult.result;
     const lastBlock = lastBlockWithResult.block;
     const executionResults: TransactionExecutionResult[] = [];
@@ -147,7 +156,7 @@ export class BlockProductionService {
 
     if (executionResults.length === 0 && !allowEmptyBlocks) {
       log.info(
-        "After sequencing, block has no sequencable transactions left, skipping block"
+        "After sequencing, block has no sequenceable transactions left, skipping block"
       );
       return undefined;
     }
@@ -175,8 +184,11 @@ export class BlockProductionService {
     const hash = Block.hash(block);
 
     return {
-      ...block,
-      hash,
+      block: {
+        ...block,
+        hash,
+      },
+      stateChanges: stateService,
     };
   }
 }

packages/sequencer/src/protocol/production/sequencing/BlockResultService.ts

@@ -103,9 +103,14 @@ export class BlockResultService {
     );
 
     this.stateServiceProvider.popCurrentStateService();
-    await cachedStateService.mergeIntoParent();
+    await cachedStateService.applyStateTransitions(
+      executionResult.stateTransitions
+    );
 
-    return executionResult;
+    return {
+      executionResult,
+      cachedStateService,
+    };
   }
 
   /** Update the block hash tree with this block */
@@ -134,11 +139,11 @@ export class BlockResultService {
     );
     const blockHashWitness = blockHashTree.getWitness(block.height.toBigInt());
     const newBlockHashRoot = blockHashTree.getRoot();
-    await blockHashInMemoryStore.mergeIntoParent();
 
     return {
       blockHashWitness,
       blockHashRoot: newBlockHashRoot,
+      cachedBlockHashTreeStore: blockHashInMemoryStore,
     };
   }
 
@@ -168,9 +173,13 @@ export class BlockResultService {
     block: Block,
     merkleTreeStore: AsyncMerkleTreeStore,
     blockHashTreeStore: AsyncMerkleTreeStore,
-    stateService: AsyncStateService,
-    modifyTreeStore = true
-  ): Promise<BlockResult> {
+    stateService: AsyncStateService
+  ): Promise<{
+    result: BlockResult;
+    treeStore: CachedMerkleTreeStore;
+    blockHashTreeStore: CachedMerkleTreeStore;
+    stateService: CachedStateService;
+  }> {
     const combinedDiff = createCombinedStateDiff(
       block.transactions,
       block.beforeBlockStateTransitions
@@ -182,10 +191,13 @@ export class BlockResultService {
 
     const witnessedStateRoot = tree.getRoot();
 
-    const { blockHashWitness, blockHashRoot } =
+    const { blockHashWitness, blockHashRoot, cachedBlockHashTreeStore } =
       await this.insertIntoBlockHashTree(block, blockHashTreeStore);
 
-    const { stateTransitions, methodResult } = await this.executeAfterBlockHook(
+    const {
+      executionResult: { stateTransitions, methodResult },
+      cachedStateService,
+    } = await this.executeAfterBlockHook(
       {
         blockHashRoot,
         stateRoot: witnessedStateRoot,
@@ -208,22 +220,24 @@ export class BlockResultService {
     );
 
     const stateRoot = tree2.getRoot();
-    if (modifyTreeStore) {
-      await inMemoryStore.mergeIntoParent();
-    }
 
     return {
-      afterNetworkState: methodResult,
-      // This is the state root after the last tx and before the afterBlock hook
-      stateRoot: stateRoot.toBigInt(),
-      witnessedRoots: [witnessedStateRoot.toBigInt()],
-      blockHashRoot: blockHashRoot.toBigInt(),
-      blockHashWitness,
-
-      afterBlockStateTransitions: stateTransitions.map((st) =>
-        UntypedStateTransition.fromStateTransition(st)
-      ),
-      blockHash: block.hash.toBigInt(),
+      result: {
+        afterNetworkState: methodResult,
+        // This is the state root after the last tx and the afterBlock hook
+        stateRoot: stateRoot.toBigInt(),
+        witnessedRoots: [witnessedStateRoot.toBigInt()],
+        blockHashRoot: blockHashRoot.toBigInt(),
+        blockHashWitness,
+
+        afterBlockStateTransitions: stateTransitions.map((st) =>
+          UntypedStateTransition.fromStateTransition(st)
+        ),
+        blockHash: block.hash.toBigInt(),
+      },
+      treeStore: inMemoryStore,
+      blockHashTreeStore: cachedBlockHashTreeStore,
+      stateService: cachedStateService,
     };
   }
 }