Added recovery mechanism for missing block result

Author: rpanic

packages/persistance/src/services/prisma/PrismaBlockStorage.ts

@@ -8,6 +8,7 @@ import {
   BlockStorage,
   BlockWithResult,
   BlockWithPreviousResult,
+  BlockWithMaybeResult,
 } from "@proto-kit/sequencer";
 import { filterNonNull, log } from "@proto-kit/common";
 import {
@@ -39,7 +40,7 @@ export class PrismaBlockStorage
 
   private async getBlockByQuery(
     where: { height: number } | { hash: string }
-  ): Promise<BlockWithResult | undefined> {
+  ): Promise<BlockWithMaybeResult | undefined> {
     const dbResult = await this.connection.prismaClient.block.findFirst({
       where,
       include: {
@@ -57,18 +58,15 @@ export class PrismaBlockStorage
     const transactions = dbResult.transactions.map<TransactionExecutionResult>(
       (txresult) => this.transactionResultMapper.mapIn([txresult, txresult.tx])
     );
-    if (dbResult.result === undefined || dbResult.result === null) {
-      throw new Error(
-        `No Metadata has been set for block ${JSON.stringify(where)} yet`
-      );
-    }
 
     return {
       block: {
         ...this.blockMapper.mapIn(dbResult),
         transactions,
       },
-      result: this.blockResultMapper.mapIn(dbResult.result),
+      result: dbResult.result
+        ? this.blockResultMapper.mapIn(dbResult.result)
+        : undefined,
     };
   }
 
@@ -169,7 +167,9 @@ export class PrismaBlockStorage
     return (result?._max.height ?? -1) + 1;
   }
 
-  public async getLatestBlock(): Promise<BlockWithResult | undefined> {
+  public async getLatestBlockAndResult(): Promise<
+    BlockWithMaybeResult | undefined
+  > {
     const latestBlock = await this.connection.prismaClient.$queryRaw<
       { hash: string }[]
     >`SELECT b1."hash" FROM "Block" b1 
@@ -185,6 +185,22 @@ export class PrismaBlockStorage
     });
   }
 
+  public async getLatestBlock(): Promise<BlockWithResult | undefined> {
+    const result = await this.getLatestBlockAndResult();
+    if (result !== undefined) {
+      if (result.result === undefined) {
+        throw new Error(
+          `Block result for block ${result.block.height.toString()} not found`
+        );
+      }
+      return {
+        block: result.block,
+        result: result.result,
+      };
+    }
+    return result;
+  }
+
   public async getNewBlocks(): Promise<BlockWithPreviousResult[]> {
     const blocks = await this.connection.prismaClient.block.findMany({
       where: {

packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts

@@ -158,19 +158,31 @@ export class BlockProducerModule extends SequencerModule<BlockConfig> {
   }> {
     const txs = await this.mempool.getTxs(this.maximumBlockSize());
 
-    const parentBlock = await this.blockQueue.getLatestBlock();
+    const parentBlock = await this.blockQueue.getLatestBlockAndResult();
+
+    let metadata: BlockWithResult;
 
     if (parentBlock === undefined) {
       log.debug(
         "No block metadata given, assuming first block, generating genesis metadata"
       );
+      metadata = BlockWithResult.createEmpty();
+    } else if (parentBlock.result === undefined) {
+      throw new Error(
+        `Metadata for block at height ${parentBlock.block.height.toString()} not available`
+      );
+    } else {
+      metadata = {
+        block: parentBlock.block,
+        // By reconstructing this object, typescript correctly infers the result to be defined
+        result: parentBlock.result,
+      };
     }
 
     const messages = await this.messageStorage.getMessages(
       parentBlock?.block.toMessagesHash.toString() ??
         ACTIONS_EMPTY_HASH.toString()
     );
-    const metadata = parentBlock ?? BlockWithResult.createEmpty();
 
     log.debug(
       `Block collected, ${txs.length} txs, ${messages.length} messages`
@@ -215,6 +227,17 @@ export class BlockProducerModule extends SequencerModule<BlockConfig> {
   }
 
   public async start() {
-    noop();
+    // Check if metadata height is behind block production.
+    // This can happen when the sequencer crashes after a block has been produced
+    // but before the metadata generation has finished
+    const latestBlock = await this.blockQueue.getLatestBlockAndResult();
+    // eslint-disable-next-line sonarjs/no-collapsible-if
+    if (latestBlock !== undefined) {
+      if (latestBlock.result === undefined) {
+        await this.generateMetadata(latestBlock.block);
+      }
+      // Here, the metadata has been computed already
+    }
+    // If we reach here, its a genesis startup, no blocks exist yet
   }
 }

packages/sequencer/src/storage/inmemory/InMemoryBlockStorage.ts

@@ -5,7 +5,12 @@ import {
   BlockQueue,
   BlockStorage,
 } from "../repositories/BlockStorage";
-import type { Block, BlockResult, BlockWithResult } from "../model/Block";
+import type {
+  Block,
+  BlockResult,
+  BlockWithMaybeResult,
+  BlockWithResult,
+} from "../model/Block";
 import { BlockWithPreviousResult } from "../../protocol/production/BatchProducerModule";
 import { BatchStorage } from "../repositories/BatchStorage";
 
@@ -29,10 +34,12 @@ export class InMemoryBlockStorage
     return this.blocks.length;
   }
 
-  public async getLatestBlock(): Promise<BlockWithResult | undefined> {
+  public async getLatestBlockAndResult(): Promise<
+    BlockWithMaybeResult | undefined
+  > {
     const currentHeight = await this.getCurrentBlockHeight();
     const block = await this.getBlockAt(currentHeight - 1);
-    const result = this.results[currentHeight - 1];
+    const result: BlockResult | undefined = this.results[currentHeight - 1];
     if (block === undefined) {
       return undefined;
     }
@@ -42,6 +49,22 @@ export class InMemoryBlockStorage
     };
   }
 
+  public async getLatestBlock(): Promise<BlockWithResult | undefined> {
+    const result = await this.getLatestBlockAndResult();
+    if (result !== undefined) {
+      if (result.result === undefined) {
+        throw new Error(
+          `Block result for block ${result.block.height.toString()} not found`
+        );
+      }
+      return {
+        block: result.block,
+        result: result.result,
+      };
+    }
+    return result;
+  }
+
   public async getNewBlocks(): Promise<BlockWithPreviousResult[]> {
     const latestBatch = await this.batchStorage.getLatestBatch();
 

packages/sequencer/src/storage/model/Block.ts

@@ -61,6 +61,11 @@ export interface BlockWithResult {
   result: BlockResult;
 }
 
+export interface BlockWithMaybeResult {
+  block: Block;
+  result?: BlockResult;
+}
+
 // eslint-disable-next-line @typescript-eslint/no-redeclare
 export const BlockWithResult = {
   createEmpty: () =>

packages/sequencer/src/storage/repositories/BlockStorage.ts

@@ -1,11 +1,16 @@
 import { BlockWithPreviousResult } from "../../protocol/production/BatchProducerModule";
-import type { Block, BlockResult, BlockWithResult } from "../model/Block";
+import {
+  Block,
+  BlockResult,
+  BlockWithMaybeResult,
+  BlockWithResult,
+} from "../model/Block";
 
 export interface BlockQueue {
   pushBlock: (block: Block) => Promise<void>;
   pushResult: (result: BlockResult) => Promise<void>;
   getNewBlocks: () => Promise<BlockWithPreviousResult[]>;
-  getLatestBlock: () => Promise<BlockWithResult | undefined>;
+  getLatestBlockAndResult: () => Promise<BlockWithMaybeResult | undefined>;
 }
 
 export interface BlockStorage {