From d2f725171d3c2cc3b8e0aaa0ae8dc159e86c4ae0 Mon Sep 17 00:00:00 2001 From: Raphael Panic Date: Fri, 22 Nov 2024 16:10:48 +0100 Subject: [PATCH 1/5] Pulled apart block production and metadata generation --- .../sequencing/BlockProducerModule.ts | 45 ++++++++++++------- .../sequencing/TransactionExecutionService.ts | 36 ++++++++------- .../production/trigger/BlockTrigger.ts | 33 +++++++------- .../production/trigger/ManualBlockTrigger.ts | 12 ++--- .../production/trigger/TimedBlockTrigger.ts | 2 +- 5 files changed, 69 insertions(+), 59 deletions(-) diff --git a/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts b/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts index 106833f48..ec71fb1fa 100644 --- a/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts +++ b/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts @@ -18,7 +18,11 @@ import { BlockQueue } from "../../../storage/repositories/BlockStorage"; import { PendingTransaction } from "../../../mempool/PendingTransaction"; import { AsyncMerkleTreeStore } from "../../../state/async/AsyncMerkleTreeStore"; import { AsyncStateService } from "../../../state/async/AsyncStateService"; -import { Block, BlockWithResult } from "../../../storage/model/Block"; +import { + Block, + BlockResult, + BlockWithResult, +} from "../../../storage/model/Block"; import { CachedStateService } from "../../../state/state/CachedStateService"; import { MessageStorage } from "../../../storage/repositories/MessageStorage"; @@ -99,7 +103,23 @@ export class BlockProducerModule extends SequencerModule { } } - public async tryProduceBlock(): Promise { + public async generateMetadata(block: Block): Promise { + const { result, blockHashTreeStore, treeStore } = + await this.executionService.generateMetadataForNextBlock( + block, + this.unprovenMerkleStore, + this.blockTreeStore + ); + + await blockHashTreeStore.mergeIntoParent(); + await treeStore.mergeIntoParent(); + + await this.blockQueue.pushResult(result); + + return result; + } + + public async tryProduceBlock(): Promise { if (!this.productionInProgress) { try { const block = await this.produceBlock(); @@ -118,20 +138,7 @@ export class BlockProducerModule extends SequencerModule { ); this.prettyPrintBlockContents(block); - // Generate metadata for next block - - // TODO: make async of production in the future - const result = await this.executionService.generateMetadataForNextBlock( - block, - this.unprovenMerkleStore, - this.blockTreeStore, - true - ); - - return { - block, - result, - }; + return block; } catch (error: unknown) { if (error instanceof Error) { throw error; @@ -196,7 +203,11 @@ export class BlockProducerModule extends SequencerModule { this.allowEmptyBlock() ); - await cachedStateService.mergeIntoParent(); + if (block !== undefined) { + await cachedStateService.mergeIntoParent(); + + await this.blockQueue.pushBlock(block); + } this.productionInProgress = false; diff --git a/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts b/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts index dae2a7b7a..6b107f28c 100644 --- a/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts +++ b/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts @@ -323,9 +323,12 @@ export class TransactionExecutionService { public async generateMetadataForNextBlock( block: Block, merkleTreeStore: AsyncMerkleTreeStore, - blockHashTreeStore: AsyncMerkleTreeStore, - modifyTreeStore = true - ): Promise { + blockHashTreeStore: AsyncMerkleTreeStore + ): Promise<{ + result: BlockResult; + treeStore: CachedMerkleTreeStore; + blockHashTreeStore: CachedMerkleTreeStore; + }> { // Flatten diff list into a single diff by applying them over each other const combinedDiff = block.transactions .map((tx) => { @@ -403,22 +406,21 @@ export class TransactionExecutionService { ); const blockHashWitness = blockHashTree.getWitness(block.height.toBigInt()); const newBlockHashRoot = blockHashTree.getRoot(); - await blockHashInMemoryStore.mergeIntoParent(); - - if (modifyTreeStore) { - await inMemoryStore.mergeIntoParent(); - } return { - afterNetworkState: resultingNetworkState, - stateRoot: stateRoot.toBigInt(), - blockHashRoot: newBlockHashRoot.toBigInt(), - blockHashWitness, - - blockStateTransitions: reducedStateTransitions.map((st) => - UntypedStateTransition.fromStateTransition(st) - ), - blockHash: block.hash.toBigInt(), + result: { + afterNetworkState: resultingNetworkState, + stateRoot: stateRoot.toBigInt(), + blockHashRoot: newBlockHashRoot.toBigInt(), + blockHashWitness, + + blockStateTransitions: reducedStateTransitions.map((st) => + UntypedStateTransition.fromStateTransition(st) + ), + blockHash: block.hash.toBigInt(), + }, + treeStore: inMemoryStore, + blockHashTreeStore: blockHashInMemoryStore, }; } diff --git a/packages/sequencer/src/protocol/production/trigger/BlockTrigger.ts b/packages/sequencer/src/protocol/production/trigger/BlockTrigger.ts index 4a1862cb4..97252e565 100644 --- a/packages/sequencer/src/protocol/production/trigger/BlockTrigger.ts +++ b/packages/sequencer/src/protocol/production/trigger/BlockTrigger.ts @@ -65,28 +65,29 @@ export class BlockTriggerBase< return undefined; } - protected async produceBlockWithResult( - enqueueInSettlementQueue: boolean - ): Promise { + protected async produceBlockWithResult(): Promise< + BlockWithResult | undefined + > { const block = await this.blockProducerModule.tryProduceBlock(); + if (block) { + this.events.emit("block-produced", block); - if (block && enqueueInSettlementQueue) { - await this.blockQueue.pushBlock(block.block); - this.events.emit("block-produced", block.block); + const result = await this.blockProducerModule.generateMetadata(block); - await this.blockQueue.pushResult(block.result); - this.events.emit("block-metadata-produced", block); - } + const blockWithMetadata = { + block, + result, + }; + + this.events.emit("block-metadata-produced", blockWithMetadata); - return block; + return blockWithMetadata; + } + return undefined; } - protected async produceBlock( - enqueueInSettlementQueue: boolean - ): Promise { - const blockWithResult = await this.produceBlockWithResult( - enqueueInSettlementQueue - ); + protected async produceBlock(): Promise { + const blockWithResult = await this.produceBlockWithResult(); return blockWithResult?.block; } diff --git a/packages/sequencer/src/protocol/production/trigger/ManualBlockTrigger.ts b/packages/sequencer/src/protocol/production/trigger/ManualBlockTrigger.ts index 2a8b755ff..386991250 100644 --- a/packages/sequencer/src/protocol/production/trigger/ManualBlockTrigger.ts +++ b/packages/sequencer/src/protocol/production/trigger/ManualBlockTrigger.ts @@ -62,15 +62,11 @@ export class ManualBlockTrigger return await super.settle(batch); } - public async produceBlock( - enqueueInSettlementQueue: boolean = true - ): Promise { - return await super.produceBlock(enqueueInSettlementQueue); + public async produceBlock(): Promise { + return await super.produceBlock(); } - public async produceBlockWithResult( - enqueueInSettlementQueue: boolean = true - ): Promise { - return await super.produceBlockWithResult(enqueueInSettlementQueue); + public async produceBlockWithResult(): Promise { + return await super.produceBlockWithResult(); } } diff --git a/packages/sequencer/src/protocol/production/trigger/TimedBlockTrigger.ts b/packages/sequencer/src/protocol/production/trigger/TimedBlockTrigger.ts index 5c892329b..c1b8e132c 100644 --- a/packages/sequencer/src/protocol/production/trigger/TimedBlockTrigger.ts +++ b/packages/sequencer/src/protocol/production/trigger/TimedBlockTrigger.ts @@ -129,7 +129,7 @@ export class TimedBlockTrigger // Produce a block if either produceEmptyBlocks is true or we have more // than 1 tx in mempool if (mempoolTxs.length > 0 || (this.config.produceEmptyBlocks ?? true)) { - await this.produceBlock(true); + await this.produceBlock(); } } From 42d26fd27b8c5aff74020b759be8998046f459e6 Mon Sep 17 00:00:00 2001 From: Raphael Panic Date: Fri, 22 Nov 2024 17:25:31 +0100 Subject: [PATCH 2/5] Added recovery mechanism for missing block result --- .../src/services/prisma/PrismaBlockStorage.ts | 32 ++++++++++++++----- .../sequencing/BlockProducerModule.ts | 29 +++++++++++++++-- .../storage/inmemory/InMemoryBlockStorage.ts | 29 +++++++++++++++-- packages/sequencer/src/storage/model/Block.ts | 5 +++ .../src/storage/repositories/BlockStorage.ts | 9 ++++-- 5 files changed, 88 insertions(+), 16 deletions(-) diff --git a/packages/persistance/src/services/prisma/PrismaBlockStorage.ts b/packages/persistance/src/services/prisma/PrismaBlockStorage.ts index 53be86548..52a06346e 100644 --- a/packages/persistance/src/services/prisma/PrismaBlockStorage.ts +++ b/packages/persistance/src/services/prisma/PrismaBlockStorage.ts @@ -8,6 +8,7 @@ import { BlockStorage, BlockWithResult, BlockWithPreviousResult, + BlockWithMaybeResult, } from "@proto-kit/sequencer"; import { filterNonNull, log } from "@proto-kit/common"; import { @@ -39,7 +40,7 @@ export class PrismaBlockStorage private async getBlockByQuery( where: { height: number } | { hash: string } - ): Promise { + ): Promise { const dbResult = await this.connection.prismaClient.block.findFirst({ where, include: { @@ -57,18 +58,15 @@ export class PrismaBlockStorage const transactions = dbResult.transactions.map( (txresult) => this.transactionResultMapper.mapIn([txresult, txresult.tx]) ); - if (dbResult.result === undefined || dbResult.result === null) { - throw new Error( - `No Metadata has been set for block ${JSON.stringify(where)} yet` - ); - } return { block: { ...this.blockMapper.mapIn(dbResult), transactions, }, - result: this.blockResultMapper.mapIn(dbResult.result), + result: dbResult.result + ? this.blockResultMapper.mapIn(dbResult.result) + : undefined, }; } @@ -169,7 +167,9 @@ export class PrismaBlockStorage return (result?._max.height ?? -1) + 1; } - public async getLatestBlock(): Promise { + public async getLatestBlockAndResult(): Promise< + BlockWithMaybeResult | undefined + > { const latestBlock = await this.connection.prismaClient.$queryRaw< { hash: string }[] >`SELECT b1."hash" FROM "Block" b1 @@ -185,6 +185,22 @@ export class PrismaBlockStorage }); } + public async getLatestBlock(): Promise { + const result = await this.getLatestBlockAndResult(); + if (result !== undefined) { + if (result.result === undefined) { + throw new Error( + `Block result for block ${result.block.height.toString()} not found` + ); + } + return { + block: result.block, + result: result.result, + }; + } + return result; + } + public async getNewBlocks(): Promise { const blocks = await this.connection.prismaClient.block.findMany({ where: { diff --git a/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts b/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts index ec71fb1fa..e98447d42 100644 --- a/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts +++ b/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts @@ -158,19 +158,31 @@ export class BlockProducerModule extends SequencerModule { }> { const txs = await this.mempool.getTxs(this.maximumBlockSize()); - const parentBlock = await this.blockQueue.getLatestBlock(); + const parentBlock = await this.blockQueue.getLatestBlockAndResult(); + + let metadata: BlockWithResult; if (parentBlock === undefined) { log.debug( "No block metadata given, assuming first block, generating genesis metadata" ); + metadata = BlockWithResult.createEmpty(); + } else if (parentBlock.result === undefined) { + throw new Error( + `Metadata for block at height ${parentBlock.block.height.toString()} not available` + ); + } else { + metadata = { + block: parentBlock.block, + // By reconstructing this object, typescript correctly infers the result to be defined + result: parentBlock.result, + }; } const messages = await this.messageStorage.getMessages( parentBlock?.block.toMessagesHash.toString() ?? ACTIONS_EMPTY_HASH.toString() ); - const metadata = parentBlock ?? BlockWithResult.createEmpty(); log.debug( `Block collected, ${txs.length} txs, ${messages.length} messages` @@ -215,6 +227,17 @@ export class BlockProducerModule extends SequencerModule { } public async start() { - noop(); + // Check if metadata height is behind block production. + // This can happen when the sequencer crashes after a block has been produced + // but before the metadata generation has finished + const latestBlock = await this.blockQueue.getLatestBlockAndResult(); + // eslint-disable-next-line sonarjs/no-collapsible-if + if (latestBlock !== undefined) { + if (latestBlock.result === undefined) { + await this.generateMetadata(latestBlock.block); + } + // Here, the metadata has been computed already + } + // If we reach here, its a genesis startup, no blocks exist yet } } diff --git a/packages/sequencer/src/storage/inmemory/InMemoryBlockStorage.ts b/packages/sequencer/src/storage/inmemory/InMemoryBlockStorage.ts index a12f846ee..cf332bb44 100644 --- a/packages/sequencer/src/storage/inmemory/InMemoryBlockStorage.ts +++ b/packages/sequencer/src/storage/inmemory/InMemoryBlockStorage.ts @@ -5,7 +5,12 @@ import { BlockQueue, BlockStorage, } from "../repositories/BlockStorage"; -import type { Block, BlockResult, BlockWithResult } from "../model/Block"; +import type { + Block, + BlockResult, + BlockWithMaybeResult, + BlockWithResult, +} from "../model/Block"; import { BlockWithPreviousResult } from "../../protocol/production/BatchProducerModule"; import { BatchStorage } from "../repositories/BatchStorage"; @@ -29,10 +34,12 @@ export class InMemoryBlockStorage return this.blocks.length; } - public async getLatestBlock(): Promise { + public async getLatestBlockAndResult(): Promise< + BlockWithMaybeResult | undefined + > { const currentHeight = await this.getCurrentBlockHeight(); const block = await this.getBlockAt(currentHeight - 1); - const result = this.results[currentHeight - 1]; + const result: BlockResult | undefined = this.results[currentHeight - 1]; if (block === undefined) { return undefined; } @@ -42,6 +49,22 @@ export class InMemoryBlockStorage }; } + public async getLatestBlock(): Promise { + const result = await this.getLatestBlockAndResult(); + if (result !== undefined) { + if (result.result === undefined) { + throw new Error( + `Block result for block ${result.block.height.toString()} not found` + ); + } + return { + block: result.block, + result: result.result, + }; + } + return result; + } + public async getNewBlocks(): Promise { const latestBatch = await this.batchStorage.getLatestBatch(); diff --git a/packages/sequencer/src/storage/model/Block.ts b/packages/sequencer/src/storage/model/Block.ts index 5aa87c4ac..ef7c09943 100644 --- a/packages/sequencer/src/storage/model/Block.ts +++ b/packages/sequencer/src/storage/model/Block.ts @@ -61,6 +61,11 @@ export interface BlockWithResult { result: BlockResult; } +export interface BlockWithMaybeResult { + block: Block; + result?: BlockResult; +} + // eslint-disable-next-line @typescript-eslint/no-redeclare export const BlockWithResult = { createEmpty: () => diff --git a/packages/sequencer/src/storage/repositories/BlockStorage.ts b/packages/sequencer/src/storage/repositories/BlockStorage.ts index 741332ea4..2347162ff 100644 --- a/packages/sequencer/src/storage/repositories/BlockStorage.ts +++ b/packages/sequencer/src/storage/repositories/BlockStorage.ts @@ -1,11 +1,16 @@ import { BlockWithPreviousResult } from "../../protocol/production/BatchProducerModule"; -import type { Block, BlockResult, BlockWithResult } from "../model/Block"; +import { + Block, + BlockResult, + BlockWithMaybeResult, + BlockWithResult, +} from "../model/Block"; export interface BlockQueue { pushBlock: (block: Block) => Promise; pushResult: (result: BlockResult) => Promise; getNewBlocks: () => Promise; - getLatestBlock: () => Promise; + getLatestBlockAndResult: () => Promise; } export interface BlockStorage { From cd208429ee2506b696c8e67368b2322836a7dff8 Mon Sep 17 00:00:00 2001 From: Raphael Panic Date: Fri, 22 Nov 2024 18:08:10 +0100 Subject: [PATCH 3/5] Fix build --- packages/sdk/src/query/BlockStorageNetworkStateModule.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sdk/src/query/BlockStorageNetworkStateModule.ts b/packages/sdk/src/query/BlockStorageNetworkStateModule.ts index f0ecaf3f2..908b63db6 100644 --- a/packages/sdk/src/query/BlockStorageNetworkStateModule.ts +++ b/packages/sdk/src/query/BlockStorageNetworkStateModule.ts @@ -51,7 +51,7 @@ export class BlockStorageNetworkStateModule * with afterBundle() hooks executed */ public async getStagedNetworkState(): Promise { - const result = await this.unprovenQueue.getLatestBlock(); + const result = await this.unprovenStorage.getLatestBlock(); return result?.result.afterNetworkState; } From fc031332eec7212fdf8b36d7771594ce7c176d0c Mon Sep 17 00:00:00 2001 From: Raphael Panic Date: Fri, 22 Nov 2024 18:37:24 +0100 Subject: [PATCH 4/5] Fixed linting --- .../sequencing/BlockProducerModule.ts | 2 +- .../test/integration/BlockProduction.test.ts | 10 ++++++---- .../sequencer/test/settlement/Settlement.ts | 17 ++++++++++++----- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts b/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts index e98447d42..e245b239d 100644 --- a/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts +++ b/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts @@ -1,5 +1,5 @@ import { inject } from "tsyringe"; -import { log, noop } from "@proto-kit/common"; +import { log } from "@proto-kit/common"; import { ACTIONS_EMPTY_HASH } from "@proto-kit/protocol"; import { MethodIdResolver, diff --git a/packages/sequencer/test/integration/BlockProduction.test.ts b/packages/sequencer/test/integration/BlockProduction.test.ts index f411bdbf1..e94048d0b 100644 --- a/packages/sequencer/test/integration/BlockProduction.test.ts +++ b/packages/sequencer/test/integration/BlockProduction.test.ts @@ -1,4 +1,4 @@ -import { log, range, MOCK_PROOF } from "@proto-kit/common"; +import { log, range, MOCK_PROOF, expectDefined } from "@proto-kit/common"; import { VanillaProtocolModules } from "@proto-kit/library"; import { Runtime, @@ -168,7 +168,7 @@ describe("block production", () => { }); it("should produce a dummy block proof", async () => { - expect.assertions(25); + expect.assertions(27); log.setLevel("TRACE"); @@ -199,7 +199,7 @@ describe("block production", () => { const latestBlockWithResult = await sequencer .resolve("BlockQueue") - .getLatestBlock(); + .getLatestBlockAndResult(); let batch = await blockTrigger.produceBatch(); @@ -208,8 +208,10 @@ describe("block production", () => { expect(batch!.blockHashes).toHaveLength(1); expect(batch!.proof.proof).toBe(MOCK_PROOF); + expectDefined(latestBlockWithResult); + expectDefined(latestBlockWithResult.result); expect( - latestBlockWithResult!.result.afterNetworkState.hash().toString() + latestBlockWithResult.result.afterNetworkState.hash().toString() ).toStrictEqual(batch!.toNetworkState.hash().toString()); // Check if the batchstorage has received the block diff --git a/packages/sequencer/test/settlement/Settlement.ts b/packages/sequencer/test/settlement/Settlement.ts index 7735552fb..75b9e2443 100644 --- a/packages/sequencer/test/settlement/Settlement.ts +++ b/packages/sequencer/test/settlement/Settlement.ts @@ -1,5 +1,10 @@ /* eslint-disable no-inner-declarations */ -import { log, mapSequential, RollupMerkleTree } from "@proto-kit/common"; +import { + expectDefined, + log, + mapSequential, + RollupMerkleTree, +} from "@proto-kit/common"; import { VanillaProtocolModules } from "@proto-kit/library"; import { Runtime } from "@proto-kit/module"; import { @@ -277,7 +282,7 @@ export const settlementTestFn = ( RollupMerkleTree.EMPTY_ROOT ); - const lastBlock = await blockQueue.getLatestBlock(); + const lastBlock = await blockQueue.getLatestBlockAndResult(); await trigger.settle(batch!); nonceCounter++; @@ -287,14 +292,16 @@ export const settlementTestFn = ( console.log("Block settled"); const { settlement } = settlementModule.getContracts(); + expectDefined(lastBlock); + expectDefined(lastBlock.result); expect(settlement.networkStateHash.get().toBigInt()).toStrictEqual( - lastBlock!.result.afterNetworkState.hash().toBigInt() + lastBlock.result.afterNetworkState.hash().toBigInt() ); expect(settlement.stateRoot.get().toBigInt()).toStrictEqual( - lastBlock!.result.stateRoot + lastBlock.result.stateRoot ); expect(settlement.blockHashRoot.get().toBigInt()).toStrictEqual( - lastBlock!.result.blockHashRoot + lastBlock.result.blockHashRoot ); }, timeout From 13c3b6940fa197c927355a553f1d82c34e70e7ef Mon Sep 17 00:00:00 2001 From: Raphael Panic Date: Sun, 24 Nov 2024 16:55:49 +0100 Subject: [PATCH 5/5] Fix test --- packages/sequencer/test/integration/StorageIntegration.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sequencer/test/integration/StorageIntegration.test.ts b/packages/sequencer/test/integration/StorageIntegration.test.ts index 687beb163..a3aba46ad 100644 --- a/packages/sequencer/test/integration/StorageIntegration.test.ts +++ b/packages/sequencer/test/integration/StorageIntegration.test.ts @@ -141,7 +141,7 @@ describe.each([["InMemory", InMemoryDatabase]])( const generatedBlock = await sequencer .resolve("BlockTrigger") - .produceBlock(true); + .produceBlock(); expectDefined(generatedBlock);