From a40d09ef242863ea937d329bb116881a323cc25b Mon Sep 17 00:00:00 2001 From: Raphael Panic Date: Tue, 7 Jan 2025 15:17:47 +0100 Subject: [PATCH 1/5] Pulled apart TransactionExecutionService into multiple services --- .../sequencing/BlockProducerModule.ts | 9 +- .../sequencing/BlockProductionService.ts | 141 ++++++++++ .../sequencing/BlockResultService.ts | 153 ++++++++++ .../sequencing/TransactionExecutionService.ts | 265 +----------------- 4 files changed, 305 insertions(+), 263 deletions(-) create mode 100644 packages/sequencer/src/protocol/production/sequencing/BlockProductionService.ts create mode 100644 packages/sequencer/src/protocol/production/sequencing/BlockResultService.ts diff --git a/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts b/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts index 106833f48..fd9698f81 100644 --- a/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts +++ b/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts @@ -23,6 +23,8 @@ import { CachedStateService } from "../../../state/state/CachedStateService"; import { MessageStorage } from "../../../storage/repositories/MessageStorage"; import { TransactionExecutionService } from "./TransactionExecutionService"; +import { BlockProductionService } from "./BlockProductionService"; +import { BlockResultService } from "./BlockResultService"; export interface BlockConfig { allowEmptyBlock?: boolean; @@ -44,7 +46,8 @@ export class BlockProducerModule extends SequencerModule { private readonly blockQueue: BlockQueue, @inject("BlockTreeStore") private readonly blockTreeStore: AsyncMerkleTreeStore, - private readonly executionService: TransactionExecutionService, + private readonly productionService: BlockProductionService, + private readonly resultService: BlockResultService, @inject("MethodIdResolver") private readonly methodIdResolver: MethodIdResolver, @inject("Runtime") private readonly runtime: Runtime @@ -121,7 +124,7 @@ export class BlockProducerModule extends SequencerModule { // Generate metadata for next block // TODO: make async of production in the future - const result = await this.executionService.generateMetadataForNextBlock( + const result = await this.resultService.generateMetadataForNextBlock( block, this.unprovenMerkleStore, this.blockTreeStore, @@ -189,7 +192,7 @@ export class BlockProducerModule extends SequencerModule { this.unprovenStateService ); - const block = await this.executionService.createBlock( + const block = await this.productionService.createBlock( cachedStateService, txs, metadata, diff --git a/packages/sequencer/src/protocol/production/sequencing/BlockProductionService.ts b/packages/sequencer/src/protocol/production/sequencing/BlockProductionService.ts new file mode 100644 index 000000000..c19234016 --- /dev/null +++ b/packages/sequencer/src/protocol/production/sequencing/BlockProductionService.ts @@ -0,0 +1,141 @@ +import { inject, injectable, Lifecycle, scoped } from "tsyringe"; +import { + DefaultProvableHashList, + MandatoryProtocolModulesRecord, + MinaActions, + MinaActionsHashList, + NetworkState, + Protocol, + ProtocolModulesRecord, + ProvableBlockHook, +} from "@proto-kit/protocol"; +import { Field } from "o1js"; +import { log } from "@proto-kit/common"; + +import { + Block, + BlockWithResult, + TransactionExecutionResult, +} from "../../../storage/model/Block"; +import { CachedStateService } from "../../../state/state/CachedStateService"; +import { PendingTransaction } from "../../../mempool/PendingTransaction"; +import { TransactionExecutionService } from "./TransactionExecutionService"; + +@injectable() +@scoped(Lifecycle.ContainerScoped) +export class BlockProductionService { + private readonly blockHooks: ProvableBlockHook[]; + + public constructor( + @inject("Protocol") + protocol: Protocol, + private readonly transactionExecutionService: TransactionExecutionService + ) { + this.blockHooks = + protocol.dependencyContainer.resolveAll("ProvableBlockHook"); + } + + /** + * Main entry point for creating a unproven block with everything + * attached that is needed for tracing + */ + public async createBlock( + stateService: CachedStateService, + transactions: PendingTransaction[], + lastBlockWithResult: BlockWithResult, + allowEmptyBlocks: boolean + ): Promise { + const lastResult = lastBlockWithResult.result; + const lastBlock = lastBlockWithResult.block; + const executionResults: TransactionExecutionResult[] = []; + + const transactionsHashList = new DefaultProvableHashList(Field); + const eternalTransactionsHashList = new DefaultProvableHashList( + Field, + Field(lastBlock.toEternalTransactionsHash) + ); + + const incomingMessagesList = new MinaActionsHashList( + Field(lastBlock.toMessagesHash) + ); + + // Get used networkState by executing beforeBlock() hooks + const networkState = await this.blockHooks.reduce>( + async (reduceNetworkState, hook) => + await hook.beforeBlock(await reduceNetworkState, { + blockHashRoot: Field(lastResult.blockHashRoot), + eternalTransactionsHash: lastBlock.toEternalTransactionsHash, + stateRoot: Field(lastResult.stateRoot), + transactionsHash: Field(0), + networkStateHash: lastResult.afterNetworkState.hash(), + incomingMessagesHash: lastBlock.toMessagesHash, + }), + Promise.resolve(lastResult.afterNetworkState) + ); + + for (const tx of transactions) { + try { + // Create execution trace + const executionTrace = + // eslint-disable-next-line no-await-in-loop + await this.transactionExecutionService.createExecutionTrace( + stateService, + tx, + networkState + ); + + // Push result to results and transaction onto bundle-hash + executionResults.push(executionTrace); + if (!tx.isMessage) { + transactionsHashList.push(tx.hash()); + eternalTransactionsHashList.push(tx.hash()); + } else { + const actionHash = MinaActions.actionHash( + tx.toRuntimeTransaction().hashData() + ); + + incomingMessagesList.push(actionHash); + } + } catch (error) { + if (error instanceof Error) { + log.error("Error in inclusion of tx, skipping", error); + } + } + } + + const previousBlockHash = + lastResult.blockHash === 0n ? undefined : Field(lastResult.blockHash); + + if (executionResults.length === 0 && !allowEmptyBlocks) { + log.info( + "After sequencing, block has no sequencable transactions left, skipping block" + ); + return undefined; + } + + const block: Omit = { + transactions: executionResults, + transactionsHash: transactionsHashList.commitment, + fromEternalTransactionsHash: lastBlock.toEternalTransactionsHash, + toEternalTransactionsHash: eternalTransactionsHashList.commitment, + height: + lastBlock.hash.toBigInt() !== 0n ? lastBlock.height.add(1) : Field(0), + fromBlockHashRoot: Field(lastResult.blockHashRoot), + fromMessagesHash: lastBlock.toMessagesHash, + toMessagesHash: incomingMessagesList.commitment, + previousBlockHash, + + networkState: { + before: new NetworkState(lastResult.afterNetworkState), + during: networkState, + }, + }; + + const hash = Block.hash(block); + + return { + ...block, + hash, + }; + } +} diff --git a/packages/sequencer/src/protocol/production/sequencing/BlockResultService.ts b/packages/sequencer/src/protocol/production/sequencing/BlockResultService.ts new file mode 100644 index 000000000..4bc25ccf0 --- /dev/null +++ b/packages/sequencer/src/protocol/production/sequencing/BlockResultService.ts @@ -0,0 +1,153 @@ +import type { StateRecord } from "../BatchProducerModule"; +import { Bool, Field, Poseidon } from "o1js"; +import { RollupMerkleTree } from "@proto-kit/common"; +import { + BlockHashMerkleTree, + BlockHashTreeEntry, + BlockProverState, + MandatoryProtocolModulesRecord, + NetworkState, + Protocol, + ProtocolModulesRecord, + ProvableBlockHook, + reduceStateTransitions, + RuntimeMethodExecutionContext, + RuntimeTransaction, +} from "@proto-kit/protocol"; +import { inject, injectable, Lifecycle, scoped } from "tsyringe"; + +import { Block, BlockResult } from "../../../storage/model/Block"; +import { AsyncMerkleTreeStore } from "../../../state/async/AsyncMerkleTreeStore"; +import { CachedMerkleTreeStore } from "../../../state/merkle/CachedMerkleTreeStore"; +import { UntypedStateTransition } from "../helpers/UntypedStateTransition"; + +function collectStateDiff( + stateTransitions: UntypedStateTransition[] +): StateRecord { + return stateTransitions.reduce>( + (state, st) => { + if (st.toValue.isSome.toBoolean()) { + state[st.path.toString()] = st.toValue.value; + } + return state; + }, + {} + ); +} + +@injectable() +@scoped(Lifecycle.ContainerScoped) +export class BlockResultService { + private readonly blockHooks: ProvableBlockHook[]; + + public constructor( + private readonly executionContext: RuntimeMethodExecutionContext, + @inject("Protocol") + protocol: Protocol + ) { + this.blockHooks = + protocol.dependencyContainer.resolveAll("ProvableBlockHook"); + } + + public async generateMetadataForNextBlock( + block: Block, + merkleTreeStore: AsyncMerkleTreeStore, + blockHashTreeStore: AsyncMerkleTreeStore, + modifyTreeStore = true + ): Promise { + // Flatten diff list into a single diff by applying them over each other + const combinedDiff = block.transactions + .map((tx) => { + const transitions = tx.protocolTransitions.concat( + tx.status.toBoolean() ? tx.stateTransitions : [] + ); + return collectStateDiff(transitions); + }) + .reduce((accumulator, diff) => { + // accumulator properties will be overwritten by diff's values + return Object.assign(accumulator, diff); + }, {}); + + const inMemoryStore = new CachedMerkleTreeStore(merkleTreeStore); + const tree = new RollupMerkleTree(inMemoryStore); + const blockHashInMemoryStore = new CachedMerkleTreeStore( + blockHashTreeStore + ); + const blockHashTree = new BlockHashMerkleTree(blockHashInMemoryStore); + + await inMemoryStore.preloadKeys(Object.keys(combinedDiff).map(BigInt)); + + // In case the diff is empty, we preload key 0 in order to + // retrieve the root, which we need later + if (Object.keys(combinedDiff).length === 0) { + await inMemoryStore.preloadKey(0n); + } + + // TODO This can be optimized a lot (we are only interested in the root at this step) + await blockHashInMemoryStore.preloadKey(block.height.toBigInt()); + + Object.entries(combinedDiff).forEach(([key, state]) => { + const treeValue = state !== undefined ? Poseidon.hash(state) : Field(0); + tree.setLeaf(BigInt(key), treeValue); + }); + + const stateRoot = tree.getRoot(); + const fromBlockHashRoot = blockHashTree.getRoot(); + + const state: BlockProverState = { + stateRoot, + transactionsHash: block.transactionsHash, + networkStateHash: block.networkState.during.hash(), + eternalTransactionsHash: block.toEternalTransactionsHash, + blockHashRoot: fromBlockHashRoot, + incomingMessagesHash: block.toMessagesHash, + }; + + // TODO Set StateProvider for @state access to state + this.executionContext.clear(); + this.executionContext.setup({ + networkState: block.networkState.during, + transaction: RuntimeTransaction.dummyTransaction(), + }); + + const resultingNetworkState = await this.blockHooks.reduce< + Promise + >( + async (networkState, hook) => + await hook.afterBlock(await networkState, state), + Promise.resolve(block.networkState.during) + ); + + const { stateTransitions } = this.executionContext.result; + this.executionContext.clear(); + const reducedStateTransitions = reduceStateTransitions(stateTransitions); + + // Update the block hash tree with this block + blockHashTree.setLeaf( + block.height.toBigInt(), + new BlockHashTreeEntry({ + blockHash: Poseidon.hash([block.height, state.transactionsHash]), + closed: Bool(true), + }).hash() + ); + const blockHashWitness = blockHashTree.getWitness(block.height.toBigInt()); + const newBlockHashRoot = blockHashTree.getRoot(); + await blockHashInMemoryStore.mergeIntoParent(); + + if (modifyTreeStore) { + await inMemoryStore.mergeIntoParent(); + } + + return { + afterNetworkState: resultingNetworkState, + stateRoot: stateRoot.toBigInt(), + blockHashRoot: newBlockHashRoot.toBigInt(), + blockHashWitness, + + blockStateTransitions: reducedStateTransitions.map((st) => + UntypedStateTransition.fromStateTransition(st) + ), + blockHash: block.hash.toBigInt(), + }; + } +} diff --git a/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts b/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts index 841a956b6..f581d5a99 100644 --- a/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts +++ b/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts @@ -1,8 +1,6 @@ import { container, inject, injectable, Lifecycle, scoped } from "tsyringe"; import { BlockProverExecutionData, - BlockProverState, - DefaultProvableHashList, NetworkState, Protocol, ProtocolModulesRecord, @@ -10,24 +8,12 @@ import { RuntimeMethodExecutionContext, RuntimeMethodExecutionData, RuntimeProvableMethodExecutionResult, - RuntimeTransaction, - StateTransition, - ProvableBlockHook, - BlockHashMerkleTree, StateServiceProvider, MandatoryProtocolModulesRecord, - BlockHashTreeEntry, - MinaActions, - MinaActionsHashList, reduceStateTransitions, } from "@proto-kit/protocol"; -import { Bool, Field, Poseidon } from "o1js"; -import { - AreProofsEnabled, - log, - RollupMerkleTree, - mapSequential, -} from "@proto-kit/common"; +import { Field } from "o1js"; +import { AreProofsEnabled, log, mapSequential } from "@proto-kit/common"; import { MethodParameterEncoder, Runtime, @@ -37,17 +23,8 @@ import { import { PendingTransaction } from "../../../mempool/PendingTransaction"; import { CachedStateService } from "../../../state/state/CachedStateService"; -import { distinctByString } from "../../../helpers/utils"; -import { CachedMerkleTreeStore } from "../../../state/merkle/CachedMerkleTreeStore"; -import { AsyncMerkleTreeStore } from "../../../state/async/AsyncMerkleTreeStore"; -import { - TransactionExecutionResult, - Block, - BlockResult, - BlockWithResult, -} from "../../../storage/model/Block"; +import { TransactionExecutionResult } from "../../../storage/model/Block"; import { UntypedStateTransition } from "../helpers/UntypedStateTransition"; -import type { StateRecord } from "../BatchProducerModule"; const errors = { methodIdNotFound: (methodId: string) => @@ -61,15 +38,10 @@ export type SomeRuntimeMethod = (...args: unknown[]) => Promise; export class TransactionExecutionService { private readonly transactionHooks: ProvableTransactionHook[]; - private readonly blockHooks: ProvableBlockHook[]; - public constructor( @inject("Runtime") private readonly runtime: Runtime, @inject("Protocol") - private readonly protocol: Protocol< - MandatoryProtocolModulesRecord & ProtocolModulesRecord - >, - private readonly executionContext: RuntimeMethodExecutionContext, + protocol: Protocol, // Coming in from the appchain scope (accessible by protocol & runtime) @inject("StateServiceProvider") private readonly stateServiceProvider: StateServiceProvider @@ -77,28 +49,6 @@ export class TransactionExecutionService { this.transactionHooks = protocol.dependencyContainer.resolveAll( "ProvableTransactionHook" ); - this.blockHooks = - protocol.dependencyContainer.resolveAll("ProvableBlockHook"); - } - - private allKeys(stateTransitions: StateTransition[]): Field[] { - // We have to do the distinct with strings because - // array.indexOf() doesn't work with fields - return stateTransitions.map((st) => st.path).filter(distinctByString); - } - - private collectStateDiff( - stateTransitions: UntypedStateTransition[] - ): StateRecord { - return stateTransitions.reduce>( - (state, st) => { - if (st.toValue.isSome.toBoolean()) { - state[st.path.toString()] = st.toValue.value; - } - return state; - }, - {} - ); } private async decodeTransaction(tx: PendingTransaction): Promise<{ @@ -217,212 +167,7 @@ export class TransactionExecutionService { ); } - /** - * Main entry point for creating a unproven block with everything - * attached that is needed for tracing - */ - public async createBlock( - stateService: CachedStateService, - transactions: PendingTransaction[], - lastBlockWithResult: BlockWithResult, - allowEmptyBlocks: boolean - ): Promise { - const lastResult = lastBlockWithResult.result; - const lastBlock = lastBlockWithResult.block; - const executionResults: TransactionExecutionResult[] = []; - - const transactionsHashList = new DefaultProvableHashList(Field); - const eternalTransactionsHashList = new DefaultProvableHashList( - Field, - Field(lastBlock.toEternalTransactionsHash) - ); - - const incomingMessagesList = new MinaActionsHashList( - Field(lastBlock.toMessagesHash) - ); - - // Get used networkState by executing beforeBlock() hooks - const networkState = await this.blockHooks.reduce>( - async (reduceNetworkState, hook) => - await hook.beforeBlock(await reduceNetworkState, { - blockHashRoot: Field(lastResult.blockHashRoot), - eternalTransactionsHash: lastBlock.toEternalTransactionsHash, - stateRoot: Field(lastResult.stateRoot), - transactionsHash: Field(0), - networkStateHash: lastResult.afterNetworkState.hash(), - incomingMessagesHash: lastBlock.toMessagesHash, - }), - Promise.resolve(lastResult.afterNetworkState) - ); - - for (const [, tx] of transactions.entries()) { - try { - // Create execution trace - // eslint-disable-next-line no-await-in-loop - const executionTrace = await this.createExecutionTrace( - stateService, - tx, - networkState - ); - - // Push result to results and transaction onto bundle-hash - executionResults.push(executionTrace); - if (!tx.isMessage) { - transactionsHashList.push(tx.hash()); - eternalTransactionsHashList.push(tx.hash()); - } else { - const actionHash = MinaActions.actionHash( - tx.toRuntimeTransaction().hashData() - ); - - incomingMessagesList.push(actionHash); - } - } catch (error) { - if (error instanceof Error) { - log.error("Error in inclusion of tx, skipping", error); - } - } - } - - const previousBlockHash = - lastResult.blockHash === 0n ? undefined : Field(lastResult.blockHash); - - if (executionResults.length === 0 && !allowEmptyBlocks) { - log.info( - "After sequencing, block has no sequencable transactions left, skipping block" - ); - return undefined; - } - - const block: Omit = { - transactions: executionResults, - transactionsHash: transactionsHashList.commitment, - fromEternalTransactionsHash: lastBlock.toEternalTransactionsHash, - toEternalTransactionsHash: eternalTransactionsHashList.commitment, - height: - lastBlock.hash.toBigInt() !== 0n ? lastBlock.height.add(1) : Field(0), - fromBlockHashRoot: Field(lastResult.blockHashRoot), - fromMessagesHash: lastBlock.toMessagesHash, - toMessagesHash: incomingMessagesList.commitment, - previousBlockHash, - - networkState: { - before: new NetworkState(lastResult.afterNetworkState), - during: networkState, - }, - }; - - const hash = Block.hash(block); - - return { - ...block, - hash, - }; - } - - public async generateMetadataForNextBlock( - block: Block, - merkleTreeStore: AsyncMerkleTreeStore, - blockHashTreeStore: AsyncMerkleTreeStore, - modifyTreeStore = true - ): Promise { - // Flatten diff list into a single diff by applying them over each other - const combinedDiff = block.transactions - .map((tx) => { - const transitions = tx.protocolTransitions.concat( - tx.status.toBoolean() ? tx.stateTransitions : [] - ); - return this.collectStateDiff(transitions); - }) - .reduce((accumulator, diff) => { - // accumulator properties will be overwritten by diff's values - return Object.assign(accumulator, diff); - }, {}); - - const inMemoryStore = new CachedMerkleTreeStore(merkleTreeStore); - const tree = new RollupMerkleTree(inMemoryStore); - const blockHashInMemoryStore = new CachedMerkleTreeStore( - blockHashTreeStore - ); - const blockHashTree = new BlockHashMerkleTree(blockHashInMemoryStore); - - await inMemoryStore.preloadKeys(Object.keys(combinedDiff).map(BigInt)); - - // In case the diff is empty, we preload key 0 in order to - // retrieve the root, which we need later - if (Object.keys(combinedDiff).length === 0) { - await inMemoryStore.preloadKey(0n); - } - - // TODO This can be optimized a lot (we are only interested in the root at this step) - await blockHashInMemoryStore.preloadKey(block.height.toBigInt()); - - Object.entries(combinedDiff).forEach(([key, state]) => { - const treeValue = state !== undefined ? Poseidon.hash(state) : Field(0); - tree.setLeaf(BigInt(key), treeValue); - }); - - const stateRoot = tree.getRoot(); - const fromBlockHashRoot = blockHashTree.getRoot(); - - const state: BlockProverState = { - stateRoot, - transactionsHash: block.transactionsHash, - networkStateHash: block.networkState.during.hash(), - eternalTransactionsHash: block.toEternalTransactionsHash, - blockHashRoot: fromBlockHashRoot, - incomingMessagesHash: block.toMessagesHash, - }; - - // TODO Set StateProvider for @state access to state - this.executionContext.clear(); - this.executionContext.setup({ - networkState: block.networkState.during, - transaction: RuntimeTransaction.dummyTransaction(), - }); - - const resultingNetworkState = await this.blockHooks.reduce< - Promise - >( - async (networkState, hook) => - await hook.afterBlock(await networkState, state), - Promise.resolve(block.networkState.during) - ); - - const { stateTransitions } = this.executionContext.result; - this.executionContext.clear(); - const reducedStateTransitions = reduceStateTransitions(stateTransitions); - - // Update the block hash tree with this block - blockHashTree.setLeaf( - block.height.toBigInt(), - new BlockHashTreeEntry({ - blockHash: Poseidon.hash([block.height, state.transactionsHash]), - closed: Bool(true), - }).hash() - ); - const blockHashWitness = blockHashTree.getWitness(block.height.toBigInt()); - const newBlockHashRoot = blockHashTree.getRoot(); - await blockHashInMemoryStore.mergeIntoParent(); - - if (modifyTreeStore) { - await inMemoryStore.mergeIntoParent(); - } - - return { - afterNetworkState: resultingNetworkState, - stateRoot: stateRoot.toBigInt(), - blockHashRoot: newBlockHashRoot.toBigInt(), - blockHashWitness, - - blockStateTransitions: reducedStateTransitions.map((st) => - UntypedStateTransition.fromStateTransition(st) - ), - blockHash: block.hash.toBigInt(), - }; - } - - private async createExecutionTrace( + public async createExecutionTrace( asyncStateService: CachedStateService, tx: PendingTransaction, networkState: NetworkState From 6bf074133d159aba622beddcfd1f0c8cf905803a Mon Sep 17 00:00:00 2001 From: Raphael Panic Date: Tue, 7 Jan 2025 18:02:43 +0100 Subject: [PATCH 2/5] Functional refactor of block production --- .../sequencing/BlockProducerModule.ts | 1 - .../sequencing/BlockProductionService.ts | 1 + .../sequencing/BlockResultService.ts | 68 ++--- .../sequencing/TransactionExecutionService.ts | 236 +++++++++--------- packages/stack/test/graphql/graphql.test.ts | 3 +- 5 files changed, 162 insertions(+), 147 deletions(-) diff --git a/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts b/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts index fd9698f81..d0bb41338 100644 --- a/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts +++ b/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts @@ -22,7 +22,6 @@ import { Block, BlockWithResult } from "../../../storage/model/Block"; import { CachedStateService } from "../../../state/state/CachedStateService"; import { MessageStorage } from "../../../storage/repositories/MessageStorage"; -import { TransactionExecutionService } from "./TransactionExecutionService"; import { BlockProductionService } from "./BlockProductionService"; import { BlockResultService } from "./BlockResultService"; diff --git a/packages/sequencer/src/protocol/production/sequencing/BlockProductionService.ts b/packages/sequencer/src/protocol/production/sequencing/BlockProductionService.ts index c19234016..b36c3b004 100644 --- a/packages/sequencer/src/protocol/production/sequencing/BlockProductionService.ts +++ b/packages/sequencer/src/protocol/production/sequencing/BlockProductionService.ts @@ -19,6 +19,7 @@ import { } from "../../../storage/model/Block"; import { CachedStateService } from "../../../state/state/CachedStateService"; import { PendingTransaction } from "../../../mempool/PendingTransaction"; + import { TransactionExecutionService } from "./TransactionExecutionService"; @injectable() diff --git a/packages/sequencer/src/protocol/production/sequencing/BlockResultService.ts b/packages/sequencer/src/protocol/production/sequencing/BlockResultService.ts index 4bc25ccf0..b03e5bb56 100644 --- a/packages/sequencer/src/protocol/production/sequencing/BlockResultService.ts +++ b/packages/sequencer/src/protocol/production/sequencing/BlockResultService.ts @@ -1,4 +1,3 @@ -import type { StateRecord } from "../BatchProducerModule"; import { Bool, Field, Poseidon } from "o1js"; import { RollupMerkleTree } from "@proto-kit/common"; import { @@ -10,16 +9,21 @@ import { Protocol, ProtocolModulesRecord, ProvableBlockHook, - reduceStateTransitions, - RuntimeMethodExecutionContext, RuntimeTransaction, } from "@proto-kit/protocol"; import { inject, injectable, Lifecycle, scoped } from "tsyringe"; -import { Block, BlockResult } from "../../../storage/model/Block"; +import { + Block, + BlockResult, + TransactionExecutionResult, +} from "../../../storage/model/Block"; import { AsyncMerkleTreeStore } from "../../../state/async/AsyncMerkleTreeStore"; import { CachedMerkleTreeStore } from "../../../state/merkle/CachedMerkleTreeStore"; import { UntypedStateTransition } from "../helpers/UntypedStateTransition"; +import type { StateRecord } from "../BatchProducerModule"; + +import { executeWithExecutionContext } from "./TransactionExecutionService"; function collectStateDiff( stateTransitions: UntypedStateTransition[] @@ -35,13 +39,27 @@ function collectStateDiff( ); } +function createCombinedStateDiff(transactions: TransactionExecutionResult[]) { + // Flatten diff list into a single diff by applying them over each other + return transactions + .map((tx) => { + const transitions = tx.protocolTransitions.concat( + tx.status.toBoolean() ? tx.stateTransitions : [] + ); + return collectStateDiff(transitions); + }) + .reduce((accumulator, diff) => { + // accumulator properties will be overwritten by diff's values + return Object.assign(accumulator, diff); + }, {}); +} + @injectable() @scoped(Lifecycle.ContainerScoped) export class BlockResultService { private readonly blockHooks: ProvableBlockHook[]; public constructor( - private readonly executionContext: RuntimeMethodExecutionContext, @inject("Protocol") protocol: Protocol ) { @@ -55,18 +73,7 @@ export class BlockResultService { blockHashTreeStore: AsyncMerkleTreeStore, modifyTreeStore = true ): Promise { - // Flatten diff list into a single diff by applying them over each other - const combinedDiff = block.transactions - .map((tx) => { - const transitions = tx.protocolTransitions.concat( - tx.status.toBoolean() ? tx.stateTransitions : [] - ); - return collectStateDiff(transitions); - }) - .reduce((accumulator, diff) => { - // accumulator properties will be overwritten by diff's values - return Object.assign(accumulator, diff); - }, {}); + const combinedDiff = createCombinedStateDiff(block.transactions); const inMemoryStore = new CachedMerkleTreeStore(merkleTreeStore); const tree = new RollupMerkleTree(inMemoryStore); @@ -104,23 +111,22 @@ export class BlockResultService { }; // TODO Set StateProvider for @state access to state - this.executionContext.clear(); - this.executionContext.setup({ + const context = { networkState: block.networkState.during, transaction: RuntimeTransaction.dummyTransaction(), - }); + }; - const resultingNetworkState = await this.blockHooks.reduce< - Promise - >( - async (networkState, hook) => - await hook.afterBlock(await networkState, state), - Promise.resolve(block.networkState.during) + const executionResult = await executeWithExecutionContext( + async () => + await this.blockHooks.reduce>( + async (networkState, hook) => + await hook.afterBlock(await networkState, state), + Promise.resolve(block.networkState.during) + ), + context ); - const { stateTransitions } = this.executionContext.result; - this.executionContext.clear(); - const reducedStateTransitions = reduceStateTransitions(stateTransitions); + const { stateTransitions, methodResult } = executionResult; // Update the block hash tree with this block blockHashTree.setLeaf( @@ -139,12 +145,12 @@ export class BlockResultService { } return { - afterNetworkState: resultingNetworkState, + afterNetworkState: methodResult, stateRoot: stateRoot.toBigInt(), blockHashRoot: newBlockHashRoot.toBigInt(), blockHashWitness, - blockStateTransitions: reducedStateTransitions.map((st) => + blockStateTransitions: stateTransitions.map((st) => UntypedStateTransition.fromStateTransition(st) ), blockHash: block.hash.toBigInt(), diff --git a/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts b/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts index f581d5a99..9663b8929 100644 --- a/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts +++ b/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts @@ -11,6 +11,7 @@ import { StateServiceProvider, MandatoryProtocolModulesRecord, reduceStateTransitions, + StateTransition, } from "@proto-kit/protocol"; import { Field } from "o1js"; import { AreProofsEnabled, log, mapSequential } from "@proto-kit/common"; @@ -33,6 +34,119 @@ const errors = { export type SomeRuntimeMethod = (...args: unknown[]) => Promise; +export type RuntimeContextReducedExecutionResult = Pick< + RuntimeProvableMethodExecutionResult, + "stateTransitions" | "status" | "statusMessage" | "stackTrace" | "events" +>; + +function getAreProofsEnabledFromModule( + module: RuntimeModule +): AreProofsEnabled { + if (module.runtime === undefined) { + throw new Error("Runtime on RuntimeModule not set"); + } + if (module.runtime.areProofsEnabled === undefined) { + throw new Error("AppChain on Runtime not set"); + } + const { areProofsEnabled } = module.runtime; + return areProofsEnabled; +} + +async function decodeTransaction( + tx: PendingTransaction, + runtime: Runtime +): Promise<{ + method: SomeRuntimeMethod; + args: unknown[]; + module: RuntimeModule; +}> { + const methodDescriptors = runtime.methodIdResolver.getMethodNameFromId( + tx.methodId.toBigInt() + ); + + const method = runtime.getMethodById(tx.methodId.toBigInt()); + + if (methodDescriptors === undefined || method === undefined) { + throw errors.methodIdNotFound(tx.methodId.toString()); + } + + const [moduleName, methodName] = methodDescriptors; + const module: RuntimeModule = runtime.resolve(moduleName); + + const parameterDecoder = MethodParameterEncoder.fromMethod( + module, + methodName + ); + const args = await parameterDecoder.decode(tx.argsFields, tx.auxiliaryData); + + return { + method, + args, + module, + }; +} + +function extractEvents( + runtimeResult: RuntimeContextReducedExecutionResult +): { eventName: string; data: Field[] }[] { + return runtimeResult.events.reduce( + (acc, event) => { + if (event.condition.toBoolean()) { + const obj = { + eventName: event.eventName, + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + data: event.eventType.toFields(event.event), + }; + acc.push(obj); + } + return acc; + }, + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + [] as { eventName: string; data: Field[] }[] + ); +} + +export async function executeWithExecutionContext( + method: () => Promise, + contextInputs: RuntimeMethodExecutionData, + runSimulated = false +): Promise< + RuntimeContextReducedExecutionResult & { methodResult: MethodResult } +> { + // Set up context + const executionContext = container.resolve(RuntimeMethodExecutionContext); + + executionContext.setup(contextInputs); + executionContext.setSimulated(runSimulated); + + // Execute method + const methodResult = await method(); + + const { stateTransitions, status, statusMessage, events } = + executionContext.current().result; + + const reducedSTs = reduceStateTransitions(stateTransitions); + + return { + stateTransitions: reducedSTs, + status, + statusMessage, + events, + methodResult, + }; +} + +function traceSTs(msg: string, stateTransitions: StateTransition[]) { + log.trace( + msg, + JSON.stringify( + stateTransitions.map((x) => x.toJSON()), + null, + 2 + ) + ); +} + @injectable() @scoped(Lifecycle.ContainerScoped) export class TransactionExecutionService { @@ -51,88 +165,12 @@ export class TransactionExecutionService { ); } - private async decodeTransaction(tx: PendingTransaction): Promise<{ - method: SomeRuntimeMethod; - args: unknown[]; - module: RuntimeModule; - }> { - const methodDescriptors = this.runtime.methodIdResolver.getMethodNameFromId( - tx.methodId.toBigInt() - ); - - const method = this.runtime.getMethodById(tx.methodId.toBigInt()); - - if (methodDescriptors === undefined || method === undefined) { - throw errors.methodIdNotFound(tx.methodId.toString()); - } - - const [moduleName, methodName] = methodDescriptors; - const module: RuntimeModule = this.runtime.resolve(moduleName); - - const parameterDecoder = MethodParameterEncoder.fromMethod( - module, - methodName - ); - const args = await parameterDecoder.decode(tx.argsFields, tx.auxiliaryData); - - return { - method, - args, - module, - }; - } - - private getAppChainForModule( - module: RuntimeModule - ): AreProofsEnabled { - if (module.runtime === undefined) { - throw new Error("Runtime on RuntimeModule not set"); - } - if (module.runtime.areProofsEnabled === undefined) { - throw new Error("AppChain on Runtime not set"); - } - const { areProofsEnabled } = module.runtime; - return areProofsEnabled; - } - - private async executeWithExecutionContext( - method: () => Promise, - contextInputs: RuntimeMethodExecutionData, - runSimulated = false - ): Promise< - Pick< - RuntimeProvableMethodExecutionResult, - "stateTransitions" | "status" | "statusMessage" | "stackTrace" | "events" - > - > { - // Set up context - const executionContext = container.resolve(RuntimeMethodExecutionContext); - - executionContext.setup(contextInputs); - executionContext.setSimulated(runSimulated); - - // Execute method - await method(); - - const { stateTransitions, status, statusMessage, events } = - executionContext.current().result; - - const reducedSTs = reduceStateTransitions(stateTransitions); - - return { - stateTransitions: reducedSTs, - status, - statusMessage, - events, - }; - } - private async executeRuntimeMethod( method: SomeRuntimeMethod, args: unknown[], contextInputs: RuntimeMethodExecutionData ) { - return await this.executeWithExecutionContext(async () => { + return await executeWithExecutionContext(async () => { await method(...args); }, contextInputs); } @@ -152,7 +190,7 @@ export class TransactionExecutionService { blockContextInputs: BlockProverExecutionData, runSimulated = false ) { - return await this.executeWithExecutionContext( + return await executeWithExecutionContext( async () => await this.wrapHooksForContext(async () => { await mapSequential( @@ -175,10 +213,10 @@ export class TransactionExecutionService { // TODO Use RecordingStateService -> async asProver needed const recordingStateService = new CachedStateService(asyncStateService); - const { method, args, module } = await this.decodeTransaction(tx); + const { method, args, module } = await decodeTransaction(tx, this.runtime); // Disable proof generation for tracing - const appChain = this.getAppChainForModule(module); + const appChain = getAreProofsEnabledFromModule(module); const previousProofsEnabled = appChain.areProofsEnabled; appChain.setProofsEnabled(false); @@ -212,14 +250,7 @@ export class TransactionExecutionService { throw error; } - log.trace( - "PSTs:", - JSON.stringify( - protocolResult.stateTransitions.map((x) => x.toJSON()), - null, - 2 - ) - ); + traceSTs("STs:", protocolResult.stateTransitions); // Apply protocol STs await recordingStateService.applyStateTransitions( @@ -231,14 +262,7 @@ export class TransactionExecutionService { args, runtimeContextInputs ); - log.trace( - "STs:", - JSON.stringify( - runtimeResult.stateTransitions.map((x) => x.toJSON()), - null, - 2 - ) - ); + traceSTs("STs:", runtimeResult.stateTransitions); // Apply runtime STs (only if the tx succeeded) if (runtimeResult.status.toBoolean()) { @@ -256,22 +280,8 @@ export class TransactionExecutionService { // Reset proofs enabled appChain.setProofsEnabled(previousProofsEnabled); - const eventsReduced: { eventName: string; data: Field[] }[] = - runtimeResult.events.reduce( - (acc, event) => { - if (event.condition.toBoolean()) { - const obj = { - eventName: event.eventName, - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - data: event.eventType.toFields(event.event), - }; - acc.push(obj); - } - return acc; - }, - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - [] as { eventName: string; data: Field[] }[] - ); + const events = extractEvents(runtimeResult); + return { tx, status: runtimeResult.status, @@ -285,7 +295,7 @@ export class TransactionExecutionService { UntypedStateTransition.fromStateTransition(st) ), - events: eventsReduced, + events, }; } } diff --git a/packages/stack/test/graphql/graphql.test.ts b/packages/stack/test/graphql/graphql.test.ts index c80a33095..3a477c162 100644 --- a/packages/stack/test/graphql/graphql.test.ts +++ b/packages/stack/test/graphql/graphql.test.ts @@ -11,8 +11,7 @@ import { import { Field, PrivateKey } from "o1js"; import { sleep } from "@proto-kit/common"; import { ManualBlockTrigger, Sequencer } from "@proto-kit/sequencer"; -import { GraphqlServer } from "@proto-kit/api"; -import { +import {w AppChain, InMemorySigner, GraphqlTransactionSender, From 9a23eec756fc715882e1b0a8b32ef63eb185f5a2 Mon Sep 17 00:00:00 2001 From: Raphael Panic Date: Tue, 7 Jan 2025 18:15:26 +0100 Subject: [PATCH 3/5] Added missing clear for hooks --- .../production/sequencing/TransactionExecutionService.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts b/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts index 9663b8929..922826e44 100644 --- a/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts +++ b/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts @@ -116,6 +116,7 @@ export async function executeWithExecutionContext( // Set up context const executionContext = container.resolve(RuntimeMethodExecutionContext); + executionContext.clear(); executionContext.setup(contextInputs); executionContext.setSimulated(runSimulated); @@ -250,7 +251,7 @@ export class TransactionExecutionService { throw error; } - traceSTs("STs:", protocolResult.stateTransitions); + traceSTs("PSTs:", protocolResult.stateTransitions); // Apply protocol STs await recordingStateService.applyStateTransitions( From 04241578fcd7d2c36cb58023c9d09a34dcd41665 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20=C5=A0ima?= Date: Tue, 28 Jan 2025 15:38:43 +0100 Subject: [PATCH 4/5] Update graphql.test.ts --- packages/stack/test/graphql/graphql.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/stack/test/graphql/graphql.test.ts b/packages/stack/test/graphql/graphql.test.ts index 3a477c162..b79fcd0fb 100644 --- a/packages/stack/test/graphql/graphql.test.ts +++ b/packages/stack/test/graphql/graphql.test.ts @@ -11,7 +11,7 @@ import { import { Field, PrivateKey } from "o1js"; import { sleep } from "@proto-kit/common"; import { ManualBlockTrigger, Sequencer } from "@proto-kit/sequencer"; -import {w +import { AppChain, InMemorySigner, GraphqlTransactionSender, From cd55c790afc98867e201c3f8e89da2b7eb2d53ed Mon Sep 17 00:00:00 2001 From: Raphael Panic Date: Tue, 28 Jan 2025 16:33:11 +0100 Subject: [PATCH 5/5] Resolved merge problems --- .../sequencing/BlockProducerModule.ts | 2 +- .../sequencing/BlockResultService.ts | 33 ++++++++++--------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts b/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts index bf0563254..83fe9bbbf 100644 --- a/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts +++ b/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts @@ -109,7 +109,7 @@ export class BlockProducerModule extends SequencerModule { public async generateMetadata(block: Block): Promise { const { result, blockHashTreeStore, treeStore } = - await this.executionService.generateMetadataForNextBlock( + await this.resultService.generateMetadataForNextBlock( block, this.unprovenMerkleStore, this.blockTreeStore diff --git a/packages/sequencer/src/protocol/production/sequencing/BlockResultService.ts b/packages/sequencer/src/protocol/production/sequencing/BlockResultService.ts index b03e5bb56..41c3d9ba5 100644 --- a/packages/sequencer/src/protocol/production/sequencing/BlockResultService.ts +++ b/packages/sequencer/src/protocol/production/sequencing/BlockResultService.ts @@ -72,7 +72,11 @@ export class BlockResultService { merkleTreeStore: AsyncMerkleTreeStore, blockHashTreeStore: AsyncMerkleTreeStore, modifyTreeStore = true - ): Promise { + ): Promise<{ + result: BlockResult; + treeStore: CachedMerkleTreeStore; + blockHashTreeStore: CachedMerkleTreeStore; + }> { const combinedDiff = createCombinedStateDiff(block.transactions); const inMemoryStore = new CachedMerkleTreeStore(merkleTreeStore); @@ -138,22 +142,21 @@ export class BlockResultService { ); const blockHashWitness = blockHashTree.getWitness(block.height.toBigInt()); const newBlockHashRoot = blockHashTree.getRoot(); - await blockHashInMemoryStore.mergeIntoParent(); - - if (modifyTreeStore) { - await inMemoryStore.mergeIntoParent(); - } return { - afterNetworkState: methodResult, - stateRoot: stateRoot.toBigInt(), - blockHashRoot: newBlockHashRoot.toBigInt(), - blockHashWitness, - - blockStateTransitions: stateTransitions.map((st) => - UntypedStateTransition.fromStateTransition(st) - ), - blockHash: block.hash.toBigInt(), + result: { + afterNetworkState: methodResult, + stateRoot: stateRoot.toBigInt(), + blockHashRoot: newBlockHashRoot.toBigInt(), + blockHashWitness, + + blockStateTransitions: stateTransitions.map((st) => + UntypedStateTransition.fromStateTransition(st) + ), + blockHash: block.hash.toBigInt(), + }, + treeStore: inMemoryStore, + blockHashTreeStore: blockHashInMemoryStore, }; } }