diff --git a/packages/common/src/utils.ts b/packages/common/src/utils.ts index 0576aacb5..73eb0233b 100644 --- a/packages/common/src/utils.ts +++ b/packages/common/src/utils.ts @@ -295,3 +295,21 @@ export function assertDefined( throw new Error(msg ?? "Value is undefined"); } } + +export function compareStrings(a: string, b: string): number { + // eslint-disable-next-line no-nested-ternary + return a === b ? 0 : a > b ? 1 : -1; +} + +export function recordByKey( + array: Object[], + keyMapper: (obj: Object) => string +): Record { + return Object.fromEntries( + array.map((obj) => { + const key = keyMapper(obj); + const copy = { ...obj }; + return [key, copy]; + }) + ); +} diff --git a/packages/library/src/hooks/RuntimeFeeAnalyzerService.ts b/packages/library/src/hooks/RuntimeFeeAnalyzerService.ts index 707d80a4c..6deb0ff10 100644 --- a/packages/library/src/hooks/RuntimeFeeAnalyzerService.ts +++ b/packages/library/src/hooks/RuntimeFeeAnalyzerService.ts @@ -2,6 +2,7 @@ import { ConfigurableModule, createMerkleTree, InMemoryMerkleTreeStorage, + recordByKey, } from "@proto-kit/common"; import { Runtime, RuntimeModulesRecord } from "@proto-kit/module"; import { container, inject } from "tsyringe"; @@ -10,6 +11,7 @@ import { RuntimeTransaction, NetworkState, } from "@proto-kit/protocol"; +import { RuntimeAnalyzerService } from "@proto-kit/sequencer"; import { Field, Poseidon, Struct } from "o1js"; import { UInt64 } from "../math/UInt64"; @@ -65,6 +67,7 @@ export class RuntimeFeeAnalyzerService extends ConfigurableModule ) { super(); @@ -81,71 +84,41 @@ export class RuntimeFeeAnalyzerService extends ConfigurableModule - >( - async (accum, program) => { - const [valuesProg, indexesProg] = await accum; - const analyzedMethods = await program.analyzeMethods(); - const [valuesMeth, indexesMeth] = Object.keys(program.methods).reduce< - [FeeTreeValues, FeeIndexes] - >( - // eslint-disable-next-line @typescript-eslint/no-shadow - ([values, indexes], combinedMethodName) => { - const { rows } = analyzedMethods[combinedMethodName]; - // const rows = 1000; - const [moduleName, methodName] = combinedMethodName.split("."); - const methodId = this.runtime.methodIdResolver.getMethodId( - moduleName, - methodName - ); - - /** - * Determine the fee config for the given method id, and merge it with - * the default fee config. - */ - return [ - { - ...values, - - [methodId.toString()]: { - methodId, - - baseFee: - this.config.methods[combinedMethodName]?.baseFee ?? - this.config.baseFee, - - perWeightUnitFee: - this.config.methods[combinedMethodName] - ?.perWeightUnitFee ?? this.config.perWeightUnitFee, - - weight: - this.config.methods[combinedMethodName]?.weight ?? - BigInt(rows), - }, - }, - { - ...indexes, - // eslint-disable-next-line no-plusplus - [methodId.toString()]: BigInt(methodCounter++), - }, - ]; - }, - [{}, {}] - ); - return [ - { ...valuesProg, ...valuesMeth }, - { ...indexesProg, ...indexesMeth }, - ]; - }, - Promise.resolve([{}, {}]) - ); + const runtimeInfo = await this.runtimeAnalyzerService.getRuntimeInfo(); + + const values = Object.entries(runtimeInfo).map( + ([combinedMethodName, { rows }]) => { + // const rows = 1000; + const [moduleName, methodName] = combinedMethodName.split("."); + const methodId = this.runtime.methodIdResolver.getMethodId( + moduleName, + methodName + ); + + /** + * Determine the fee config for the given method id, and merge it with + * the default fee config. + */ + return { + methodId, + + baseFee: + this.config.methods[combinedMethodName]?.baseFee ?? + this.config.baseFee, + + perWeightUnitFee: + this.config.methods[combinedMethodName]?.perWeightUnitFee ?? + this.config.perWeightUnitFee, + + weight: + this.config.methods[combinedMethodName]?.weight ?? BigInt(rows), + }; + } + ); const tree = new FeeTree(new InMemoryMerkleTreeStorage()); - Object.values(values).forEach((value, index) => { + const indexes = values.map((value, index) => { const feeConfig = new MethodFeeConfigData({ methodId: Field(value.methodId), baseFee: UInt64.from(value.baseFee), @@ -153,14 +126,19 @@ export class RuntimeFeeAnalyzerService extends ConfigurableModule v.methodId.toString()), + indexes: Object.fromEntries(indexes), + }; } public getFeeTree() { if (this.persistedFeeTree === undefined) { - throw new Error("Fee Tree not intialized"); + throw new Error("Fee Tree not initialized"); } return this.persistedFeeTree; diff --git a/packages/library/src/hooks/TransactionFeeHook.ts b/packages/library/src/hooks/TransactionFeeHook.ts index 20d9b0adc..8b23f36cf 100644 --- a/packages/library/src/hooks/TransactionFeeHook.ts +++ b/packages/library/src/hooks/TransactionFeeHook.ts @@ -12,6 +12,7 @@ import { } from "@proto-kit/protocol"; import { Field, Provable, PublicKey } from "o1js"; import { noop } from "@proto-kit/common"; +import { RuntimeAnalyzerService } from "@proto-kit/sequencer"; import { UInt64 } from "../math/UInt64"; import { Balance, TokenId } from "../runtime/Balances"; @@ -53,7 +54,9 @@ export class TransactionFeeHook extends ProvableTransactionHook, - @inject("Balances") public balances: Balances + @inject("Balances") public balances: Balances, + // TODO Check that the container is the right one here + public runtimeAnalyzerService: RuntimeAnalyzerService ) { super(); } @@ -79,7 +82,10 @@ export class TransactionFeeHook extends ProvableTransactionHook[] { - type Methods = Record< - string, - { - privateInputs: any; - method: AsyncWrappedMethod; - } - >; // We need to use explicit type annotations here, // therefore we can't use destructuring - // eslint-disable-next-line prefer-destructuring - const runtime: Runtime = this.runtime; - const MAXIMUM_METHODS_PER_ZK_PROGRAM = 8; - const runtimeMethods = runtime.runtimeModuleNames.reduce( - (allMethods, runtimeModuleName) => { - runtime.isValidModuleName( - runtime.definition.modules, - runtimeModuleName - ); - - /** - * Couldnt find a better way to circumvent the type assertion - * regarding resolving only known modules. We assert in the line above - * but we cast it to any anyways to satisfy the proof system. - */ + const runtimeMethods = this.runtime.collectMethods(); - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - const runtimeModule = runtime.resolve(runtimeModuleName as any); - - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - const modulePrototype = Object.getPrototypeOf(runtimeModule) as Record< - string, - // Technically not all methods have to be async, but for this context it's ok - (...args: unknown[]) => Promise - >; - - const modulePrototypeMethods = getAllPropertyNames(runtimeModule).map( - (method) => method.toString() - ); - - const moduleMethods = modulePrototypeMethods.reduce( - (allModuleMethods, methodName) => { - if (isRuntimeMethod(runtimeModule, methodName)) { - const combinedMethodName = combineMethodName( - runtimeModuleName, - methodName - ); - const method = modulePrototype[methodName]; - const invocationType = Reflect.getMetadata( - runtimeMethodTypeMetadataKey, - runtimeModule, - methodName - ); - - const wrappedMethod: AsyncWrappedMethod = Reflect.apply( - toWrappedMethod, - runtimeModule, - [methodName, method, { invocationType }] - ); - - const privateInputs = Reflect.getMetadata( - "design:paramtypes", - runtimeModule, - methodName - ); - - return { - ...allModuleMethods, - - [combinedMethodName]: { - privateInputs, - method: wrappedMethod, - }, - }; - } - - return allModuleMethods; - }, - {} - ); - - return { - ...allMethods, - ...moduleMethods, - }; - }, - {} - ); - - const sortedRuntimeMethods = Object.fromEntries( - Object.entries(runtimeMethods).sort() + const sortedRuntimeMethods = runtimeMethods.sort( + ({ combinedMethodName: name1 }, { combinedMethodName: name2 }) => + compareStrings(name1, name2) ); const splitRuntimeMethods = () => { @@ -191,9 +111,15 @@ export class RuntimeZkProgrammable< } > > = []; - Object.entries(sortedRuntimeMethods).forEach( - async ([methodName, method]) => { + sortedRuntimeMethods.forEach( + async ({ combinedMethodName: methodName, method, privateInputs }) => { let methodAdded = false; + + const methodDefinition = { + privateInputs, + method, + }; + for (const bucket of buckets) { if (buckets.length === 0) { const record: Record< @@ -203,7 +129,7 @@ export class RuntimeZkProgrammable< method: AsyncWrappedMethod; } > = {}; - record[methodName] = method; + record[methodName] = methodDefinition; buckets.push(record); methodAdded = true; break; @@ -211,11 +137,12 @@ export class RuntimeZkProgrammable< Object.keys(bucket).length <= MAXIMUM_METHODS_PER_ZK_PROGRAM - 1 ) { - bucket[methodName] = method; + bucket[methodName] = methodDefinition; methodAdded = true; break; } } + if (!methodAdded) { const record: Record< string, @@ -224,7 +151,7 @@ export class RuntimeZkProgrammable< method: AsyncWrappedMethod; } > = {}; - record[methodName] = method; + record[methodName] = methodDefinition; buckets.push(record); } } @@ -381,6 +308,71 @@ export class Runtime return Object.keys(this.definition.modules); } + public collectMethods() { + return this.runtimeModuleNames.flatMap((runtimeModuleName) => { + this.isValidModuleName(this.definition.modules, runtimeModuleName); + + /** + * Couldnt find a better way to circumvent the type assertion + * regarding resolving only known modules. We assert in the line above + * but we cast it to any anyways to satisfy the proof system. + */ + + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + const runtimeModule = this.resolve(runtimeModuleName as any); + + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + const modulePrototype = Object.getPrototypeOf(runtimeModule) as Record< + string, + // Technically not all methods have to be async, but for this context it's ok + (...args: unknown[]) => Promise + >; + + const modulePrototypeMethods = getAllPropertyNames(runtimeModule).map( + (method) => method.toString() + ); + + return modulePrototypeMethods + .map((methodName) => { + if (isRuntimeMethod(runtimeModule, methodName)) { + const combinedMethodName = combineMethodName( + runtimeModuleName, + methodName + ); + const method = modulePrototype[methodName]; + const invocationType: RuntimeMethodInvocationType = + Reflect.getMetadata( + runtimeMethodTypeMetadataKey, + runtimeModule, + methodName + ); + + const wrappedMethod: AsyncWrappedMethod = Reflect.apply( + toWrappedMethod, + runtimeModule, + [methodName, method, { invocationType }] + ); + + const privateInputs: any[] | undefined = Reflect.getMetadata( + "design:paramtypes", + runtimeModule, + methodName + ); + + return { + combinedMethodName, + privateInputs: privateInputs ?? [], + method: wrappedMethod, + invocationType, + }; + } else { + return undefined; + } + }) + .filter(filterNonUndefined); + }, {}); + } + public async compile(registry: CompileRegistry) { const context = container.resolve(RuntimeMethodExecutionContext); context.setup({ diff --git a/packages/module/test/TestingRuntime.ts b/packages/module/src/testing/TestingRuntime.ts similarity index 88% rename from packages/module/test/TestingRuntime.ts rename to packages/module/src/testing/TestingRuntime.ts index 5b46ce97d..8b0903f9b 100644 --- a/packages/module/test/TestingRuntime.ts +++ b/packages/module/src/testing/TestingRuntime.ts @@ -2,7 +2,8 @@ import { ModulesConfig } from "@proto-kit/common"; import { StateServiceProvider } from "@proto-kit/protocol"; import { container } from "tsyringe"; -import { InMemoryStateService, Runtime, RuntimeModulesRecord } from "../src"; +import { Runtime, RuntimeModulesRecord } from "../runtime/Runtime"; +import { InMemoryStateService } from "../state/InMemoryStateService"; export function createTestingRuntime( modules: Modules, diff --git a/packages/module/test/modules/Balances.test.ts b/packages/module/test/modules/Balances.test.ts index d61f6a3d9..4afdb00b3 100644 --- a/packages/module/test/modules/Balances.test.ts +++ b/packages/module/test/modules/Balances.test.ts @@ -13,7 +13,7 @@ import { } from "@proto-kit/protocol"; import { Runtime } from "../../src"; -import { createTestingRuntime } from "../TestingRuntime"; +import { createTestingRuntime } from "../../src/testing/TestingRuntime"; import { Balances } from "./Balances.js"; import { Admin } from "./Admin.js"; diff --git a/packages/module/test/modules/MethodIdResolver.test.ts b/packages/module/test/modules/MethodIdResolver.test.ts index b6ae685da..c7a2a5773 100644 --- a/packages/module/test/modules/MethodIdResolver.test.ts +++ b/packages/module/test/modules/MethodIdResolver.test.ts @@ -7,7 +7,7 @@ import { container } from "tsyringe"; import { Runtime } from "../../src/runtime/Runtime"; import { MethodIdResolver } from "../../src/runtime/MethodIdResolver"; import { runtimeMethod, RuntimeModule, runtimeModule } from "../../src"; -import { createTestingRuntime } from "../TestingRuntime"; +import { createTestingRuntime } from "../../src/testing/TestingRuntime"; import { Balances } from "./Balances"; diff --git a/packages/module/test/modules/State.test.ts b/packages/module/test/modules/State.test.ts index 8f5f55533..b91687851 100644 --- a/packages/module/test/modules/State.test.ts +++ b/packages/module/test/modules/State.test.ts @@ -10,7 +10,7 @@ import { import { expectDefined } from "@proto-kit/common"; import { Runtime } from "../../src"; -import { createTestingRuntime } from "../TestingRuntime"; +import { createTestingRuntime } from "../../src/testing/TestingRuntime"; import { Admin } from "./Admin"; import { Balances } from "./Balances"; diff --git a/packages/module/test/runtimeMethod.test.ts b/packages/module/test/runtimeMethod.test.ts index ca895ae31..179d7be54 100644 --- a/packages/module/test/runtimeMethod.test.ts +++ b/packages/module/test/runtimeMethod.test.ts @@ -25,10 +25,10 @@ import { runtimeMethod, toEventsHash, RuntimeEvents, + createTestingRuntime, } from "../src"; import { Balances } from "./modules/Balances"; -import { createTestingRuntime } from "./TestingRuntime"; export class PrimaryTestEvent extends Struct({ message: Bool, diff --git a/packages/protocol/src/state/State.ts b/packages/protocol/src/state/State.ts index 36400d8c1..e5db2a8e5 100644 --- a/packages/protocol/src/state/State.ts +++ b/packages/protocol/src/state/State.ts @@ -8,7 +8,10 @@ import { Option } from "../model/Option"; import { StateTransition } from "../model/StateTransition"; import { StateServiceProvider } from "./StateServiceProvider"; -import { RuntimeMethodExecutionContext } from "./context/RuntimeMethodExecutionContext"; +import { + RuntimeMethodExecutionContext, + StateAccessType, +} from "./context/RuntimeMethodExecutionContext"; export class WithPath { public path?: Field; @@ -50,7 +53,10 @@ export class State extends Mixin(WithPath, WithStateServiceProvider) { return new State(valueType); } - public constructor(public valueType: FlexibleProvablePure) { + public constructor( + public valueType: FlexibleProvablePure, + public accessType: StateAccessType = "static" + ) { super(); } @@ -139,7 +145,7 @@ export class State extends Mixin(WithPath, WithStateServiceProvider) { container .resolve(RuntimeMethodExecutionContext) - .addStateTransition(stateTransition); + .addStateTransition(stateTransition, this.accessType); return option; } @@ -170,6 +176,6 @@ export class State extends Mixin(WithPath, WithStateServiceProvider) { container .resolve(RuntimeMethodExecutionContext) - .addStateTransition(stateTransition); + .addStateTransition(stateTransition, this.accessType); } } diff --git a/packages/protocol/src/state/StateMap.ts b/packages/protocol/src/state/StateMap.ts index 53fe51be4..328fd77d3 100644 --- a/packages/protocol/src/state/StateMap.ts +++ b/packages/protocol/src/state/StateMap.ts @@ -43,10 +43,14 @@ export class StateMap extends Mixin( * Obtains a value for the provided key in the current state map. * * @param key - Key to obtain the state for + * // TODO dynamic * @returns Value for the provided key. */ - public async get(key: KeyType): Promise> { - const state = State.from(this.valueType); + public async get( + key: KeyType, + { dynamic }: { dynamic: boolean } = { dynamic: false } + ): Promise> { + const state = new State(this.valueType, dynamic ? "dynamic" : "static"); this.hasPathOrFail(); this.hasStateServiceOrFail(); @@ -60,9 +64,14 @@ export class StateMap extends Mixin( * * @param key - Key to store the value under * @param value - Value to be stored under the given key + * TODO options */ - public async set(key: KeyType, value: ValueType): Promise { - const state = State.from(this.valueType); + public async set( + key: KeyType, + value: ValueType, + { dynamic }: { dynamic: boolean } = { dynamic: false } + ): Promise { + const state = new State(this.valueType, dynamic ? "dynamic" : "static"); this.hasPathOrFail(); this.hasStateServiceOrFail(); diff --git a/packages/protocol/src/state/context/RuntimeMethodExecutionContext.ts b/packages/protocol/src/state/context/RuntimeMethodExecutionContext.ts index 4bc75dabc..dc72d6fb9 100644 --- a/packages/protocol/src/state/context/RuntimeMethodExecutionContext.ts +++ b/packages/protocol/src/state/context/RuntimeMethodExecutionContext.ts @@ -16,6 +16,8 @@ const errors = { ), }; +export type StateAccessType = "static" | "dynamic"; + export class RuntimeProvableMethodExecutionResult extends ProvableMethodExecutionResult { public stateTransitions: StateTransition[] = []; @@ -25,6 +27,8 @@ export class RuntimeProvableMethodExecutionResult extends ProvableMethodExecutio public stackTrace?: string; + public accessTypes: StateAccessType[] = []; + public events: { eventType: FlexibleProvablePure; event: any; @@ -80,9 +84,13 @@ export class RuntimeMethodExecutionContext extends ProvableMethodExecutionContex * Adds an in-method generated state transition to the current context * @param stateTransition - State transition to add to the context */ - public addStateTransition(stateTransition: StateTransition) { + public addStateTransition( + stateTransition: StateTransition, + accessType: StateAccessType + ) { this.assertSetupCalled(); this.result.stateTransitions.push(stateTransition); + this.result.accessTypes.push(accessType); } public addEvent( diff --git a/packages/sequencer/src/index.ts b/packages/sequencer/src/index.ts index 8800bc888..376830bb3 100644 --- a/packages/sequencer/src/index.ts +++ b/packages/sequencer/src/index.ts @@ -60,6 +60,7 @@ export * from "./protocol/production/tracing/BlockTracingService"; export * from "./protocol/production/tracing/BatchTracingService"; export * from "./protocol/production/tracing/StateTransitionTracingService"; export * from "./protocol/production/tracing/TransactionTracingService"; +export * from "./protocol/runtime/RuntimeAnalyzerService"; export * from "./sequencer/SequencerStartupModule"; export * from "./storage/model/Batch"; export * from "./storage/model/Block"; diff --git a/packages/sequencer/src/protocol/production/sequencing/BlockProductionService.ts b/packages/sequencer/src/protocol/production/sequencing/BlockProductionService.ts index 385789a25..6085c047c 100644 --- a/packages/sequencer/src/protocol/production/sequencing/BlockProductionService.ts +++ b/packages/sequencer/src/protocol/production/sequencing/BlockProductionService.ts @@ -16,23 +16,20 @@ import { import { Field } from "o1js"; import { log } from "@proto-kit/common"; -import { - Block, - BlockWithResult, - TransactionExecutionResult, -} from "../../../storage/model/Block"; +import { Block, BlockWithResult } from "../../../storage/model/Block"; import { CachedStateService } from "../../../state/state/CachedStateService"; import { PendingTransaction } from "../../../mempool/PendingTransaction"; import { AsyncStateService } from "../../../state/async/AsyncStateService"; import { UntypedStateTransition } from "../helpers/UntypedStateTransition"; import { Tracer } from "../../../logging/Tracer"; import { trace } from "../../../logging/trace"; +import { TransactionUtils } from "../utils/transaction-utils"; import { BlockTrackers, - executeWithExecutionContext, TransactionExecutionService, } from "./TransactionExecutionService"; +import { TransactionPreprocessor } from "./preprocessing/TransactionPreprocessor"; @injectable() @scoped(Lifecycle.ContainerScoped) @@ -46,7 +43,8 @@ export class BlockProductionService { public readonly tracer: Tracer, private readonly transactionExecutionService: TransactionExecutionService, @inject("StateServiceProvider") - private readonly stateServiceProvider: StateServiceProvider + private readonly stateServiceProvider: StateServiceProvider, + private readonly transactionPreprocessor: TransactionPreprocessor ) { this.blockHooks = protocol.dependencyContainer.resolveAll("ProvableBlockHook"); @@ -66,7 +64,7 @@ export class BlockProductionService { transaction: RuntimeTransaction.dummyTransaction(), }; - const executionResult = await executeWithExecutionContext( + const executionResult = await TransactionUtils.executeWithExecutionContext( async () => await this.blockHooks.reduce>( async (networkState, hook) => @@ -100,6 +98,9 @@ export class BlockProductionService { } | undefined > { + const preprocessedTransactions = + await this.transactionPreprocessor.batchPreprocess(transactions); + const stateService = new CachedStateService(asyncStateService); const lastResult = lastBlockWithResult.result; @@ -135,7 +136,7 @@ export class BlockProductionService { const [newBlockState, executionResults] = await this.transactionExecutionService.createExecutionTraces( stateService, - transactions, + preprocessedTransactions, networkState, blockState ); diff --git a/packages/sequencer/src/protocol/production/sequencing/BlockResultService.ts b/packages/sequencer/src/protocol/production/sequencing/BlockResultService.ts index 5419344bb..536741033 100644 --- a/packages/sequencer/src/protocol/production/sequencing/BlockResultService.ts +++ b/packages/sequencer/src/protocol/production/sequencing/BlockResultService.ts @@ -29,8 +29,7 @@ import { trace } from "../../../logging/trace"; import { Tracer } from "../../../logging/Tracer"; import { AsyncLinkedLeafStore } from "../../../state/async/AsyncLinkedLeafStore"; import { CachedLinkedLeafStore } from "../../../state/lmt/CachedLinkedLeafStore"; - -import { executeWithExecutionContext } from "./TransactionExecutionService"; +import { TransactionUtils } from "../utils/transaction-utils"; // This is ordered, because javascript maintains the order based on time of first insertion function collectOrderedStateDiff( @@ -100,7 +99,7 @@ export class BlockResultService { transaction: RuntimeTransaction.dummyTransaction(), }; - const executionResult = await executeWithExecutionContext( + const executionResult = await TransactionUtils.executeWithExecutionContext( async () => await this.blockHooks.reduce>( async (networkState, hook) => diff --git a/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts b/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts index 0e12cd7a1..86f59c26c 100644 --- a/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts +++ b/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts @@ -8,7 +8,6 @@ import { ProvableTransactionHook, RuntimeMethodExecutionContext, RuntimeMethodExecutionData, - RuntimeProvableMethodExecutionResult, StateServiceProvider, MandatoryProtocolModulesRecord, reduceStateTransitions, @@ -27,7 +26,6 @@ import { import { Bool, Field } from "o1js"; import { AreProofsEnabled, log, mapSequential } from "@proto-kit/common"; import { - MethodParameterEncoder, Runtime, RuntimeModule, RuntimeModulesRecord, @@ -45,18 +43,12 @@ import { import { UntypedStateTransition } from "../helpers/UntypedStateTransition"; import { trace } from "../../../logging/trace"; import { Tracer } from "../../../logging/Tracer"; +import { distinct } from "../../../helpers/utils"; +import { TransactionUtils } from "../utils/transaction-utils"; -const errors = { - methodIdNotFound: (methodId: string) => - new Error(`Can't find runtime method with id ${methodId}`), -}; +import { PreprocessedTransaction } from "./preprocessing/TransactionPreprocessor"; -export type SomeRuntimeMethod = (...args: unknown[]) => Promise; - -export type RuntimeContextReducedExecutionResult = Pick< - RuntimeProvableMethodExecutionResult, - "stateTransitions" | "status" | "statusMessage" | "stackTrace" | "events" ->; +import SomeRuntimeMethod = TransactionUtils.SomeRuntimeMethod; export type BlockTrackers = Pick< BlockProverState, @@ -79,70 +71,6 @@ function getAreProofsEnabledFromModule( return areProofsEnabled; } -async function decodeTransaction( - tx: PendingTransaction, - runtime: Runtime -): Promise<{ - method: SomeRuntimeMethod; - args: unknown[]; - module: RuntimeModule; -}> { - const methodDescriptors = runtime.methodIdResolver.getMethodNameFromId( - tx.methodId.toBigInt() - ); - - const method = runtime.getMethodById(tx.methodId.toBigInt()); - - if (methodDescriptors === undefined || method === undefined) { - throw errors.methodIdNotFound(tx.methodId.toString()); - } - - const [moduleName, methodName] = methodDescriptors; - const module: RuntimeModule = runtime.resolve(moduleName); - - const parameterDecoder = MethodParameterEncoder.fromMethod( - module, - methodName - ); - const args = await parameterDecoder.decode(tx.argsFields, tx.auxiliaryData); - - return { - method, - args, - module, - }; -} - -function extractEvents( - runtimeResult: RuntimeContextReducedExecutionResult, - source: "afterTxHook" | "beforeTxHook" | "runtime" -): { - eventName: string; - data: Field[]; - source: "afterTxHook" | "beforeTxHook" | "runtime"; -}[] { - return runtimeResult.events.reduce( - (acc, event) => { - if (event.condition.toBoolean()) { - const obj = { - eventName: event.eventName, - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - data: event.eventType.toFields(event.event), - source: source, - }; - acc.push(obj); - } - return acc; - }, - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - [] as { - eventName: string; - data: Field[]; - source: "afterTxHook" | "beforeTxHook" | "runtime"; - }[] - ); -} - // TODO Also use this in tracing as a replacement of toStateTransitionHash export function toStateTransitionHashNonProvable( stateTransitions: StateTransition[] @@ -155,37 +83,6 @@ export function toStateTransitionHashNonProvable( return list.commitment; } -export async function executeWithExecutionContext( - method: () => Promise, - contextInputs: RuntimeMethodExecutionData, - runSimulated = false -): Promise< - RuntimeContextReducedExecutionResult & { methodResult: MethodResult } -> { - // Set up context - const executionContext = container.resolve(RuntimeMethodExecutionContext); - - executionContext.clear(); - executionContext.setup(contextInputs); - executionContext.setSimulated(runSimulated); - - // Execute method - const methodResult = await method(); - - const { stateTransitions, status, statusMessage, events } = - executionContext.current().result; - - const reducedSTs = reduceStateTransitions(stateTransitions); - - return { - stateTransitions: reducedSTs, - status, - statusMessage, - events, - methodResult, - }; -} - function traceLogSTs(msg: string, stateTransitions: StateTransition[]) { log.trace( msg, @@ -226,7 +123,7 @@ export class TransactionExecutionService { args: unknown[], contextInputs: RuntimeMethodExecutionData ) { - return await executeWithExecutionContext(async () => { + return await TransactionUtils.executeWithExecutionContext(async () => { await method(...args); }, contextInputs); } @@ -252,7 +149,7 @@ export class TransactionExecutionService { hookName: string, runSimulated = false ) { - const result = await executeWithExecutionContext( + const result = await TransactionUtils.executeWithExecutionContext( async () => await this.wrapHooksForContext(async () => { await mapSequential( @@ -317,18 +214,32 @@ export class TransactionExecutionService { ); } + @trace("block.preprocess.preload-state") + public async preloadKnownStatePaths( + transactions: PreprocessedTransaction[], + asyncStateService: CachedStateService + ) { + const paths = transactions + .flatMap(({ accessedStatePaths }) => accessedStatePaths) + .filter(distinct); + + await asyncStateService.preloadKeys(paths.map(Field)); + } + public async createExecutionTraces( asyncStateService: CachedStateService, - transactions: PendingTransaction[], + transactions: PreprocessedTransaction[], networkState: NetworkState, state: BlockTrackers ): Promise<[BlockTrackers, TransactionExecutionResult[]]> { + await this.preloadKnownStatePaths(transactions, asyncStateService); + let blockState = state; const executionResults: TransactionExecutionResult[] = []; const networkStateHash = networkState.hash(); - for (const tx of transactions) { + for (const { tx } of transactions) { try { const newState = this.addTransactionToBlockProverState(state, tx); @@ -375,7 +286,10 @@ export class TransactionExecutionService { // TODO Use RecordingStateService -> async asProver needed const recordingStateService = new CachedStateService(asyncStateService); - const { method, args, module } = await decodeTransaction(tx, this.runtime); + const { method, args, module } = await TransactionUtils.decodeTransaction( + tx, + this.runtime + ); // Disable proof generation for sequencing the runtime // TODO Is that even needed? @@ -409,7 +323,10 @@ export class TransactionExecutionService { "beforeTx" ) ); - const beforeHookEvents = extractEvents(beforeTxHookResult, "beforeTxHook"); + const beforeHookEvents = TransactionUtils.extractEvents( + beforeTxHookResult, + "beforeTxHook" + ); await recordingStateService.applyStateTransitions( beforeTxHookResult.stateTransitions @@ -458,7 +375,10 @@ export class TransactionExecutionService { "afterTx" ) ); - const afterHookEvents = extractEvents(afterTxHookResult, "afterTxHook"); + const afterHookEvents = TransactionUtils.extractEvents( + afterTxHookResult, + "afterTxHook" + ); await recordingStateService.applyStateTransitions( afterTxHookResult.stateTransitions ); @@ -472,7 +392,10 @@ export class TransactionExecutionService { appChain.setProofsEnabled(previousProofsEnabled); // Extract sequencing results - const runtimeResultEvents = extractEvents(runtimeResult, "runtime"); + const runtimeResultEvents = TransactionUtils.extractEvents( + runtimeResult, + "runtime" + ); const stateTransitions = this.buildSTBatches( [ beforeTxHookResult.stateTransitions, diff --git a/packages/sequencer/src/protocol/production/sequencing/preprocessing/TransactionPreprocessor.ts b/packages/sequencer/src/protocol/production/sequencing/preprocessing/TransactionPreprocessor.ts new file mode 100644 index 000000000..ba444dd73 --- /dev/null +++ b/packages/sequencer/src/protocol/production/sequencing/preprocessing/TransactionPreprocessor.ts @@ -0,0 +1,109 @@ +import { inject, injectable } from "tsyringe"; +import { + InMemoryStateService, + Runtime, + RuntimeModulesRecord, +} from "@proto-kit/module"; +import { mapSequential } from "@proto-kit/common"; +import { NetworkState, StateServiceProvider } from "@proto-kit/protocol"; + +import { PendingTransaction } from "../../../../mempool/PendingTransaction"; +import { distinct } from "../../../../helpers/utils"; +import { + RuntimeAnalyzerService, + RuntimeInfo, +} from "../../../runtime/RuntimeAnalyzerService"; +import { TransactionUtils } from "../../utils/transaction-utils"; +import { Tracer } from "../../../../logging/Tracer"; +import { trace } from "../../../../logging/trace"; + +export type PreprocessedTransaction = { + tx: PendingTransaction; + dynamicKeyAccess: boolean; + accessedStatePaths: bigint[]; +}; + +@injectable() +export class TransactionPreprocessor { + public constructor( + @inject("Runtime") private readonly runtime: Runtime, + @inject("StateServiceProvider") + private readonly stateServiceProvider: StateServiceProvider, + @inject("Tracer") + public readonly tracer: Tracer, + private readonly runtimeAnalyzerService: RuntimeAnalyzerService + ) {} + + @trace("block.preprocess.txs") + public async batchPreprocess( + txs: PendingTransaction[] + ): Promise { + this.setupStateService(); + const runtimeInfo = await this.runtimeAnalyzerService.getRuntimeInfo(); + + const preprocessedTransactions = await mapSequential(txs, async (tx) => { + return await this.preprocessSingleTransaction(tx, runtimeInfo); + }); + + this.stateServiceProvider.popCurrentStateService(); + + return preprocessedTransactions; + } + + public async preprocess( + tx: PendingTransaction + ): Promise { + this.setupStateService(); + + const preprocessedTransaction = await this.preprocessSingleTransaction( + tx, + await this.runtimeAnalyzerService.getRuntimeInfo() + ); + + this.stateServiceProvider.popCurrentStateService(); + + return preprocessedTransaction; + } + + private setupStateService() { + this.stateServiceProvider.setCurrentStateService( + new InMemoryStateService() + ); + } + + private async preprocessSingleTransaction( + tx: PendingTransaction, + runtimeInfo: RuntimeInfo + ): Promise { + const { args, method, combinedMethodName } = + await TransactionUtils.decodeTransaction(tx, this.runtime); + const { dynamicKeyAccess } = runtimeInfo[combinedMethodName]; + + if (!dynamicKeyAccess) { + const result = await TransactionUtils.executeWithExecutionContext( + async () => { + await method(...args); + }, + { + transaction: tx.toRuntimeTransaction(), + networkState: NetworkState.empty(), + }, + true + ); + + return { + tx, + dynamicKeyAccess, + accessedStatePaths: result.stateTransitions + .map((x) => x.path.toBigInt()) + .filter(distinct), + }; + } else { + return { + tx, + dynamicKeyAccess, + accessedStatePaths: [], + }; + } + } +} diff --git a/packages/sequencer/src/protocol/production/utils/transaction-utils.ts b/packages/sequencer/src/protocol/production/utils/transaction-utils.ts new file mode 100644 index 000000000..794eb14d0 --- /dev/null +++ b/packages/sequencer/src/protocol/production/utils/transaction-utils.ts @@ -0,0 +1,127 @@ +import { + MethodParameterEncoder, + Runtime, + RuntimeModule, + RuntimeModulesRecord, +} from "@proto-kit/module"; +import { + reduceStateTransitions, + RuntimeMethodExecutionContext, + RuntimeMethodExecutionData, + RuntimeProvableMethodExecutionResult, +} from "@proto-kit/protocol"; +import { container } from "tsyringe"; +import { Field } from "o1js"; + +import { PendingTransaction } from "../../../mempool/PendingTransaction"; + +export namespace TransactionUtils { + const errors = { + methodIdNotFound: (methodId: string) => + new Error(`Can't find runtime method with id ${methodId}`), + }; + + export type SomeRuntimeMethod = (...args: unknown[]) => Promise; + + export type RuntimeContextReducedExecutionResult = Pick< + RuntimeProvableMethodExecutionResult, + "stateTransitions" | "status" | "statusMessage" | "stackTrace" | "events" + >; + + export async function decodeTransaction( + tx: PendingTransaction, + runtime: Runtime + ): Promise<{ + method: SomeRuntimeMethod; + args: unknown[]; + module: RuntimeModule; + combinedMethodName: string; + }> { + const methodDescriptors = runtime.methodIdResolver.getMethodNameFromId( + tx.methodId.toBigInt() + ); + + const method = runtime.getMethodById(tx.methodId.toBigInt()); + + if (methodDescriptors === undefined || method === undefined) { + throw errors.methodIdNotFound(tx.methodId.toString()); + } + + const [moduleName, methodName] = methodDescriptors; + const module: RuntimeModule = runtime.resolve(moduleName); + + const parameterDecoder = MethodParameterEncoder.fromMethod( + module, + methodName + ); + const args = await parameterDecoder.decode(tx.argsFields, tx.auxiliaryData); + + return { + method, + args, + module, + combinedMethodName: `${moduleName}.${methodName}`, + }; + } + + export function extractEvents( + runtimeResult: RuntimeContextReducedExecutionResult, + source: "afterTxHook" | "beforeTxHook" | "runtime" + ): { + eventName: string; + data: Field[]; + source: "afterTxHook" | "beforeTxHook" | "runtime"; + }[] { + return runtimeResult.events.reduce( + (acc, event) => { + if (event.condition.toBoolean()) { + const obj = { + eventName: event.eventName, + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + data: event.eventType.toFields(event.event), + source: source, + }; + acc.push(obj); + } + return acc; + }, + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + [] as { + eventName: string; + data: Field[]; + source: "afterTxHook" | "beforeTxHook" | "runtime"; + }[] + ); + } + + export async function executeWithExecutionContext( + method: () => Promise, + contextInputs: RuntimeMethodExecutionData, + runSimulated = false + ): Promise< + RuntimeContextReducedExecutionResult & { methodResult: MethodResult } + > { + // Set up context + const executionContext = container.resolve(RuntimeMethodExecutionContext); + + executionContext.clear(); + executionContext.setup(contextInputs); + executionContext.setSimulated(runSimulated); + + // Execute method + const methodResult = await method(); + + const { stateTransitions, status, statusMessage, events } = + executionContext.current().result; + + const reducedSTs = reduceStateTransitions(stateTransitions); + + return { + stateTransitions: reducedSTs, + status, + statusMessage, + events, + methodResult, + }; + } +} diff --git a/packages/sequencer/src/protocol/runtime/RuntimeAnalyzerService.ts b/packages/sequencer/src/protocol/runtime/RuntimeAnalyzerService.ts new file mode 100644 index 000000000..20706f2be --- /dev/null +++ b/packages/sequencer/src/protocol/runtime/RuntimeAnalyzerService.ts @@ -0,0 +1,90 @@ +import { container, inject, injectable } from "tsyringe"; +import { Runtime, RuntimeModulesRecord } from "@proto-kit/module"; +import { + NetworkState, + RuntimeMethodExecutionContext, + RuntimeMethodInvocationType, + RuntimeTransaction, +} from "@proto-kit/protocol"; +import { mapSequential } from "@proto-kit/common"; +import { + analyzeMethod, + sortMethodArguments, +} from "o1js/dist/node/lib/proof-system/zkprogram"; +import { Void } from "o1js"; + +export type RuntimeMethodMetadata = { + rows: number; + dynamicKeyAccess: boolean; + invocationType: RuntimeMethodInvocationType; +}; + +export type RuntimeInfo = Record; + +@injectable() +export class RuntimeAnalyzerService { + public constructor( + @inject("Runtime") public runtime: Runtime + ) {} + + private computedRuntimeInfo: + | Record + | undefined = undefined; + + private async computeRuntimeInfo(): Promise { + const context = container.resolve( + RuntimeMethodExecutionContext + ); + + context.setup({ + transaction: RuntimeTransaction.dummyTransaction(), + networkState: NetworkState.empty(), + }); + context.clear(); + + const runtimeMethods = this.runtime.collectMethods(); + + const infos = await mapSequential( + runtimeMethods, + async ({ + combinedMethodName, + privateInputs, + method, + invocationType, + }): Promise<[string, RuntimeMethodMetadata]> => { + const methodIntf = sortMethodArguments( + "", + combinedMethodName, + privateInputs, + // eslint-disable-next-line max-len + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions,@typescript-eslint/no-unsafe-argument + undefined as any + ); + + const constraintSystem = await analyzeMethod(Void, methodIntf, method); + + const { result } = context.current(); + const dynamicKeyAccess = result.accessTypes.some( + (x) => x === "dynamic" + ); + + return [ + combinedMethodName, + { + rows: constraintSystem.rows, + dynamicKeyAccess, + invocationType, + }, + ]; + } + ); + return Object.fromEntries(infos); + } + + public async getRuntimeInfo(): Promise { + if (this.computedRuntimeInfo === undefined) { + this.computedRuntimeInfo = await this.computeRuntimeInfo(); + } + return this.computedRuntimeInfo; + } +} diff --git a/packages/sequencer/src/sequencer/executor/Sequencer.ts b/packages/sequencer/src/sequencer/executor/Sequencer.ts index 4a746c8bd..cd749fc6a 100644 --- a/packages/sequencer/src/sequencer/executor/Sequencer.ts +++ b/packages/sequencer/src/sequencer/executor/Sequencer.ts @@ -18,11 +18,11 @@ import { } from "@proto-kit/protocol"; import { DependencyContainer, injectable } from "tsyringe"; -import { sequencerModule, SequencerModule } from "../builder/SequencerModule"; -import { closeable, Closeable } from "../builder/Closeable"; +import { SequencerModule } from "../builder/SequencerModule"; +import { Closeable } from "../builder/Closeable"; +import { ConsoleTracingFactory } from "../../logging/ConsoleTracingFactory"; import { Sequenceable } from "./Sequenceable"; -import { ConsoleTracingFactory } from "../../logging/ConsoleTracingFactory"; export type SequencerModulesRecord = ModulesRecord< TypedClass> diff --git a/packages/sequencer/src/state/state/CachedStateService.ts b/packages/sequencer/src/state/state/CachedStateService.ts index cc1167015..659f95f8b 100644 --- a/packages/sequencer/src/state/state/CachedStateService.ts +++ b/packages/sequencer/src/state/state/CachedStateService.ts @@ -57,7 +57,9 @@ export class CachedStateService if (this.parent !== undefined) { // Only preload it if it hasn't been preloaded previously // TODO Not safe for deletes - const keysToBeLoaded = keys.filter((key) => this.get(key) === undefined); + const keysToBeLoaded = keys.filter( + (key) => this.getNullAware(key) === undefined + ); const loaded = await this.parent.getMany(keysToBeLoaded); log.trace( @@ -75,29 +77,33 @@ export class CachedStateService public async getMany(keys: Field[]): Promise { const remoteKeys: Field[] = []; - const local: StateEntry[] = []; + let stateEntries: StateEntry[] = []; keys.forEach((key) => { const localValue = this.getNullAware(key); if (localValue !== undefined) { - local.push({ key, value: localValue ?? undefined }); + stateEntries.push({ key, value: localValue ?? undefined }); } else { remoteKeys.push(key); } }); - const remote = await this.parent?.getMany(remoteKeys); + if (remoteKeys.length > 0) { + const remote = await this.parent?.getMany(remoteKeys); - if (remote !== undefined) { - // Update the remotely fetched keys into local cache - await mapSequential(remote, async ({ key, value }) => { - if (this.getNullAware(key) === undefined) { - await this.set(key, value); - } - }); + if (remote !== undefined) { + // Update the remotely fetched keys into local cache + await mapSequential(remote, async ({ key, value }) => { + if (this.getNullAware(key) === undefined) { + await this.set(key, value); + } + }); + + stateEntries = stateEntries.concat(...remote); + } } - return local.concat(remote ?? []); + return stateEntries; } public async get(key: Field): Promise { diff --git a/packages/sequencer/test-integration/benchmarks/tps.test.ts b/packages/sequencer/test-integration/benchmarks/tps.test.ts index fe4e27436..33a483190 100644 --- a/packages/sequencer/test-integration/benchmarks/tps.test.ts +++ b/packages/sequencer/test-integration/benchmarks/tps.test.ts @@ -127,7 +127,7 @@ export async function createAppChain() { const timeout = 600000; -describe.skip("tps", () => { +describe("tps", () => { let appChain: Awaited>; let privateKeys: PrivateKey[] = []; let balances: Balances; diff --git a/packages/sequencer/test/integration/mocks/Balance.ts b/packages/sequencer/test/integration/mocks/Balance.ts index 7f66cf63f..8e140bc66 100644 --- a/packages/sequencer/test/integration/mocks/Balance.ts +++ b/packages/sequencer/test/integration/mocks/Balance.ts @@ -111,4 +111,10 @@ export class Balance extends RuntimeModule { await this.totalSupply.set(supply.add(UInt64.from(100))); }); } + + @runtimeMethod() + public async dynamicKeyTest(address: PublicKey) { + // Not actually dynamic but enough for the unit tests + await this.balances.set(address, UInt64.one, { dynamic: true }); + } } diff --git a/packages/sequencer/test/protocol/runtime/RuntimeAnalyzeService.test.ts b/packages/sequencer/test/protocol/runtime/RuntimeAnalyzeService.test.ts new file mode 100644 index 000000000..4cce5af5e --- /dev/null +++ b/packages/sequencer/test/protocol/runtime/RuntimeAnalyzeService.test.ts @@ -0,0 +1,42 @@ +import "reflect-metadata"; +import { + createTestingRuntime, + Runtime, + RuntimeModulesRecord, +} from "@proto-kit/module"; +import { expectDefined } from "@proto-kit/common"; + +import { RuntimeAnalyzerService } from "../../../src"; +import { Balance } from "../../integration/mocks/Balance"; + +describe("RuntimeAnalyzerService", () => { + let service: RuntimeAnalyzerService | undefined = undefined; + + beforeAll(() => { + const { runtime } = createTestingRuntime({ Balance }, { Balance: {} }); + + service = new RuntimeAnalyzerService( + runtime as unknown as Runtime + ); + }); + + it("should analyze static method correctly", async () => { + const info = await service!.getRuntimeInfo(); + const depositInfo = info["Balance.deposit"]; + + expectDefined(depositInfo); + expect(depositInfo.rows).toBeGreaterThan(10); + expect(depositInfo.dynamicKeyAccess).toBe(false); + expect(depositInfo.invocationType).toBe("INCOMING_MESSAGE"); + }); + + it("should analyze dynamic method correctly", async () => { + const info = await service!.getRuntimeInfo(); + const depositInfo = info["Balance.dynamicKeyTest"]; + + expectDefined(depositInfo); + expect(depositInfo.rows).toBeGreaterThan(10); + expect(depositInfo.dynamicKeyAccess).toBe(true); + expect(depositInfo.invocationType).toBe("SIGNATURE"); + }); +});