diff --git a/packages/api/src/metrics/OpenTelemetryTracer.ts b/packages/api/src/metrics/OpenTelemetryTracer.ts index f020127d8..215ecab56 100644 --- a/packages/api/src/metrics/OpenTelemetryTracer.ts +++ b/packages/api/src/metrics/OpenTelemetryTracer.ts @@ -1,7 +1,21 @@ import opentelemetry, { SpanStatusCode } from "@opentelemetry/api"; import { Tracer } from "@proto-kit/sequencer"; +import { inject, injectable } from "tsyringe"; +import { noop } from "@proto-kit/common"; +import type { OpenTelemetryServer } from "./OpenTelemetryServer"; + +@injectable() export class OpenTelemetryTracer implements Tracer { + public constructor( + // We need to import this here, so that the OpenTelemetryServer will be resolved + // before this module, and therefore will be already started when this module is + // eventually consumed and used + @inject("OpenTelemetryServer") openTelemetryServer: OpenTelemetryServer + ) { + noop(); + } + private tracer: ReturnType | undefined = undefined; diff --git a/packages/common/src/config/ModuleContainer.ts b/packages/common/src/config/ModuleContainer.ts index 2d625b327..3977a21e6 100644 --- a/packages/common/src/config/ModuleContainer.ts +++ b/packages/common/src/config/ModuleContainer.ts @@ -263,6 +263,10 @@ export class ModuleContainer< this.onAfterModuleResolution(moduleName); this.registerAliases(moduleName, useClass); + + if (this.isDependencyFactory(useClass)) { + this.useDependencyFactory(useClass); + } } }); } @@ -285,16 +289,6 @@ export class ModuleContainer< }); } - protected registerClasses(modules: Record>) { - Object.entries(modules).forEach(([moduleName, useClass]) => { - this.container.register( - moduleName, - { useClass }, - { lifecycle: Lifecycle.ContainerScoped } - ); - }); - } - /** * Provide additional configuration after the ModuleContainer was created. * @@ -403,6 +397,7 @@ export class ModuleContainer< protected useDependencyFactory(factory: DependencyFactory) { const dependencies = factory.dependencies(); + // eslint-disable-next-line sonarjs/cognitive-complexity Object.entries(dependencies).forEach(([rawKey, declaration]) => { const key = rawKey.charAt(0).toUpperCase() + rawKey.slice(1); @@ -410,7 +405,10 @@ export class ModuleContainer< !this.container.isRegistered(key) || declaration.forceOverwrite === true ) { - if (this.container.isRegistered(key) && declaration.forceOverwrite) { + if ( + this.container.isRegistered(key) && + (declaration?.forceOverwrite ?? false) + ) { log.warn( `You are trying to overwrite dependency ${key}, which is already registered. This is currently not supported. Try to define your dependency earlier.` ); @@ -434,6 +432,11 @@ export class ModuleContainer< // eslint-disable-next-line @typescript-eslint/consistent-type-assertions declaration.useClass as TypedClass ); + + // Register static dependencies + if (this.isDependencyFactory(declaration.useClass)) { + this.useDependencyFactory(declaration.useClass); + } } else if (isTokenProvider(declaration)) { this.container.register(key, declaration, { lifecycle: Lifecycle.Singleton, diff --git a/packages/deployment/docker/monitoring/grafana/grafana-datasources.yaml b/packages/deployment/docker/monitoring/grafana/grafana-datasources.yaml index cc275cb47..0a34a69ec 100644 --- a/packages/deployment/docker/monitoring/grafana/grafana-datasources.yaml +++ b/packages/deployment/docker/monitoring/grafana/grafana-datasources.yaml @@ -30,4 +30,4 @@ datasources: serviceMap: datasourceUid: prometheus streamingEnabled: - search: true \ No newline at end of file + search: true diff --git a/packages/deployment/docker/monitoring/otel-collector/otel-collector.yaml b/packages/deployment/docker/monitoring/otel-collector/otel-collector.yaml index e11401eb2..87c46f72c 100644 --- a/packages/deployment/docker/monitoring/otel-collector/otel-collector.yaml +++ b/packages/deployment/docker/monitoring/otel-collector/otel-collector.yaml @@ -13,4 +13,4 @@ service: pipelines: traces: receivers: [otlp] - exporters: [otlp] \ No newline at end of file + exporters: [otlp] diff --git a/packages/deployment/docker/monitoring/promtail/promtail.yaml b/packages/deployment/docker/monitoring/promtail/promtail.yaml index dbc28d701..1f3da4c7c 100644 --- a/packages/deployment/docker/monitoring/promtail/promtail.yaml +++ b/packages/deployment/docker/monitoring/promtail/promtail.yaml @@ -17,10 +17,10 @@ scrape_configs: - name: label values: ["logging=promtail"] relabel_configs: - - source_labels: ['__meta_docker_container_name'] - regex: '/(.*)' - target_label: 'container' - - source_labels: ['__meta_docker_container_log_stream'] - target_label: 'logstream' - - source_labels: ['__meta_docker_container_label_logging_jobname'] - target_label: 'job' \ No newline at end of file + - source_labels: ["__meta_docker_container_name"] + regex: "/(.*)" + target_label: "container" + - source_labels: ["__meta_docker_container_log_stream"] + target_label: "logstream" + - source_labels: ["__meta_docker_container_label_logging_jobname"] + target_label: "job" diff --git a/packages/deployment/docker/monitoring/tempo/tempo.yaml b/packages/deployment/docker/monitoring/tempo/tempo.yaml index 828f1cac1..b463e923a 100644 --- a/packages/deployment/docker/monitoring/tempo/tempo.yaml +++ b/packages/deployment/docker/monitoring/tempo/tempo.yaml @@ -1,4 +1,3 @@ - stream_over_http_enabled: true server: http_listen_port: 3200 @@ -22,11 +21,11 @@ distributor: endpoint: "tempo:4317" ingester: - max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally + max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally compactor: compaction: - block_retention: 1h # overall Tempo trace retention. set for demo purposes + block_retention: 1h # overall Tempo trace retention. set for demo purposes metrics_generator: registry: @@ -43,9 +42,9 @@ metrics_generator: storage: trace: - backend: local # backend configuration to use + backend: local # backend configuration to use wal: - path: /var/tempo/wal # where to store the wal locally + path: /var/tempo/wal # where to store the wal locally local: path: /var/tempo/blocks diff --git a/packages/module/src/factories/MethodIdFactory.ts b/packages/module/src/factories/MethodIdFactory.ts index c80718f6a..fe6be798c 100644 --- a/packages/module/src/factories/MethodIdFactory.ts +++ b/packages/module/src/factories/MethodIdFactory.ts @@ -1,9 +1,9 @@ -import { DependencyFactory, DependencyRecord } from "@proto-kit/common"; +import { DependencyRecord } from "@proto-kit/common"; import { MethodIdResolver } from "../runtime/MethodIdResolver"; -export class MethodIdFactory implements DependencyFactory { - public dependencies() { +export class MethodIdFactory { + public static dependencies() { return { methodIdResolver: { useClass: MethodIdResolver, diff --git a/packages/module/src/runtime/Runtime.ts b/packages/module/src/runtime/Runtime.ts index bae879cc8..1a5795d96 100644 --- a/packages/module/src/runtime/Runtime.ts +++ b/packages/module/src/runtime/Runtime.ts @@ -303,7 +303,7 @@ export class Runtime public create(childContainerProvider: ChildContainerProvider) { super.create(childContainerProvider); - this.useDependencyFactory(this.container.resolve(MethodIdFactory)); + this.useDependencyFactory(MethodIdFactory); } public get areProofsEnabled(): AreProofsEnabled | undefined { diff --git a/packages/persistance/src/PrismaDatabaseConnection.ts b/packages/persistance/src/PrismaDatabaseConnection.ts index 4c477a86c..234f7977a 100644 --- a/packages/persistance/src/PrismaDatabaseConnection.ts +++ b/packages/persistance/src/PrismaDatabaseConnection.ts @@ -58,7 +58,7 @@ export class PrismaDatabaseConnection > { return { asyncStateService: { - useFactory: () => new PrismaStateService(this, "batch", this.tracer), + useFactory: () => new PrismaStateService(this, this.tracer, "batch"), }, batchStorage: { useClass: PrismaBatchStore, @@ -70,7 +70,7 @@ export class PrismaDatabaseConnection useClass: PrismaBlockStorage, }, unprovenStateService: { - useFactory: () => new PrismaStateService(this, "block", this.tracer), + useFactory: () => new PrismaStateService(this, this.tracer, "block"), }, settlementStorage: { useClass: PrismaSettlementStorage, diff --git a/packages/persistance/src/PrismaRedisDatabase.ts b/packages/persistance/src/PrismaRedisDatabase.ts index e4bb57b7c..92a68d0b5 100644 --- a/packages/persistance/src/PrismaRedisDatabase.ts +++ b/packages/persistance/src/PrismaRedisDatabase.ts @@ -9,6 +9,7 @@ import { import { ChildContainerProvider } from "@proto-kit/common"; import { PrismaClient } from "@prisma/client"; import { RedisClientType } from "redis"; +import { inject } from "tsyringe"; import { PrismaConnection, @@ -21,7 +22,6 @@ import { RedisConnectionModule, RedisTransaction, } from "./RedisConnection"; -import { inject } from "tsyringe"; export interface PrismaRedisCombinedConfig { prisma: PrismaDatabaseConfig; diff --git a/packages/persistance/src/services/prisma/PrismaStateService.ts b/packages/persistance/src/services/prisma/PrismaStateService.ts index 7e52cf044..73881127d 100644 --- a/packages/persistance/src/services/prisma/PrismaStateService.ts +++ b/packages/persistance/src/services/prisma/PrismaStateService.ts @@ -28,8 +28,8 @@ export class PrismaStateService implements AsyncStateService { */ public constructor( private readonly connection: PrismaConnection, - private readonly mask: string, - public readonly tracer: Tracer + public readonly tracer: Tracer, + private readonly mask: string ) {} @trace("db.state.commit") diff --git a/packages/persistance/src/services/prisma/mappers/EventMapper.ts b/packages/persistance/src/services/prisma/mappers/EventMapper.ts index 10d1488fc..b56416a87 100644 --- a/packages/persistance/src/services/prisma/mappers/EventMapper.ts +++ b/packages/persistance/src/services/prisma/mappers/EventMapper.ts @@ -4,45 +4,53 @@ import { Field } from "o1js"; import { ObjectMapper } from "../../../ObjectMapper"; +type EventData = { + eventName: string; + data: Field[]; + source: "afterTxHook" | "beforeTxHook" | "runtime"; +}; + @singleton() -export class EventMapper - implements - ObjectMapper<{ eventName: string; data: Field[] }, Prisma.JsonObject> -{ - public mapIn(input: Prisma.JsonObject): { eventName: string; data: Field[] } { - if (input === undefined) return { eventName: "", data: [] }; +export class EventMapper implements ObjectMapper { + public mapIn(input: Prisma.JsonObject): EventData { return { eventName: input.eventName as string, data: (input.data as Prisma.JsonArray).map((field) => Field.fromJSON(field as string) ), + source: this.sourceConvert(input.source as string), }; } - public mapOut(input: { - eventName: string; - data: Field[]; - }): Prisma.JsonObject { + public mapOut(input: EventData): Prisma.JsonObject { return { eventName: input.eventName, data: input.data.map((field) => field.toString()), + source: input.source, } as Prisma.JsonObject; } + + private sourceConvert(input: string) { + if ( + input === "beforeTxHook" || + input === "afterTxHook" || + input === "runtime" + ) { + return input; + } + throw new Error( + "Event Source must be one of 'beforeTxHook', 'afterTxHook' or 'runtime'" + ); + } } @singleton() export class EventArrayMapper - implements - ObjectMapper< - { eventName: string; data: Field[] }[], - Prisma.JsonValue | undefined - > + implements ObjectMapper { public constructor(private readonly eventMapper: EventMapper) {} - public mapIn( - input: Prisma.JsonValue | undefined - ): { eventName: string; data: Field[] }[] { + public mapIn(input: Prisma.JsonValue | undefined): EventData[] { if (input === undefined) return []; if (Array.isArray(input)) { @@ -53,9 +61,7 @@ export class EventArrayMapper return []; } - public mapOut( - input: { eventName: string; data: Field[] }[] - ): Prisma.JsonValue { + public mapOut(input: EventData[]): Prisma.JsonValue { return input.map((event) => this.eventMapper.mapOut(event) ) as Prisma.JsonArray; diff --git a/packages/persistance/test/connection.test.ts b/packages/persistance/test/connection.test.ts index ef83f454b..5252587cb 100644 --- a/packages/persistance/test/connection.test.ts +++ b/packages/persistance/test/connection.test.ts @@ -1,7 +1,7 @@ import "reflect-metadata"; import { describe } from "@jest/globals"; import { Field } from "o1js"; -import { CachedMerkleTreeStore } from "@proto-kit/sequencer"; +import { CachedMerkleTreeStore, ConsoleTracer } from "@proto-kit/sequencer"; import { expectDefined, RollupMerkleTree } from "@proto-kit/common"; import { @@ -14,15 +14,17 @@ import { // TODO Pull apart and test properly // Needs redis instance describe.skip("prisma", () => { + const tracer = new ConsoleTracer(); + it("merkle store", async () => { - const db = new RedisConnectionModule(); + const db = new RedisConnectionModule(tracer); db.config = { host: "localhost", port: 6379, password: "password", }; await db.start(); - const store = new RedisMerkleTreeStore(db); + const store = new RedisMerkleTreeStore(db, tracer); const cached = new CachedMerkleTreeStore(store); const tree = new RollupMerkleTree(cached); @@ -38,7 +40,7 @@ describe.skip("prisma", () => { console.log(`Root ${tree.getRoot().toBigInt()}`); - const store2 = new RedisMerkleTreeStore(db); + const store2 = new RedisMerkleTreeStore(db, tracer); const cached2 = new CachedMerkleTreeStore(store2); const tree2 = new RollupMerkleTree(cached2); @@ -57,10 +59,10 @@ describe.skip("prisma", () => { }); it("fill and get", async () => { - const db = new PrismaDatabaseConnection(); + const db = new PrismaDatabaseConnection(tracer); db.config = {}; await db.start(); - const service = new PrismaStateService(db, "testMask"); + const service = new PrismaStateService(db, tracer, "testMask"); await service.openTransaction(); service.writeStates([ diff --git a/packages/sdk/src/appChain/AppChain.ts b/packages/sdk/src/appChain/AppChain.ts index ac8d3c352..f7f052dbe 100644 --- a/packages/sdk/src/appChain/AppChain.ts +++ b/packages/sdk/src/appChain/AppChain.ts @@ -313,9 +313,9 @@ export class AppChain< ) { this.create(() => dependencyContainer); - this.useDependencyFactory(this.container.resolve(AreProofsEnabledFactory)); - this.useDependencyFactory(this.container.resolve(SharedDependencyFactory)); - this.useDependencyFactory(this.container.resolve(ConsoleLoggingFactory)); + this.useDependencyFactory(AreProofsEnabledFactory); + this.useDependencyFactory(SharedDependencyFactory); + this.useDependencyFactory(ConsoleLoggingFactory); this.container .resolve("AreProofsEnabled") diff --git a/packages/sdk/src/appChain/AreProofsEnabledFactory.ts b/packages/sdk/src/appChain/AreProofsEnabledFactory.ts index a22207db0..9492875a1 100644 --- a/packages/sdk/src/appChain/AreProofsEnabledFactory.ts +++ b/packages/sdk/src/appChain/AreProofsEnabledFactory.ts @@ -1,9 +1,5 @@ import { injectable } from "tsyringe"; -import { - AreProofsEnabled, - DependencyFactory, - DependencyRecord, -} from "@proto-kit/common"; +import { AreProofsEnabled, DependencyRecord } from "@proto-kit/common"; @injectable() export class InMemoryAreProofsEnabled implements AreProofsEnabled { @@ -18,8 +14,8 @@ export class InMemoryAreProofsEnabled implements AreProofsEnabled { } } -export class AreProofsEnabledFactory implements DependencyFactory { - public dependencies() { +export class AreProofsEnabledFactory { + public static dependencies() { return { areProofsEnabled: { useClass: InMemoryAreProofsEnabled, diff --git a/packages/sdk/src/appChain/SharedDependencyFactory.ts b/packages/sdk/src/appChain/SharedDependencyFactory.ts index 31bb5c075..fee1eda6a 100644 --- a/packages/sdk/src/appChain/SharedDependencyFactory.ts +++ b/packages/sdk/src/appChain/SharedDependencyFactory.ts @@ -1,8 +1,4 @@ -import { - DependencyDeclaration, - DependencyFactory, - DependencyRecord, -} from "@proto-kit/common"; +import { DependencyDeclaration, DependencyRecord } from "@proto-kit/common"; import { StateServiceProvider } from "@proto-kit/protocol"; import { MethodIdResolver } from "@proto-kit/module"; @@ -11,8 +7,8 @@ export interface SharedDependencyRecord extends DependencyRecord { methodIdResolver: DependencyDeclaration; } -export class SharedDependencyFactory implements DependencyFactory { - public dependencies(): SharedDependencyRecord { +export class SharedDependencyFactory { + public static dependencies(): SharedDependencyRecord { return { stateServiceProvider: { useClass: StateServiceProvider, diff --git a/packages/sequencer/src/logging/ConsoleLoggingFactory.ts b/packages/sequencer/src/logging/ConsoleLoggingFactory.ts index 819d7e679..51678b357 100644 --- a/packages/sequencer/src/logging/ConsoleLoggingFactory.ts +++ b/packages/sequencer/src/logging/ConsoleLoggingFactory.ts @@ -1,11 +1,9 @@ -import { injectable } from "tsyringe"; -import { DependencyFactory, DependencyRecord } from "@proto-kit/common"; +import { DependencyRecord } from "@proto-kit/common"; import { ConsoleTracer } from "./ConsoleTracer"; -@injectable() -export class ConsoleLoggingFactory implements DependencyFactory { - public dependencies() { +export class ConsoleLoggingFactory { + public static dependencies() { return { Tracer: { useClass: ConsoleTracer, diff --git a/packages/sequencer/src/mempool/private/PrivateMempool.ts b/packages/sequencer/src/mempool/private/PrivateMempool.ts index 208ae66e9..c57553c3d 100644 --- a/packages/sequencer/src/mempool/private/PrivateMempool.ts +++ b/packages/sequencer/src/mempool/private/PrivateMempool.ts @@ -37,8 +37,15 @@ type MempoolTransactionPaths = { paths: Field[]; }; +interface PrivateMempoolConfig { + validationEnabled?: boolean; +} + @sequencerModule() -export class PrivateMempool extends SequencerModule implements Mempool { +export class PrivateMempool + extends SequencerModule + implements Mempool +{ public readonly events = new EventEmitter(); private readonly accountStateHook: AccountStateHook; @@ -113,13 +120,16 @@ export class PrivateMempool extends SequencerModule implements Mempool { const networkState = (await this.getStagedNetworkState()) ?? NetworkState.empty(); - const sortedTxs = await this.checkTxValid( - txs, - baseCachedStateService, - this.protocol.stateServiceProvider, - networkState, - limit - ); + const validationEnabled = this.config.validationEnabled ?? true; + const sortedTxs = validationEnabled + ? await this.checkTxValid( + txs, + baseCachedStateService, + this.protocol.stateServiceProvider, + networkState, + limit + ) + : txs; this.protocol.stateServiceProvider.popCurrentStateService(); return sortedTxs; diff --git a/packages/sequencer/src/protocol/baselayer/MinaBaseLayer.ts b/packages/sequencer/src/protocol/baselayer/MinaBaseLayer.ts index 9b99b6a08..a5ebac5a3 100644 --- a/packages/sequencer/src/protocol/baselayer/MinaBaseLayer.ts +++ b/packages/sequencer/src/protocol/baselayer/MinaBaseLayer.ts @@ -1,4 +1,8 @@ -import { AreProofsEnabled, DependencyFactory } from "@proto-kit/common"; +import { + AreProofsEnabled, + DependencyFactory, + DependencyRecord, +} from "@proto-kit/common"; import { Mina } from "o1js"; import { match } from "ts-pattern"; import { inject } from "tsyringe"; @@ -14,6 +18,7 @@ import { Sequencer, SequencerModulesRecord, } from "../../sequencer/executor/Sequencer"; +import { IncomingMessagesService } from "../../settlement/messages/IncomingMessagesService"; import { BaseLayer } from "./BaseLayer"; import { LocalBlockchainUtils } from "./network-utils/LocalBlockchainUtils"; @@ -62,6 +67,14 @@ export class MinaBaseLayer super(); } + public static dependencies() { + return { + IncomingMessagesService: { + useClass: IncomingMessagesService, + }, + } satisfies DependencyRecord; + } + public dependencies() { const NetworkUtilsClass = match(this.config.network.type) .with("local", () => LocalBlockchainUtils) diff --git a/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts b/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts index 18e5fcf8e..10b8da93b 100644 --- a/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts +++ b/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts @@ -113,8 +113,13 @@ async function decodeTransaction( } function extractEvents( - runtimeResult: RuntimeContextReducedExecutionResult -): { eventName: string; data: Field[] }[] { + runtimeResult: RuntimeContextReducedExecutionResult, + source: "afterTxHook" | "beforeTxHook" | "runtime" +): { + eventName: string; + data: Field[]; + source: "afterTxHook" | "beforeTxHook" | "runtime"; +}[] { return runtimeResult.events.reduce( (acc, event) => { if (event.condition.toBoolean()) { @@ -122,13 +127,18 @@ function extractEvents( eventName: event.eventName, // eslint-disable-next-line @typescript-eslint/no-unsafe-argument data: event.eventType.toFields(event.event), + source: source, }; acc.push(obj); } return acc; }, // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - [] as { eventName: string; data: Field[] }[] + [] as { + eventName: string; + data: Field[]; + source: "afterTxHook" | "beforeTxHook" | "runtime"; + }[] ); } @@ -323,6 +333,7 @@ export class TransactionExecutionService { async (hook, hookArgs) => await hook.beforeTransaction(hookArgs), "beforeTx" ); + const beforeHookEvents = extractEvents(beforeTxHookResult, "beforeTxHook"); await recordingStateService.applyStateTransitions( beforeTxHookResult.stateTransitions @@ -372,6 +383,7 @@ export class TransactionExecutionService { async (hook, hookArgs) => await hook.afterTransaction(hookArgs), "afterTx" ); + const afterHookEvents = extractEvents(afterTxHookResult, "afterTxHook"); await recordingStateService.applyStateTransitions( afterTxHookResult.stateTransitions ); @@ -385,7 +397,7 @@ export class TransactionExecutionService { appChain.setProofsEnabled(previousProofsEnabled); // Extract sequencing results - const events = extractEvents(runtimeResult); + const runtimeResultEvents = extractEvents(runtimeResult, "runtime"); const stateTransitions = this.buildSTBatches( [ beforeTxHookResult.stateTransitions, @@ -403,7 +415,7 @@ export class TransactionExecutionService { statusMessage: runtimeResult.statusMessage, stateTransitions, - events, + events: beforeHookEvents.concat(runtimeResultEvents, afterHookEvents), }, ]; } diff --git a/packages/sequencer/src/sequencer/executor/Sequencer.ts b/packages/sequencer/src/sequencer/executor/Sequencer.ts index bcd1b7ef7..88e87b99e 100644 --- a/packages/sequencer/src/sequencer/executor/Sequencer.ts +++ b/packages/sequencer/src/sequencer/executor/Sequencer.ts @@ -75,7 +75,7 @@ export class Sequencer // ensure that we start modules based on the order they were resolved. // We iterate through the methods three times: - this.useDependencyFactory(this.container.resolve(MethodIdFactory)); + this.useDependencyFactory(MethodIdFactory); // Log startup info const moduleClassNames = Object.values(this.definition.modules).map( diff --git a/packages/sequencer/src/settlement/SettlementModule.ts b/packages/sequencer/src/settlement/SettlementModule.ts index 38992db3f..7e2b44a7e 100644 --- a/packages/sequencer/src/settlement/SettlementModule.ts +++ b/packages/sequencer/src/settlement/SettlementModule.ts @@ -46,7 +46,6 @@ import { ProvenSettlementPermissions } from "./permissions/ProvenSettlementPermi import { SignedSettlementPermissions } from "./permissions/SignedSettlementPermissions"; import { SettlementUtils } from "./utils/SettlementUtils"; import { BridgingModule } from "./BridgingModule"; -import { IncomingMessagesService } from "./messages/IncomingMessagesService"; export interface SettlementModuleConfig { feepayer: PrivateKey; @@ -105,9 +104,6 @@ export class SettlementModule BridgingModule: { useClass: BridgingModule, }, - IncomingMessagesService: { - useClass: IncomingMessagesService, - }, }; } diff --git a/packages/sequencer/src/storage/model/Block.ts b/packages/sequencer/src/storage/model/Block.ts index 8f1da00a4..823b3c8c6 100644 --- a/packages/sequencer/src/storage/model/Block.ts +++ b/packages/sequencer/src/storage/model/Block.ts @@ -20,7 +20,11 @@ export interface TransactionExecutionResult { stateTransitions: StateTransitionBatch[]; status: Bool; statusMessage?: string; - events: { eventName: string; data: Field[] }[]; + events: { + eventName: string; + data: Field[]; + source: "afterTxHook" | "beforeTxHook" | "runtime"; + }[]; } // TODO Why is Block using Fields, but BlockResult bigints? Align that towards the best option diff --git a/packages/sequencer/test/TestingSequencer.ts b/packages/sequencer/test/TestingSequencer.ts index 9be30dd52..41715c40a 100644 --- a/packages/sequencer/test/TestingSequencer.ts +++ b/packages/sequencer/test/TestingSequencer.ts @@ -8,7 +8,6 @@ import { ManualBlockTrigger, NoopBaseLayer, PrivateMempool, - Sequencer, SequencerModulesRecord, TaskWorkerModulesRecord, BlockProducerModule, @@ -30,37 +29,36 @@ export interface DefaultTestingSequencerModules extends SequencerModulesRecord { SequencerStartupModule: typeof SequencerStartupModule; } -export function testingSequencerFromModules< +export function testingSequencerModules< AdditionalModules extends SequencerModulesRecord, AdditionalTaskWorkerModules extends TaskWorkerModulesRecord, >( modules: AdditionalModules, additionalTaskWorkerModules?: AdditionalTaskWorkerModules -): TypedClass> { +) { const taskWorkerModule = LocalTaskWorkerModule.from({ ...VanillaTaskWorkerModules.withoutSettlement(), ...additionalTaskWorkerModules, }); - const defaultModules: DefaultTestingSequencerModules = { + const defaultModules = { Database: InMemoryDatabase, Mempool: PrivateMempool, BaseLayer: NoopBaseLayer, - // LocalTaskWorkerModule: taskWorkerModule, + LocalTaskWorkerModule: taskWorkerModule, BatchProducerModule, BlockProducerModule, BlockTrigger: ManualBlockTrigger, TaskQueue: LocalTaskQueue, FeeStrategy: ConstantFeeStrategy, - } as DefaultTestingSequencerModules; + SequencerStartupModule, + } satisfies DefaultTestingSequencerModules; - return Sequencer.from({ - modules: { - ...defaultModules, - ...modules, - // We need to make sure that the taskworkermodule is initialized last - LocalTaskWorkerModule: taskWorkerModule, - SequencerStartupModule, - }, - }); + return { + ...defaultModules, + ...modules, + // We need to make sure that the taskworkermodule is initialized last + LocalTaskWorkerModule: defaultModules.LocalTaskWorkerModule, + SequencerStartupModule: defaultModules.SequencerStartupModule, + } satisfies SequencerModulesRecord; } diff --git a/packages/sequencer/test/integration/BlockProduction.test.ts b/packages/sequencer/test/integration/BlockProduction.test.ts index 644427239..049b5d223 100644 --- a/packages/sequencer/test/integration/BlockProduction.test.ts +++ b/packages/sequencer/test/integration/BlockProduction.test.ts @@ -33,7 +33,7 @@ import { } from "../../src"; import { DefaultTestingSequencerModules, - testingSequencerFromModules, + testingSequencerModules, } from "../TestingSequencer"; import { Balance } from "./mocks/Balance"; @@ -111,7 +111,9 @@ describe("block production", () => { }, }); - const sequencerClass = testingSequencerFromModules({}); + const sequencerClass = Sequencer.from({ + modules: testingSequencerModules({}), + }); // TODO Analyze how we can get rid of the library import for mandatory modules const protocolClass = Protocol.from({ @@ -684,11 +686,13 @@ describe("block production", () => { const firstEventReduced = { eventName: firstExpectedEvent.eventName, data: firstExpectedEvent.eventType.toFields(firstExpectedEvent.event), + source: "runtime", }; const secondEventReduced = { eventName: secondExpectedEvent.eventName, data: secondExpectedEvent.eventType.toFields(secondExpectedEvent.event), + source: "runtime", }; const block = await test.produceBlock(); diff --git a/packages/sequencer/test/integration/BlockProductionSize.test.ts b/packages/sequencer/test/integration/BlockProductionSize.test.ts index 13a213038..14ed46a1b 100644 --- a/packages/sequencer/test/integration/BlockProductionSize.test.ts +++ b/packages/sequencer/test/integration/BlockProductionSize.test.ts @@ -15,7 +15,7 @@ import { } from "../../src"; import { DefaultTestingSequencerModules, - testingSequencerFromModules, + testingSequencerModules, } from "../TestingSequencer"; import { Balance } from "./mocks/Balance"; @@ -56,7 +56,9 @@ describe("block limit", () => { }); async function setUpAppChain(maxBlockSize: number | undefined) { - const sequencerClass = testingSequencerFromModules({}); + const sequencerClass = Sequencer.from({ + modules: testingSequencerModules({}), + }); const protocolClass = Protocol.from({ modules: VanillaProtocolModules.mandatoryModules({ @@ -85,7 +87,6 @@ describe("block limit", () => { BaseLayer: {}, TaskQueue: {}, FeeStrategy: {}, - ProtocolStartupModule: {}, SequencerStartupModule: {}, }, Runtime: { diff --git a/packages/sequencer/test/integration/Mempool.test.ts b/packages/sequencer/test/integration/Mempool.test.ts index 2d13305c3..a9b58207b 100644 --- a/packages/sequencer/test/integration/Mempool.test.ts +++ b/packages/sequencer/test/integration/Mempool.test.ts @@ -17,7 +17,7 @@ import { } from "../../src"; import { DefaultTestingSequencerModules, - testingSequencerFromModules, + testingSequencerModules, } from "../TestingSequencer"; import { Balance } from "./mocks/Balance"; @@ -86,7 +86,9 @@ describe.each([["InMemory", InMemoryDatabase]])( }, }); - const sequencerClass = testingSequencerFromModules({}); + const sequencerClass = Sequencer.from({ + modules: testingSequencerModules({}), + }); const protocolClass = Protocol.from({ modules: VanillaProtocolModules.mandatoryModules({}), diff --git a/packages/sequencer/test/integration/Proven.test.ts b/packages/sequencer/test/integration/Proven.test.ts index e456dee9d..1ba469b83 100644 --- a/packages/sequencer/test/integration/Proven.test.ts +++ b/packages/sequencer/test/integration/Proven.test.ts @@ -20,10 +20,11 @@ import { AppChain, InMemoryAreProofsEnabled } from "@proto-kit/sdk"; import { container } from "tsyringe"; import { PrivateKey, UInt64 } from "o1js"; -import { testingSequencerFromModules } from "../TestingSequencer"; +import { testingSequencerModules } from "../TestingSequencer"; import { MinaBaseLayer, ProvenSettlementPermissions, + Sequencer, SettlementModule, SettlementProvingTask, VanillaTaskWorkerModules, @@ -56,16 +57,18 @@ describe.skip("Proven", () => { }, }); - const sequencerClass = testingSequencerFromModules( - { - BaseLayer: MinaBaseLayer, - SettlementModule, - OutgoingMessageQueue: WithdrawalQueue, - }, - { - SettlementProvingTask, - } - ); + const sequencerClass = Sequencer.from({ + modules: testingSequencerModules( + { + BaseLayer: MinaBaseLayer, + SettlementModule, + OutgoingMessageQueue: WithdrawalQueue, + }, + { + SettlementProvingTask, + } + ), + }); // TODO Analyze how we can get rid of the library import for mandatory modules const protocolClass = Protocol.from({ @@ -105,7 +108,10 @@ describe.skip("Proven", () => { type: "local", }, }, - SettlementModule: {}, + SettlementModule: { + // TODO + feepayer: PrivateKey.random(), + }, OutgoingMessageQueue: {}, }, Runtime: { diff --git a/packages/sequencer/test/integration/StorageIntegration.test.ts b/packages/sequencer/test/integration/StorageIntegration.test.ts index 9fa4543b6..88213a936 100644 --- a/packages/sequencer/test/integration/StorageIntegration.test.ts +++ b/packages/sequencer/test/integration/StorageIntegration.test.ts @@ -23,7 +23,7 @@ import { } from "../../src"; import { DefaultTestingSequencerModules, - testingSequencerFromModules, + testingSequencerModules, } from "../TestingSequencer"; import { collectStateDiff, createTransaction } from "./utils"; @@ -74,8 +74,10 @@ describe.each([["InMemory", InMemoryDatabase]])( let pkNonce = 0; beforeAll(async () => { - const sequencerClass = testingSequencerFromModules({ - Database, + const sequencerClass = Sequencer.from({ + modules: testingSequencerModules({ + Database, + }), }); const runtimeClass = Runtime.from({ diff --git a/packages/sequencer/test/production/tracing/StateTransitionTracingService.test.ts b/packages/sequencer/test/production/tracing/StateTransitionTracingService.test.ts index 37932aea0..e82d845d3 100644 --- a/packages/sequencer/test/production/tracing/StateTransitionTracingService.test.ts +++ b/packages/sequencer/test/production/tracing/StateTransitionTracingService.test.ts @@ -19,6 +19,7 @@ import { StateTransitionTracingService, TracingStateTransitionBatch, StateTransitionProofParameters, + ConsoleTracer, } from "../../../src"; function createST(obj: { @@ -114,7 +115,7 @@ async function applyBatchesToTree( // return sequencer; // } -const service = new StateTransitionTracingService(); +const service = new StateTransitionTracingService(new ConsoleTracer()); describe("StateTransitionTracingService", () => { const cases: { diff --git a/packages/sequencer/test/protocol/production/sequencing/atomic-block-production.test.ts b/packages/sequencer/test/protocol/production/sequencing/atomic-block-production.test.ts index 3d6fadcf8..22a12bd41 100644 --- a/packages/sequencer/test/protocol/production/sequencing/atomic-block-production.test.ts +++ b/packages/sequencer/test/protocol/production/sequencing/atomic-block-production.test.ts @@ -10,12 +10,13 @@ import { expectDefined } from "@proto-kit/common"; import { BlockQueue, ManualBlockTrigger, + Sequencer, VanillaTaskWorkerModules, } from "../../../../src"; import { ProtocolStateTestHook } from "../../../integration/mocks/ProtocolStateTestHook"; import { DefaultTestingSequencerModules, - testingSequencerFromModules, + testingSequencerModules, } from "../../../TestingSequencer"; import { Balance } from "../../../integration/mocks/Balance"; import { BlockResultService } from "../../../../src/protocol/production/sequencing/BlockResultService"; @@ -36,7 +37,9 @@ describe("atomic block production", () => { }, }); - const sequencerClass = testingSequencerFromModules({}); + const sequencerClass = Sequencer.from({ + modules: testingSequencerModules({}), + }); const protocolClass = Protocol.from({ modules: VanillaProtocolModules.mandatoryModules({ @@ -62,7 +65,6 @@ describe("atomic block production", () => { BaseLayer: {}, TaskQueue: {}, FeeStrategy: {}, - ProtocolStartupModule: {}, SequencerStartupModule: {}, }, Runtime: { diff --git a/packages/sequencer/test/settlement/Settlement.ts b/packages/sequencer/test/settlement/Settlement.ts index 17a3966a0..4fcda0d93 100644 --- a/packages/sequencer/test/settlement/Settlement.ts +++ b/packages/sequencer/test/settlement/Settlement.ts @@ -54,9 +54,10 @@ import { SignedSettlementPermissions, ProvenSettlementPermissions, VanillaTaskWorkerModules, + Sequencer, } from "../../src"; import { BlockProofSerializer } from "../../src/protocol/production/tasks/serializers/BlockProofSerializer"; -import { testingSequencerFromModules } from "../TestingSequencer"; +import { testingSequencerModules } from "../TestingSequencer"; import { createTransaction } from "../integration/utils"; import { FeeStrategy } from "../../src/protocol/baselayer/fees/FeeStrategy"; import { BridgingModule } from "../../src/settlement/BridgingModule"; @@ -119,16 +120,18 @@ export const settlementTestFn = ( SettlementUtils.prototype["isSignedSettlement"] = () => settlementType === "signed"; - const sequencer = testingSequencerFromModules( - { - BaseLayer: MinaBaseLayer, - SettlementModule: SettlementModule, - OutgoingMessageQueue: WithdrawalQueue, - }, - { - SettlementProvingTask, - } - ); + const sequencer = Sequencer.from({ + modules: testingSequencerModules( + { + BaseLayer: MinaBaseLayer, + SettlementModule: SettlementModule, + OutgoingMessageQueue: WithdrawalQueue, + }, + { + SettlementProvingTask, + } + ), + }); const appchain = AppChain.from({ Runtime: runtime, diff --git a/packages/sequencer/test/worker/Flow.test.ts b/packages/sequencer/test/worker/Flow.test.ts index a991aca65..20f449b21 100644 --- a/packages/sequencer/test/worker/Flow.test.ts +++ b/packages/sequencer/test/worker/Flow.test.ts @@ -246,8 +246,6 @@ describe("flow", () => { const resolveReduction = async () => { let reductions = flow.state.reductionQueue; - console.log("Length:", reductions.length); - if (reductions.length === 1 && flow.tasksInProgress === 0) { resolve(reductions[0]); } @@ -276,7 +274,7 @@ describe("flow", () => { const [first, second] = flow.state.pairings[index]; if (first !== undefined && second !== undefined) { - console.log(`Found pairing ${index}`); + log.trace(`Found pairing ${index}`); await flow.pushTask( mulTask, @@ -314,7 +312,7 @@ describe("flow", () => { }); }); - console.log(computedResult); + log.trace(computedResult); expect(computedResult).toStrictEqual(result); }, diff --git a/packages/stack/test/graphql/graphql-server.ts b/packages/stack/test/graphql/graphql-server.ts new file mode 100644 index 000000000..4305ab93b --- /dev/null +++ b/packages/stack/test/graphql/graphql-server.ts @@ -0,0 +1,272 @@ +import { + AppChain, + BlockStorageNetworkStateModule, + InMemorySigner, + InMemoryTransactionSender, + StateServiceQueryModule, +} from "@proto-kit/sdk"; +import { PrivateKey, PublicKey } from "o1js"; +import { Runtime, runtimeMethod, runtimeModule } from "@proto-kit/module"; +import { Protocol, State, state } from "@proto-kit/protocol"; +import { + Balance, + Balances, + BalancesKey, + TokenId, + VanillaProtocolModules, + VanillaRuntimeModules, + UInt64, +} from "@proto-kit/library"; +import { log } from "@proto-kit/common"; +import { + BatchProducerModule, + InMemoryDatabase, + LocalTaskQueue, + LocalTaskWorkerModule, + NoopBaseLayer, + PrivateMempool, + Sequencer, + BlockProducerModule, + VanillaTaskWorkerModules, + SequencerStartupModule, + ManualBlockTrigger, +} from "@proto-kit/sequencer"; +import { + BatchStorageResolver, + GraphqlSequencerModule, + GraphqlServer, + MempoolResolver, + MerkleWitnessResolver, + NodeStatusResolver, + QueryGraphqlModule, + BlockResolver, +} from "@proto-kit/api"; +import { container } from "tsyringe"; + +@runtimeModule() +export class TestBalances extends Balances { + /** + * We use `satisfies` here in order to be able to access + * presets by key in a type safe way. + */ + // public static presets = {} satisfies Presets; + + @state() public totalSupply = State.from(UInt64); + + @runtimeMethod() + public async getBalanceForUser( + tokenId: TokenId, + address: PublicKey + ): Promise { + return await super.getBalance(tokenId, address); + } + + @runtimeMethod() + public async addBalance( + tokenId: TokenId, + address: PublicKey, + balance: UInt64 + ) { + const totalSupply = await this.totalSupply.get(); + await this.totalSupply.set(totalSupply.orElse(UInt64.zero).add(balance)); + + const previous = await this.balances.get( + new BalancesKey({ tokenId, address }) + ); + await this.balances.set( + new BalancesKey({ tokenId, address }), + previous.orElse(UInt64.zero).add(balance) + ); + } +} + +export async function startGraphqlServer() { + log.setLevel("DEBUG"); + + const appChain = AppChain.from({ + Runtime: Runtime.from({ + modules: VanillaRuntimeModules.with({ + Balances: TestBalances, + }), + }), + + Protocol: Protocol.from({ + modules: VanillaProtocolModules.with({}), + }), + + Sequencer: Sequencer.from({ + modules: { + Database: InMemoryDatabase, + // Database: PrismaRedisDatabase, + + Mempool: PrivateMempool, + GraphqlServer, + LocalTaskWorkerModule: LocalTaskWorkerModule.from( + VanillaTaskWorkerModules.withoutSettlement() + ), + + BaseLayer: NoopBaseLayer, + BatchProducerModule, + BlockProducerModule, + BlockTrigger: ManualBlockTrigger, + TaskQueue: LocalTaskQueue, + // SettlementModule: SettlementModule, + + Graphql: GraphqlSequencerModule.from({ + modules: { + MempoolResolver, + QueryGraphqlModule, + BatchStorageResolver, + BlockResolver, + NodeStatusResolver, + MerkleWitnessResolver, + }, + + config: { + MempoolResolver: {}, + QueryGraphqlModule: {}, + BatchStorageResolver: {}, + NodeStatusResolver: {}, + MerkleWitnessResolver: {}, + BlockResolver: {}, + }, + }), + + SequencerStartupModule, + }, + }), + + modules: { + Signer: InMemorySigner, + TransactionSender: InMemoryTransactionSender, + QueryTransportModule: StateServiceQueryModule, + NetworkStateTransportModule: BlockStorageNetworkStateModule, + }, + }); + + appChain.configure({ + Runtime: { + Balances: {}, + }, + + Protocol: { + BlockProver: {}, + StateTransitionProver: {}, + AccountState: {}, + BlockHeight: {}, + TransactionFee: { + tokenId: 0n, + feeRecipient: PrivateKey.random().toPublicKey().toBase58(), + baseFee: 0n, + methods: {}, + perWeightUnitFee: 0n, + }, + LastStateRoot: {}, + }, + + Sequencer: { + GraphqlServer: { + port: 8080, + host: "0.0.0.0", + graphiql: true, + }, + SequencerStartupModule: {}, + + // SettlementModule: { + // address: PrivateKey.random().toPublicKey(), + // feepayer: PrivateKey.random(), + // }, + + Graphql: { + QueryGraphqlModule: {}, + MempoolResolver: {}, + BatchStorageResolver: {}, + NodeStatusResolver: {}, + BlockResolver: {}, + MerkleWitnessResolver: {}, + }, + + Database: { + // redis: { + // host: "localhost", + // port: 6379, + // password: "password", + // }, + // prisma: { + // connection: { + // host: "localhost", + // password: "password", + // username: "user", + // port: 5432, + // db: { + // name: "protokit", + // }, + // }, + // }, + }, + + Mempool: {}, + BatchProducerModule: {}, + LocalTaskWorkerModule: VanillaTaskWorkerModules.defaultConfig(), + BaseLayer: {}, + TaskQueue: {}, + + BlockProducerModule: { + allowEmptyBlock: true, + }, + + BlockTrigger: {}, + }, + + TransactionSender: {}, + QueryTransportModule: {}, + NetworkStateTransportModule: {}, + + Signer: { + signer: PrivateKey.random(), + }, + }); + + await appChain.start(false, container.createChildContainer()); + // const pk = PublicKey.fromBase58( + // "B62qmETai5Y8vvrmWSU8F4NX7pTyPqYLMhc1pgX3wD8dGc2wbCWUcqP" + // ); + + const balances = appChain.runtime.resolve("Balances"); + + const priv = PrivateKey.fromBase58( + "EKFEMDTUV2VJwcGmCwNKde3iE1cbu7MHhzBqTmBtGAd6PdsLTifY" + ); + + const tokenId = TokenId.from(0); + + const as = await appChain.query.protocol.AccountState.accountState.get( + priv.toPublicKey() + ); + const nonce = Number(as?.nonce.toString() ?? "0"); + + const tx = await appChain.transaction( + priv.toPublicKey(), + async () => { + await balances.addBalance(tokenId, priv.toPublicKey(), UInt64.from(1000)); + }, + { + nonce, + } + ); + appChain.resolve("Signer").config.signer = priv; + await tx.sign(); + await tx.send(); + + const tx2 = await appChain.transaction( + priv.toPublicKey(), + async () => { + await balances.addBalance(tokenId, priv.toPublicKey(), UInt64.from(1000)); + }, + { nonce: nonce + 1 } + ); + await tx2.sign(); + await tx2.send(); + + return appChain; +} diff --git a/packages/stack/test/graphql/graphql.test.ts b/packages/stack/test/graphql/graphql.test.ts index d5d5c4ae1..007c045d3 100644 --- a/packages/stack/test/graphql/graphql.test.ts +++ b/packages/stack/test/graphql/graphql.test.ts @@ -22,7 +22,7 @@ import { import { beforeAll } from "@jest/globals"; import { container } from "tsyringe"; -import { startServer, TestBalances } from "../../src/scripts/graphql/server"; +import { startGraphqlServer, TestBalances } from "./graphql-server"; const pk = PrivateKey.random(); @@ -93,12 +93,12 @@ function prepareClient() { describe("graphql client test", () => { let appChain: ReturnType; - let server: Awaited>; + let server: Awaited>; let trigger: ManualBlockTrigger; const tokenId = TokenId.from(0); beforeAll(async () => { - server = await startServer(); + server = await startGraphqlServer(); await sleep(2000); diff --git a/packages/stack/test/start.test.ts b/packages/stack/test/start.test.ts index 49104172a..262291c4b 100644 --- a/packages/stack/test/start.test.ts +++ b/packages/stack/test/start.test.ts @@ -2,7 +2,7 @@ import { sleep } from "@proto-kit/common"; import { startServer } from "../src"; -describe("Start", () => { +describe.skip("Start", () => { it("a", async () => { await startServer(); await sleep(10000000);