diff --git a/.github/workflows/pull-request-develop.yml b/.github/workflows/pull-request-develop.yml index 43f2395af..c46ea8de7 100644 --- a/.github/workflows/pull-request-develop.yml +++ b/.github/workflows/pull-request-develop.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 22.9.0 # This only caches ~/.npm, therefore each subsequent # job needs to run `npm ci` to install deps from npm cache # alternative is to cache `node_modules` directly @@ -39,7 +39,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 22.9.0 cache: npm - name: "Install dependencies" @@ -57,7 +57,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 22.9.0 cache: npm - name: "Install dependencies" @@ -80,7 +80,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 22.9.0 cache: npm - name: "Install dependencies" @@ -135,7 +135,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 22.9.0 cache: npm - name: "Install dependencies" @@ -145,7 +145,7 @@ jobs: run: npm run build - name: "Migrate DB" - run: npm run migrate + run: npm run prisma:migrate - name: "Wait for lightnet readiness" uses: o1-labs/wait-for-mina-network-action@v1 diff --git a/nx.json b/nx.json index fa54d7ed7..816559d48 100644 --- a/nx.json +++ b/nx.json @@ -5,6 +5,7 @@ "options": { "cacheableOperations": [ "build", + "prisma:generate", "lint", "test", "integration" @@ -13,6 +14,12 @@ } }, "targetDefaults": { + "prisma:generate": { + "inputs": [ + "{projectRoot}/package.json", + "{projectRoot}/prisma/**/*" + ] + }, "build": { "dependsOn": [ "^build" diff --git a/package.json b/package.json index e36b5474b..9de23711f 100644 --- a/package.json +++ b/package.json @@ -5,17 +5,17 @@ "type": "module", "scripts": { "dev": "npx lerna run dev", - "build": "npx lerna run build", + "build": "npm run prisma:generate && lerna run build", "build:clean": "npx nx reset && npm run build", "build-docker": "docker build -t protokit-base -f ./packages/deployment/docker/base/Dockerfile .", "lint": "npx lerna run lint", "lint:staged": "eslint", "test": "npx lerna run test -- --passWithNoTests", - "test:ci": "npx lerna run test -- --passWithNoTests --forceExit", - "test:integration": "npx lerna run test:integration -- --passWithNoTests --forceExit", + "test:ci": "npx lerna run test --concurrency=1 -- --passWithNoTests --forceExit", + "test:integration": "npx lerna run test:integration --concurrency=1 -- --passWithNoTests --forceExit", "test:watch": "npx lerna run test:watch", - "migrate": "npx lerna run prisma-migrate", - "prisma-generate": "npx lerna run prisma-generate", + "prisma:migrate": "npx lerna run prisma:migrate", + "prisma:generate": "npx lerna run prisma:generate", "commit": "cz", "publish:canary": "npx lerna publish prerelease --no-private --exact --yes --canary --preid develop --dist-tag latest --loglevel verbose --force-git-tag --force-publish" }, diff --git a/packages/api/package.json b/packages/api/package.json index fc373f9be..5189e399b 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -8,7 +8,7 @@ "build": "tsc -p tsconfig.json", "dev": "tsc -p tsconfig.json --watch", "lint": "eslint ./src ./test", - "test:file": "node --experimental-vm-modules --experimental-wasm-modules --experimental-wasm-threads ../../node_modules/jest/bin/jest.js", + "test:file": "node --experimental-vm-modules --experimental-wasm-modules ../../node_modules/jest/bin/jest.js", "test": "npm run test:file -- ./test/**", "test:watch": "npm run test:file -- ./test/** --watch" }, diff --git a/packages/common/package.json b/packages/common/package.json index 45ccfb983..a5c8e1d79 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -8,7 +8,7 @@ "build": "tsc -p tsconfig.json", "dev": "tsc -p tsconfig.json --watch", "lint": "eslint ./src ./test", - "test:file": "node --experimental-vm-modules --experimental-wasm-modules --experimental-wasm-threads ../../node_modules/jest/bin/jest.js", + "test:file": "node --experimental-vm-modules --experimental-wasm-modules ../../node_modules/jest/bin/jest.js", "test": "npm run test:file -- ./test/**", "test:watch": "npm run test:file -- ./test/** --watch" }, diff --git a/packages/common/test/zkProgrammable/ZkProgrammable.test.ts b/packages/common/test/zkProgrammable/ZkProgrammable.test.ts index 4f606954b..080d799c5 100644 --- a/packages/common/test/zkProgrammable/ZkProgrammable.test.ts +++ b/packages/common/test/zkProgrammable/ZkProgrammable.test.ts @@ -6,14 +6,12 @@ import { Field, Struct, Proof, ZkProgram } from "o1js"; import { MOCK_PROOF, provableMethod, -} from "../../src/zkProgrammable/provableMethod"; -import { AreProofsEnabled, CompileArtifact, MOCK_VERIFICATION_KEY, ZkProgrammable, -} from "../../src/zkProgrammable/ZkProgrammable"; -import { ProvableMethodExecutionContext } from "../../src/zkProgrammable/ProvableMethodExecutionContext"; + ProvableMethodExecutionContext, +} from "../../src"; const appChainMock: AreProofsEnabled = { areProofsEnabled: false, @@ -86,6 +84,7 @@ class TestProgrammable extends ZkProgrammable< return [ { + name: program.name, compile: program.compile.bind(program), verify: program.verify.bind(program), analyzeMethods: program.analyzeMethods.bind(program), @@ -129,6 +128,7 @@ class OtherTestProgrammable extends ZkProgrammable { return [ { + name: program.name, compile: program.compile.bind(program), verify: program.verify.bind(program), analyzeMethods: program.analyzeMethods.bind(program), diff --git a/packages/deployment/package.json b/packages/deployment/package.json index fcd7c7b08..3c9f90037 100644 --- a/packages/deployment/package.json +++ b/packages/deployment/package.json @@ -8,7 +8,7 @@ "build": "tsc -p tsconfig.json", "dev": "tsc -p tsconfig.json --watch", "lint": "eslint ./src ./test", - "test:file": "node --experimental-vm-modules --experimental-wasm-modules --experimental-wasm-threads ../../node_modules/jest/bin/jest.js", + "test:file": "node --experimental-vm-modules --experimental-wasm-modules ../../node_modules/jest/bin/jest.js", "test": "npm run test:file -- ./test/**", "test:watch": "npm run test:file -- ./test/** --watch" }, diff --git a/packages/indexer/.env b/packages/indexer/.env index c947c0467..5e15b2af6 100644 --- a/packages/indexer/.env +++ b/packages/indexer/.env @@ -1 +1 @@ -DATABASE_URL=postgresql://admin:password@localhost:5432/protokit?schema=public \ No newline at end of file +INDEXER_DATABASE_URL=postgresql://admin:password@localhost:5432/protokit-indexer?schema=public \ No newline at end of file diff --git a/packages/indexer/package.json b/packages/indexer/package.json index 33dc2fb29..76d942113 100644 --- a/packages/indexer/package.json +++ b/packages/indexer/package.json @@ -9,7 +9,7 @@ "build": "tsc -p tsconfig.json", "dev": "tsc -p tsconfig.json --watch", "lint": "eslint ./src ./test", - "test:file": "node --experimental-vm-modules --experimental-wasm-modules --experimental-wasm-threads ../../node_modules/jest/bin/jest.js", + "test:file": "node --experimental-vm-modules --experimental-wasm-modules ../../node_modules/jest/bin/jest.js", "test": "npm run test:file -- ./src/** ./test/**", "test:watch": "npm run test:file -- ./src/** ./test/** --watch", "prisma:generate": "prisma generate --schema=./prisma/schema.prisma", diff --git a/packages/indexer/test/IndexerNotifier.test.ts b/packages/indexer/test/IndexerNotifier.test.ts index e59d27710..1e595e5b0 100644 --- a/packages/indexer/test/IndexerNotifier.test.ts +++ b/packages/indexer/test/IndexerNotifier.test.ts @@ -20,10 +20,14 @@ import { StateServiceQueryModule, TestingAppChain, } from "@proto-kit/sdk"; -import { LocalTaskQueue, Sequencer, TaskPayload } from "@proto-kit/sequencer"; +import { + LocalTaskQueue, + Sequencer, + TaskPayload, + VanillaTaskWorkerModules, +} from "@proto-kit/sequencer"; -import { IndexerNotifier } from "../src/IndexerNotifier"; -import { IndexBlockTaskParametersSerializer } from "../src/tasks/IndexBlockTaskParameters"; +import { IndexerNotifier, IndexBlockTaskParametersSerializer } from "../src"; class TestBalances extends Balances { @runtimeMethod() @@ -90,22 +94,15 @@ function createAppChain() { BlockTrigger: {}, Mempool: {}, BlockProducerModule: {}, - LocalTaskWorkerModule: { - StateTransitionReductionTask: {}, - StateTransitionTask: {}, - RuntimeProvingTask: {}, - BlockBuildingTask: {}, - BlockProvingTask: {}, - BlockReductionTask: {}, - CircuitCompilerTask: {}, - WorkerRegistrationTask: {}, - }, + LocalTaskWorkerModule: VanillaTaskWorkerModules.defaultConfig(), BaseLayer: {}, BatchProducerModule: {}, TaskQueue: { simulatedDuration: 0, }, + SequencerStartupModule: {}, IndexerNotifier: {}, + FeeStrategy: {}, }, Signer: { signer: PrivateKey.random(), @@ -154,18 +151,24 @@ async function sendTransactions( return await appChain.produceBlock(); } -describe("IndexerNotifier", () => { +// TODO This test currently doesn't work because the mock stops the queues +// from working as it intercepts calls. This is important both for the +// sequencer startup and also the block production +describe.skip("IndexerNotifier", () => { let appChain: ReturnType; const getQueueSpy = jest.spyOn(LocalTaskQueue.prototype, "getQueue"); - const addTaskSpy = jest.fn(async (payload: TaskPayload) => ({ - taskId: "0", - })); + const addTaskSpy = jest.fn(async (payload: TaskPayload) => { + return { + taskId: "0", + }; + }); getQueueSpy.mockImplementation(async (queueName: string) => { return { name: queueName, + offCompleted: jest.fn(async (x) => {}), addTask: addTaskSpy, - onCompleted: jest.fn(async () => {}), + onCompleted: jest.fn(async () => 5), close: jest.fn(async () => {}), }; }); @@ -173,9 +176,10 @@ describe("IndexerNotifier", () => { beforeAll(async () => { appChain = createAppChain(); - await appChain.start(); + await appChain.start(false, container.createChildContainer()); + await sendTransactions(appChain, 2); - }); + }, 20000); it("should create a task for every unproven block produced", async () => { const { block } = container diff --git a/packages/library/package.json b/packages/library/package.json index c11779527..479e7c5db 100644 --- a/packages/library/package.json +++ b/packages/library/package.json @@ -8,7 +8,7 @@ "build": "tsc -p tsconfig.json", "dev": "tsc -p tsconfig.json --watch", "lint": "eslint ./src ./test", - "test:file": "node --experimental-vm-modules --experimental-wasm-modules --experimental-wasm-threads ../../node_modules/jest/bin/jest.js", + "test:file": "node --experimental-vm-modules --experimental-wasm-modules ../../node_modules/jest/bin/jest.js", "test": "npm run test:file -- ./test/**", "test:watch": "npm run test:file -- ./test/** --watch" }, diff --git a/packages/library/src/sequencer/InMemorySequencerModules.ts b/packages/library/src/sequencer/InMemorySequencerModules.ts index 17f05d2cd..428f86391 100644 --- a/packages/library/src/sequencer/InMemorySequencerModules.ts +++ b/packages/library/src/sequencer/InMemorySequencerModules.ts @@ -10,8 +10,10 @@ import { BlockProducerModule, VanillaTaskWorkerModules, TaskWorkerModulesWithoutSettlement, + SequencerStartupModule, } from "@proto-kit/sequencer"; import { TypedClass } from "@proto-kit/common"; +import { ConstantFeeStrategy } from "@proto-kit/sequencer/src/protocol/baselayer/fees/ConstantFeeStrategy"; export type InMemorySequencerModulesRecord = { Database: typeof InMemoryDatabase; @@ -38,12 +40,14 @@ export class InMemorySequencerModules { LocalTaskWorkerModule: LocalTaskWorkerModule.from({ ...VanillaTaskWorkerModules.withoutSettlement(), }), + FeeStrategy: ConstantFeeStrategy, BaseLayer: NoopBaseLayer, BatchProducerModule, BlockProducerModule, BlockTrigger: ManualBlockTrigger, TaskQueue: LocalTaskQueue, // SettlementModule: SettlementModule, + SequencerStartupModule: SequencerStartupModule, ...additionalModules, } satisfies InMemorySequencerModulesRecord; } diff --git a/packages/module/package.json b/packages/module/package.json index 1b82d4ac6..548f23d24 100644 --- a/packages/module/package.json +++ b/packages/module/package.json @@ -8,7 +8,7 @@ "build": "tsc -p tsconfig.json", "dev": "tsc -p tsconfig.json --watch", "lint": "eslint ./src ./test", - "test:file": "node --experimental-vm-modules --experimental-wasm-modules --experimental-wasm-threads ../../node_modules/jest/bin/jest.js", + "test:file": "node --experimental-vm-modules --experimental-wasm-modules ../../node_modules/jest/bin/jest.js", "test": "npm run test:file -- ./test/**", "test:watch": "npm run test:file -- ./test/** --watch" }, diff --git a/packages/persistance/package.json b/packages/persistance/package.json index a58550706..1d2e7d640 100644 --- a/packages/persistance/package.json +++ b/packages/persistance/package.json @@ -5,12 +5,12 @@ "type": "module", "version": "0.1.1-develop.833+397881ed", "scripts": { - "build": "npm run prisma-generate && tsc -p tsconfig.json", - "prisma-generate": "npx prisma generate", - "prisma-migrate": "npx prisma migrate deploy", + "build": "npm run prisma:generate && tsc -p tsconfig.json", + "prisma:generate": "npx prisma generate", + "prisma:migrate": "npx prisma migrate deploy", "dev": "tsc -p tsconfig.json --watch", "lint": "eslint ./src ./test ./test-integration", - "test:file": "node --experimental-vm-modules --experimental-wasm-modules --experimental-wasm-threads ../../node_modules/jest/bin/jest.js", + "test:file": "node --experimental-vm-modules --experimental-wasm-modules ../../node_modules/jest/bin/jest.js", "test": "npm run test:file -- ./test/**", "test:integration": "npm run test:file -- ./test-integration/** --runInBand", "test:watch": "npm run test:file -- ./test/** --watch" diff --git a/packages/persistance/test-integration/utils.ts b/packages/persistance/test-integration/utils.ts index c0476478e..e42b336e8 100644 --- a/packages/persistance/test-integration/utils.ts +++ b/packages/persistance/test-integration/utils.ts @@ -146,16 +146,7 @@ export function createPrismaAppchain( BlockTrigger: {}, Mempool: {}, BatchProducerModule: {}, - LocalTaskWorkerModule: { - StateTransitionTask: {}, - RuntimeProvingTask: {}, - StateTransitionReductionTask: {}, - BlockReductionTask: {}, - BlockProvingTask: {}, - BlockBuildingTask: {}, - CircuitCompilerTask: {}, - WorkerRegistrationTask: {}, - }, + LocalTaskWorkerModule: VanillaTaskWorkerModules.defaultConfig(), BaseLayer: {}, BlockProducerModule: {}, TaskQueue: { diff --git a/packages/processor/package.json b/packages/processor/package.json index 069de1905..63a5a9848 100644 --- a/packages/processor/package.json +++ b/packages/processor/package.json @@ -13,7 +13,7 @@ "pretest": "npm run pre", "pre": "npm run prisma:generate", "lint": "eslint ./src ./test", - "test:file": "node --experimental-vm-modules --experimental-wasm-modules --experimental-wasm-threads ../../node_modules/jest/bin/jest.js", + "test:file": "node --experimental-vm-modules --experimental-wasm-modules ../../node_modules/jest/bin/jest.js", "prisma:generate": "prisma generate --schema=./test/prisma/schema.prisma", "test": "npm run test:file -- ./src/** ./test/**", "test:watch": "npm run test:file -- ./src/** ./test/** --watch" diff --git a/packages/protocol/package.json b/packages/protocol/package.json index 6fc203033..c71aadcbd 100644 --- a/packages/protocol/package.json +++ b/packages/protocol/package.json @@ -8,7 +8,7 @@ "build": "tsc -p tsconfig.json", "dev": "tsc -p tsconfig.json --watch", "lint": "eslint ./src ./test", - "test:file": "node --experimental-vm-modules --experimental-wasm-modules --experimental-wasm-threads ../../node_modules/jest/bin/jest.js", + "test:file": "node --experimental-vm-modules --experimental-wasm-modules ../../node_modules/jest/bin/jest.js", "test": "npm run test:file -- ./test/**", "test:watch": "npm run test:file -- ./test/** --watch" }, diff --git a/packages/protocol/test/BlockProver.test.ts b/packages/protocol/test/BlockProver.test.ts index ef8cff7ca..fae2a200a 100644 --- a/packages/protocol/test/BlockProver.test.ts +++ b/packages/protocol/test/BlockProver.test.ts @@ -47,6 +47,7 @@ class RuntimeZkProgrammable extends ZkProgrammable< return [ { + name: program.name, compile: program.compile.bind(program), verify: program.verify.bind(program), analyzeMethods: program.analyzeMethods.bind(program), diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 576d0c476..cac8eb4c0 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -8,7 +8,7 @@ "build": "tsc -p tsconfig.json", "dev": "tsc -p tsconfig.json --watch", "lint": "eslint ./src ./test", - "test:file": "node --experimental-vm-modules --experimental-wasm-modules --experimental-wasm-threads ../../node_modules/jest/bin/jest.js", + "test:file": "node --experimental-vm-modules --experimental-wasm-modules ../../node_modules/jest/bin/jest.js", "test": "npm run test:file -- ./test/**", "test:watch": "npm run test:file -- ./test/** --watch" }, diff --git a/packages/sdk/src/appChain/TestingAppChain.ts b/packages/sdk/src/appChain/TestingAppChain.ts index f1c61c6a2..f4b3eb3b8 100644 --- a/packages/sdk/src/appChain/TestingAppChain.ts +++ b/packages/sdk/src/appChain/TestingAppChain.ts @@ -112,9 +112,11 @@ export class TestingAppChain< LocalTaskWorkerModule: VanillaTaskWorkerModules.defaultConfig(), BaseLayer: {}, BlockProducerModule: {}, + SequencerStartupModule: {}, TaskQueue: { simulatedDuration: 0, }, + FeeStrategy: {}, }, Signer: { signer: PrivateKey.random(), diff --git a/packages/sequencer/package.json b/packages/sequencer/package.json index fb1307aa9..9f0a70982 100644 --- a/packages/sequencer/package.json +++ b/packages/sequencer/package.json @@ -8,11 +8,11 @@ "build": "tsc -p tsconfig.json", "dev": "tsc -p tsconfig.json --watch", "lint": "eslint ./src ./test", - "test:file": "node --experimental-vm-modules --experimental-wasm-modules --experimental-wasm-threads ../../node_modules/jest/bin/jest.js", + "test:file": "node --experimental-vm-modules --experimental-wasm-modules ../../node_modules/jest/bin/jest.js", "test": "npm run test:file -- ./test/**", "test:watch": "npm run test:file -- ./test/** --watch", "integration": "npm run test:file -- ./test-integration/** --runInBand", - "start": "npm run build && node --experimental-vm-modules --experimental-wasm-modules --experimental-wasm-threads --es-module-specifier-resolution=node ./dist/src/entry.js" + "start": "npm run build && node --experimental-vm-modules --experimental-wasm-modules --es-module-specifier-resolution=node ./dist/src/entry.js" }, "main": "dist/index.js", "publishConfig": { diff --git a/packages/sequencer/src/protocol/production/tasks/CircuitCompilerTask.ts b/packages/sequencer/src/protocol/production/tasks/CircuitCompilerTask.ts index f5f2951c5..25a5f7b7d 100644 --- a/packages/sequencer/src/protocol/production/tasks/CircuitCompilerTask.ts +++ b/packages/sequencer/src/protocol/production/tasks/CircuitCompilerTask.ts @@ -128,6 +128,8 @@ export class CircuitCompilerTask extends UnpreparingTask< } public async compute(input: CompilerTaskParams): Promise { + log.info("Computing VKs"); + this.compileRegistry.addArtifactsRaw(input.existingArtifacts); // We need to initialize the VK tree root if we have it, so that @@ -138,8 +140,6 @@ export class CircuitCompilerTask extends UnpreparingTask< .setRoot(BigInt(input.runtimeVKRoot)); } - log.info("Computing VKs"); - // TODO make adaptive const targets: Record = { runtime: this.runtime, diff --git a/packages/sequencer/src/sequencer/executor/Sequencer.ts b/packages/sequencer/src/sequencer/executor/Sequencer.ts index c15282705..b17f75d92 100644 --- a/packages/sequencer/src/sequencer/executor/Sequencer.ts +++ b/packages/sequencer/src/sequencer/executor/Sequencer.ts @@ -113,12 +113,15 @@ export class Sequencer // and start the modules in the order they were resolved. for (const moduleName of orderedModules) { const sequencerModule = this.resolve(moduleName); - // eslint-disable-next-line no-await-in-loop - await sequencerModule.start(); log.info( `Starting sequencer module ${moduleName} (${sequencerModule.constructor.name})` ); + // eslint-disable-next-line no-await-in-loop + await sequencerModule.start(); + } + if (!moduleClassNames.includes("SequencerStartupModule")) { + log.warn("SequencerStartupModule is not defined."); } } diff --git a/packages/sequencer/src/worker/queue/LocalTaskQueue.ts b/packages/sequencer/src/worker/queue/LocalTaskQueue.ts index fe2c8606f..668a96458 100644 --- a/packages/sequencer/src/worker/queue/LocalTaskQueue.ts +++ b/packages/sequencer/src/worker/queue/LocalTaskQueue.ts @@ -108,6 +108,8 @@ export class LocalTaskQueue const functions = tasks.map((task) => async () => { // Execute task in worker + log.trace(`Working ${task.payload.name} with id ${task.taskId}`); + const payload = await this.workers[queueName]?.handler( task.payload ); @@ -116,13 +118,14 @@ export class LocalTaskQueue return; } log.trace("LocalTaskQueue got", JSON.stringify(payload)); + // Notify listeners about result const listenerPromises = this.listeners[queueName]?.map( async (listener) => { await listener(payload); } ); - void Promise.all(listenerPromises || []); + await Promise.all(listenerPromises || []); }); this.queuedTasks[queueName] = []; return functions; diff --git a/packages/sequencer/src/worker/worker/LocalTaskWorkerModule.ts b/packages/sequencer/src/worker/worker/LocalTaskWorkerModule.ts index 6ffac14d1..0619918c4 100644 --- a/packages/sequencer/src/worker/worker/LocalTaskWorkerModule.ts +++ b/packages/sequencer/src/worker/worker/LocalTaskWorkerModule.ts @@ -153,14 +153,16 @@ export class VanillaTaskWorkerModules { return { StateTransitionTask: {}, RuntimeProvingTask: {}, - BlockProvingTask: {}, - BlockReductionTask: {}, + TransactionProvingTask: {}, BlockBuildingTask: {}, + BlockReductionTask: {}, StateTransitionReductionTask: {}, SettlementProvingTask: {}, CircuitCompilerTask: {}, WorkerRegistrationTask: {}, - }; + } satisfies ModulesConfig< + ReturnType + >; } } diff --git a/packages/sequencer/test/integration/BlockProduction.test.ts b/packages/sequencer/test/integration/BlockProduction.test.ts index ab3a5c011..0e8077103 100644 --- a/packages/sequencer/test/integration/BlockProduction.test.ts +++ b/packages/sequencer/test/integration/BlockProduction.test.ts @@ -24,7 +24,12 @@ import { Bool, Field, PrivateKey, PublicKey, Struct, UInt64 } from "o1js"; import "reflect-metadata"; import { container } from "tsyringe"; -import { BatchStorage, HistoricalBatchStorage, Sequencer } from "../../src"; +import { + BatchStorage, + HistoricalBatchStorage, + Sequencer, + VanillaTaskWorkerModules, +} from "../../src"; import { DefaultTestingSequencerModules, testingSequencerFromModules, @@ -91,10 +96,6 @@ describe("block production", () => { let test: BlockTestService; beforeEach(async () => { - // container.reset(); - - log.setLevel(log.levels.DEBUG); - const runtimeClass = Runtime.from({ modules: { Balance, @@ -133,7 +134,7 @@ describe("block production", () => { Mempool: {}, BatchProducerModule: {}, BlockProducerModule: {}, - LocalTaskWorkerModule: {}, + LocalTaskWorkerModule: VanillaTaskWorkerModules.defaultConfig(), BaseLayer: {}, TaskQueue: {}, FeeStrategy: {}, @@ -164,6 +165,7 @@ describe("block production", () => { appChain = app; + // @ts-ignore ({ runtime, sequencer, protocol } = app); test = app.sequencer.dependencyContainer.resolve(BlockTestService); @@ -172,8 +174,6 @@ describe("block production", () => { it("should produce a dummy block proof", async () => { expect.assertions(27); - log.setLevel("TRACE"); - const privateKey = PrivateKey.random(); const publicKey = privateKey.toPublicKey(); @@ -260,8 +260,6 @@ describe("block production", () => { expect(block).toBeDefined(); expect(block!.transactions).toHaveLength(1); - console.log(block!.transactions[0]); - console.log(block!.transactions[0].statusMessage); expect(block!.transactions[0].status.toBoolean()).toBe(true); expect(block!.transactions[0].statusMessage).toBeUndefined(); @@ -277,8 +275,6 @@ describe("block production", () => { it("should reject tx and not apply the state", async () => { expect.assertions(5); - log.setLevel("INFO"); - const privateKey = PrivateKey.random(); await test.addTransaction({ @@ -308,26 +304,20 @@ describe("block production", () => { }, 30_000); it("should produce txs in non-consecutive blocks", async () => { - log.setLevel("TRACE"); - const privateKey = PrivateKey.random(); const publicKey = privateKey.toPublicKey(); const privateKey2 = PrivateKey.random(); const publicKey2 = privateKey2.toPublicKey(); - await mempool.add( - createTransaction({ - runtime, - method: ["Balance", "setBalanceIf"], - privateKey, - args: [publicKey, UInt64.from(100), Bool(true)], - nonce: 0, - }) - ); + await test.addTransaction({ + method: ["Balance", "setBalanceIf"], + privateKey, + args: [publicKey, UInt64.from(100), Bool(true)], + }); // let [block, batch] = await blockTrigger.produceBlockAndBatch(); - const block = await blockTrigger.produceBlock(); + const block = await test.produceBlock(); expect(block).toBeDefined(); @@ -338,72 +328,52 @@ describe("block production", () => { expect(block!.transactions[0].stateTransitions).toHaveLength(1); expect(block!.transactions[0].protocolTransitions).toHaveLength(2); - await blockTrigger.produceBlock(); + await test.produceBlock(); - await mempool.add( - createTransaction({ - runtime, - method: ["Balance", "setBalanceIf"], - privateKey: privateKey2, - args: [publicKey2, UInt64.from(100), Bool(true)], - nonce: 0, - }) - ); - await blockTrigger.produceBlock(); - - await mempool.add( - createTransaction({ - runtime, - method: ["Balance", "setBalanceIf"], - privateKey: privateKey2, - args: [publicKey2, UInt64.from(100), Bool(true)], - nonce: 1, - }) - ); - await blockTrigger.produceBlock(); - - await mempool.add( - createTransaction({ - runtime, - method: ["Balance", "setBalanceIf"], - privateKey: privateKey2, - args: [publicKey2, UInt64.from(100), Bool(true)], - nonce: 2, - }) - ); - await blockTrigger.produceBlock(); - - await mempool.add( - createTransaction({ - runtime, - method: ["Balance", "setBalanceIf"], - privateKey: privateKey2, - args: [publicKey2, UInt64.from(100), Bool(true)], - nonce: 3, - }) - ); - await blockTrigger.produceBlock(); + await test.addTransaction({ + method: ["Balance", "setBalanceIf"], + privateKey: privateKey2, + args: [publicKey2, UInt64.from(100), Bool(true)], + }); + await test.produceBlock(); + + await test.addTransaction({ + method: ["Balance", "setBalanceIf"], + privateKey: privateKey2, + args: [publicKey2, UInt64.from(100), Bool(true)], + }); + + await test.produceBlock(); + + await test.addTransaction({ + method: ["Balance", "setBalanceIf"], + privateKey: privateKey2, + args: [publicKey2, UInt64.from(100), Bool(true)], + }); + + await test.produceBlock(); + + await test.addTransaction({ + method: ["Balance", "setBalanceIf"], + privateKey: privateKey2, + args: [publicKey2, UInt64.from(100), Bool(true)], + }); + await test.produceBlock(); // Second tx - await mempool.add( - createTransaction({ - runtime, - method: ["Balance", "setBalanceIf"], - privateKey, - args: [publicKey, UInt64.from(100), Bool(true)], - nonce: 1, - }) - ); + await test.addTransaction({ + method: ["Balance", "setBalanceIf"], + privateKey, + args: [publicKey, UInt64.from(100), Bool(true)], + }); log.info("Starting second block"); - const block2 = await blockTrigger.produceBlock(); + const block2 = await test.produceBlock(); expect(block2).toBeDefined(); expect(block2!.transactions).toHaveLength(1); - console.log(block2!.transactions[0]); - console.log(block2!.transactions[0].statusMessage); expect(block2!.transactions[0].status.toBoolean()).toBe(true); expect(block2!.transactions[0].statusMessage).toBeUndefined(); }, 60_000); @@ -632,8 +602,6 @@ describe("block production", () => { }, 360_000); it("regression - should produce block with no STs emitted", async () => { - log.setLevel("TRACE"); - const privateKey = PrivateKey.random(); await test.addTransaction({ diff --git a/packages/sequencer/test/integration/BlockProductionSize.test.ts b/packages/sequencer/test/integration/BlockProductionSize.test.ts index 19217559b..13a213038 100644 --- a/packages/sequencer/test/integration/BlockProductionSize.test.ts +++ b/packages/sequencer/test/integration/BlockProductionSize.test.ts @@ -7,7 +7,12 @@ import { Bool, PrivateKey, Struct, UInt64 } from "o1js"; import "reflect-metadata"; import { container } from "tsyringe"; -import { ManualBlockTrigger, PrivateMempool, Sequencer } from "../../src"; +import { + ManualBlockTrigger, + PrivateMempool, + Sequencer, + VanillaTaskWorkerModules, +} from "../../src"; import { DefaultTestingSequencerModules, testingSequencerFromModules, @@ -76,11 +81,12 @@ describe("block limit", () => { BlockProducerModule: { maximumBlockSize: maxBlockSize, }, - LocalTaskWorkerModule: {}, + LocalTaskWorkerModule: VanillaTaskWorkerModules.defaultConfig(), BaseLayer: {}, TaskQueue: {}, FeeStrategy: {}, ProtocolStartupModule: {}, + SequencerStartupModule: {}, }, Runtime: { Balance: {}, diff --git a/packages/sequencer/test/integration/Mempool.test.ts b/packages/sequencer/test/integration/Mempool.test.ts index a71877c3e..2d13305c3 100644 --- a/packages/sequencer/test/integration/Mempool.test.ts +++ b/packages/sequencer/test/integration/Mempool.test.ts @@ -13,6 +13,7 @@ import { Sequencer, SequencerModule, StorageDependencyFactory, + VanillaTaskWorkerModules, } from "../../src"; import { DefaultTestingSequencerModules, @@ -109,9 +110,10 @@ describe.each([["InMemory", InMemoryDatabase]])( FeeStrategy: {}, BatchProducerModule: {}, BlockProducerModule: {}, - LocalTaskWorkerModule: {}, + LocalTaskWorkerModule: VanillaTaskWorkerModules.defaultConfig(), BaseLayer: {}, TaskQueue: {}, + SequencerStartupModule: {}, ProtocolStartupModule: {}, }, Protocol: { @@ -124,7 +126,7 @@ describe.each([["InMemory", InMemoryDatabase]])( }); // Start AppChain - await appChain.start(container.createChildContainer()); + await appChain.start(false, container.createChildContainer()); runtime = appChain.runtime; sequencer = appChain.sequencer; diff --git a/packages/sequencer/test/integration/Proven.test.ts b/packages/sequencer/test/integration/Proven.test.ts index 5911ce034..e456dee9d 100644 --- a/packages/sequencer/test/integration/Proven.test.ts +++ b/packages/sequencer/test/integration/Proven.test.ts @@ -26,6 +26,7 @@ import { ProvenSettlementPermissions, SettlementModule, SettlementProvingTask, + VanillaTaskWorkerModules, WithdrawalQueue, } from "../../src"; import { SettlementStartupModule } from "../../src/sequencer/SettlementStartupModule"; @@ -36,7 +37,7 @@ import { ProvenBalance } from "./mocks/ProvenBalance"; const timeout = 300000; -describe("Proven", () => { +describe.skip("Proven", () => { let test: BlockTestService; let appChain: AppChain; @@ -95,7 +96,7 @@ describe("Proven", () => { Mempool: {}, BatchProducerModule: {}, BlockProducerModule: {}, - LocalTaskWorkerModule: {}, + LocalTaskWorkerModule: VanillaTaskWorkerModules.defaultConfig(), TaskQueue: {}, FeeStrategy: {}, SequencerStartupModule: {}, diff --git a/packages/sequencer/test/integration/StorageIntegration.test.ts b/packages/sequencer/test/integration/StorageIntegration.test.ts index 5081c909d..5dc4cb36f 100644 --- a/packages/sequencer/test/integration/StorageIntegration.test.ts +++ b/packages/sequencer/test/integration/StorageIntegration.test.ts @@ -19,6 +19,7 @@ import { StateRecord, StorageDependencyFactory, BlockStorage, + VanillaTaskWorkerModules, } from "../../src"; import { DefaultTestingSequencerModules, @@ -104,7 +105,7 @@ describe.each([["InMemory", InMemoryDatabase]])( Mempool: {}, BatchProducerModule: {}, BlockProducerModule: {}, - LocalTaskWorkerModule: {}, + LocalTaskWorkerModule: VanillaTaskWorkerModules.defaultConfig(), BaseLayer: {}, TaskQueue: {}, FeeStrategy: {}, @@ -119,7 +120,7 @@ describe.each([["InMemory", InMemoryDatabase]])( }, }); - await appChain.start(); + await appChain.start(false); runtime = appChain.runtime; sequencer = appChain.sequencer; diff --git a/packages/sequencer/test/protocol/production/sequencing/atomic-block-production.test.ts b/packages/sequencer/test/protocol/production/sequencing/atomic-block-production.test.ts index 23c411dcc..3d6fadcf8 100644 --- a/packages/sequencer/test/protocol/production/sequencing/atomic-block-production.test.ts +++ b/packages/sequencer/test/protocol/production/sequencing/atomic-block-production.test.ts @@ -10,7 +10,7 @@ import { expectDefined } from "@proto-kit/common"; import { BlockQueue, ManualBlockTrigger, - TransactionExecutionService, + VanillaTaskWorkerModules, } from "../../../../src"; import { ProtocolStateTestHook } from "../../../integration/mocks/ProtocolStateTestHook"; import { @@ -18,6 +18,7 @@ import { testingSequencerFromModules, } from "../../../TestingSequencer"; import { Balance } from "../../../integration/mocks/Balance"; +import { BlockResultService } from "../../../../src/protocol/production/sequencing/BlockResultService"; describe("atomic block production", () => { let appchain: AppChain; @@ -57,11 +58,12 @@ describe("atomic block production", () => { Mempool: {}, BatchProducerModule: {}, BlockProducerModule: {}, - LocalTaskWorkerModule: {}, + LocalTaskWorkerModule: VanillaTaskWorkerModules.defaultConfig(), BaseLayer: {}, TaskQueue: {}, FeeStrategy: {}, ProtocolStartupModule: {}, + SequencerStartupModule: {}, }, Runtime: { Balance: {}, @@ -79,7 +81,7 @@ describe("atomic block production", () => { appchain = app; // Start AppChain - await app.start(container.createChildContainer()); + await app.start(false, container.createChildContainer()); trigger = app.sequencer.resolve("BlockTrigger"); }); @@ -95,9 +97,8 @@ describe("atomic block production", () => { it("should recover from non-generated metadata", async () => { expect.assertions(6); - const module = appchain.sequencer.dependencyContainer.resolve( - TransactionExecutionService - ); + const module = + appchain.sequencer.dependencyContainer.resolve(BlockResultService); module.generateMetadataForNextBlock = jest .fn(module.generateMetadataForNextBlock) diff --git a/packages/sequencer/test/settlement/Settlement.ts b/packages/sequencer/test/settlement/Settlement.ts index a46e088eb..69cef5ed7 100644 --- a/packages/sequencer/test/settlement/Settlement.ts +++ b/packages/sequencer/test/settlement/Settlement.ts @@ -1,7 +1,6 @@ /* eslint-disable no-inner-declarations */ import { expectDefined, - log, mapSequential, TypedClass, RollupMerkleTree, @@ -55,6 +54,7 @@ import { MinaBaseLayerConfig, SignedSettlementPermissions, ProvenSettlementPermissions, + VanillaTaskWorkerModules, } from "../../src"; import { BlockProofSerializer } from "../../src/protocol/production/tasks/serializers/BlockProofSerializer"; import { testingSequencerFromModules } from "../TestingSequencer"; @@ -168,7 +168,7 @@ export const settlementTestFn = ( BlockTrigger: {}, Mempool: {}, BatchProducerModule: {}, - LocalTaskWorkerModule: {}, + LocalTaskWorkerModule: VanillaTaskWorkerModules.defaultConfig(), OutgoingMessageQueue: {}, BaseLayer: baseLayerConfig, BlockProducerModule: {}, @@ -255,8 +255,6 @@ export const settlementTestFn = ( } beforeAll(async () => { - log.setLevel("DEBUG"); - appChain = setupAppChain(); await appChain.start( diff --git a/packages/sequencer/test/worker/Flow.test.ts b/packages/sequencer/test/worker/Flow.test.ts index 4c795ddfa..a991aca65 100644 --- a/packages/sequencer/test/worker/Flow.test.ts +++ b/packages/sequencer/test/worker/Flow.test.ts @@ -1,7 +1,7 @@ import "reflect-metadata"; import { afterEach, beforeEach } from "@jest/globals"; -import { noop } from "@proto-kit/common"; +import { log, noop } from "@proto-kit/common"; import { container } from "tsyringe"; import { @@ -198,6 +198,8 @@ describe("flow", () => { async (inputs: [string, string][]) => { expect.assertions(1); + log.setLevel("TRACE"); + const result = inputs .map<[number, bigint]>((input) => [ Number.parseInt(input[0], 10) * 2, @@ -244,7 +246,7 @@ describe("flow", () => { const resolveReduction = async () => { let reductions = flow.state.reductionQueue; - console.log(reductions.length); + console.log("Length:", reductions.length); if (reductions.length === 1 && flow.tasksInProgress === 0) { resolve(reductions[0]); diff --git a/packages/sequencer/test/worker/Worker.test.ts b/packages/sequencer/test/worker/Worker.test.ts index 42ca7c39d..d75933222 100644 --- a/packages/sequencer/test/worker/Worker.test.ts +++ b/packages/sequencer/test/worker/Worker.test.ts @@ -3,9 +3,8 @@ import "reflect-metadata"; import { beforeAll } from "@jest/globals"; // import { BullQueue } from "@proto-kit/deployment"; -import { Closeable, TaskQueue } from "../../src/worker/queue/TaskQueue"; +import { TaskQueue, Closeable, LocalTaskQueue } from "../../src"; // TODO Properly move out -import { LocalTaskQueue } from "../../src/worker/queue/LocalTaskQueue"; // // The implementation of the task, known by both master and worker // class SumTask implements MapReduceTask { diff --git a/packages/stack/package.json b/packages/stack/package.json index 7cecd1c77..63ddbe280 100644 --- a/packages/stack/package.json +++ b/packages/stack/package.json @@ -8,11 +8,11 @@ "build": "tsc -p tsconfig.json", "dev": "tsc -p tsconfig.json --watch", "lint": "eslint ./src ./test", - "test:file": "node --experimental-vm-modules --experimental-wasm-modules --experimental-wasm-threads ../../node_modules/jest/bin/jest.js", + "test:file": "node --experimental-vm-modules ../../node_modules/jest/bin/jest.js", "test": "npm run test:file -- ./test/**", "test:watch": "npm run test:file -- ./test/** --watch", "graphql": "npm run build && node --experimental-vm-modules --experimental-wasm-modules --experimental-wasm-threads --es-module-specifier-resolution=node ./dist/src/scripts/graphql/run-graphql.js", - "start": "node --experimental-vm-modules --experimental-wasm-modules --experimental-wasm-threads --es-module-specifier-resolution=node ./dist/start.js" + "start": "node --experimental-vm-modules --es-module-specifier-resolution=node ./dist/start.js" }, "main": "dist/index.js", "publishConfig": { diff --git a/packages/stack/src/scripts/graphql/server.ts b/packages/stack/src/scripts/graphql/server.ts index c13969042..e3b448c03 100644 --- a/packages/stack/src/scripts/graphql/server.ts +++ b/packages/stack/src/scripts/graphql/server.ts @@ -34,6 +34,7 @@ import { Sequencer, BlockProducerModule, VanillaTaskWorkerModules, + SequencerStartupModule, } from "@proto-kit/sequencer"; import { BatchStorageResolver, @@ -134,6 +135,7 @@ export async function startServer() { BlockResolver: {}, }, }), + SequencerStartupModule, }, }), @@ -171,6 +173,8 @@ export async function startServer() { host: "0.0.0.0", graphiql: true, }, + SequencerStartupModule: {}, + // SettlementModule: { // address: PrivateKey.random().toPublicKey(), // feepayer: PrivateKey.random(), @@ -206,17 +210,7 @@ export async function startServer() { Mempool: {}, BatchProducerModule: {}, - LocalTaskWorkerModule: { - StateTransitionTask: {}, - // SettlementProvingTask: {}, - BlockBuildingTask: {}, - BlockProvingTask: {}, - BlockReductionTask: {}, - RuntimeProvingTask: {}, - StateTransitionReductionTask: {}, - CircuitCompilerTask: {}, - WorkerRegistrationTask: {}, - }, + LocalTaskWorkerModule: VanillaTaskWorkerModules.defaultConfig(), BaseLayer: {}, TaskQueue: {},