Merge pull request #126 from proto-kit/feature/devops-rampup

Feature/devops rampup

Author: rpanic

.github/workflows/pull-request-develop.yml

@@ -91,3 +91,46 @@ jobs:
 
       - name: "Test"
         run: npm run test:ci
+
+  integration:
+    runs-on: ubuntu-latest
+    needs: test
+
+    env:
+      POSTGRES_URL: localhost
+      REDIS_URL: localhost
+      REDIS_CI: true
+      DATABASE_URL: "postgresql://admin:password@localhost:5432/protokit?schema=public"
+
+    services:
+      postgres:
+        image: postgres:14-alpine
+        env:
+          POSTGRES_PASSWORD: password
+          POSTGRES_USER: admin
+          POSTGRES_DB: protokit
+        ports:
+          - 5432:5432
+      redis:
+        image: redis:6.2-alpine
+        ports:
+          - 6379:6379
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          cache: npm
+
+      - name: "Install dependencies"
+        run: npm ci --workspaces --include-workspace-root
+
+      - name: "Build"
+        run: npm run build
+
+      - name: "Migrate DB"
+        run: npm run migrate
+
+      - name: "Integration tests"
+        run: npm run test:integration

package.json

@@ -10,7 +10,9 @@
     "lint:staged": "eslint",
     "test": "npx lerna run test --scope=\"*/sdk\" -- --passWithNoTests",
     "test:ci": "npx lerna run test -- --passWithNoTests --forceExit",
+    "test:integration": "npx lerna run integration -- --passWithNoTests --forceExit",
     "test:watch": "npx lerna run test:watch",
+    "migrate": "npx lerna run prisma-migrate",
     "commit": "cz",
     "publish:canary": "npx lerna publish prerelease --no-private --exact --yes --canary --preid develop --dist-tag latest --loglevel verbose --force-git-tag --force-publish"
   },

packages/persistance/package.json

@@ -12,6 +12,7 @@
     "lint": "eslint ./src ./test",
     "test:file": "node --experimental-vm-modules --experimental-wasm-modules --experimental-wasm-threads ../../node_modules/jest/bin/jest.js",
     "test": "npm run test:file -- ./test/**",
+    "integration": "npm run test:file -- ./test-integration/** --runInBand",
     "test:watch": "npm run test:file -- ./test/** --watch"
   },
   "main": "dist/index.js",

packages/persistance/prisma/migrations/20240213172637_messages/migration.sql

@@ -0,0 +1,55 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `args` on the `Transaction` table. All the data in the column will be lost.
+  - Added the required column `fromMessagesHash` to the `Block` table without a default value. This is not possible if the table is not empty.
+  - Added the required column `toMessagesHash` to the `Block` table without a default value. This is not possible if the table is not empty.
+  - Added the required column `isMessage` to the `Transaction` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+ALTER TABLE "Batch" ADD COLUMN     "settlementTransactionHash" TEXT;
+
+-- AlterTable
+ALTER TABLE "Block" ADD COLUMN     "fromMessagesHash" TEXT NOT NULL,
+ADD COLUMN     "toMessagesHash" TEXT NOT NULL;
+
+-- AlterTable
+ALTER TABLE "Transaction" DROP COLUMN "args",
+ADD COLUMN     "argsFields" TEXT[],
+ADD COLUMN     "argsJSON" TEXT[],
+ADD COLUMN     "isMessage" BOOLEAN NOT NULL;
+
+-- CreateTable
+CREATE TABLE "Settlement" (
+    "transactionHash" TEXT NOT NULL,
+    "promisedMessagesHash" TEXT NOT NULL,
+
+    CONSTRAINT "Settlement_pkey" PRIMARY KEY ("transactionHash")
+);
+
+-- CreateTable
+CREATE TABLE "IncomingMessageBatchTransaction" (
+    "transactionHash" TEXT NOT NULL,
+    "batchId" INTEGER NOT NULL,
+
+    CONSTRAINT "IncomingMessageBatchTransaction_pkey" PRIMARY KEY ("transactionHash","batchId")
+);
+
+-- CreateTable
+CREATE TABLE "IncomingMessageBatch" (
+    "id" SERIAL NOT NULL,
+    "fromMessageHash" TEXT NOT NULL,
+    "toMessageHash" TEXT NOT NULL,
+
+    CONSTRAINT "IncomingMessageBatch_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "Batch" ADD CONSTRAINT "Batch_settlementTransactionHash_fkey" FOREIGN KEY ("settlementTransactionHash") REFERENCES "Settlement"("transactionHash") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "IncomingMessageBatchTransaction" ADD CONSTRAINT "IncomingMessageBatchTransaction_transactionHash_fkey" FOREIGN KEY ("transactionHash") REFERENCES "Transaction"("hash") ON DELETE RESTRICT ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "IncomingMessageBatchTransaction" ADD CONSTRAINT "IncomingMessageBatchTransaction_batchId_fkey" FOREIGN KEY ("batchId") REFERENCES "IncomingMessageBatch"("id") ON DELETE RESTRICT ON UPDATE CASCADE;

packages/persistance/src/PrismaDatabaseConnection.ts

@@ -76,6 +76,26 @@ export class PrismaDatabaseConnection
     };
   }
 
+  public async clearDatabase(): Promise<void> {
+    const tables = [
+      "TransactionExecutionResult",
+      "Transaction",
+      "Block",
+      "Batch",
+      "UnprovenBlockMetadata",
+      "State",
+      "Settlement",
+      "IncomingMessageBatch",
+      "IncomingMessageBatchTransaction",
+    ];
+
+    await this.prismaClient.$transaction(
+      tables.map((table) =>
+        this.prismaClient.$executeRawUnsafe(`TRUNCATE TABLE "${table}" CASCADE`)
+      )
+    );
+  }
+
   public async start(): Promise<void> {
     const { connection } = this.config;
     if (connection !== undefined) {
@@ -102,4 +122,8 @@ export class PrismaDatabaseConnection
       this.initializedClient = new PrismaClient();
     }
   }
+
+  public async close() {
+    await this.prismaClient.$disconnect();
+  }
 }

packages/persistance/src/PrismaRedisDatabase.ts

@@ -66,4 +66,9 @@ export class PrismaRedisDatabase
     this.redis.config = this.config.redis;
     await this.redis.start();
   }
+
+  public async close() {
+    await this.prisma.close();
+    await this.redis.close();
+  }
 }

packages/persistance/src/RedisConnection.ts

@@ -9,9 +9,9 @@ import { RedisMerkleTreeStore } from "./services/redis/RedisMerkleTreeStore";
 
 export interface RedisConnectionConfig {
   host: string;
-  password: string;
+  password?: string;
   port?: number;
-  username?: string
+  username?: string;
 }
 
 export interface RedisConnection {
@@ -50,6 +50,10 @@ export class RedisConnectionModule
     };
   }
 
+  public async clearDatabase() {
+    await this.redisClient.flushAll();
+  }
+
   public async init() {
     const { host, port, password, username } = this.config;
     this.client = createClient({
@@ -70,4 +74,8 @@ export class RedisConnectionModule
   public async start(): Promise<void> {
     await this.init();
   }
+
+  public async close() {
+    await this.redisClient.disconnect();
+  }
 }

packages/persistance/src/services/prisma/PrismaStateService.ts

@@ -6,6 +6,12 @@ import { noop } from "@proto-kit/common";
 import type { PrismaConnection } from "../../PrismaDatabaseConnection";
 import { injectable } from "tsyringe";
 
+// We need to create a correctly configured Decimal constructor
+// with our parameters
+const Decimal = Prisma.Decimal.clone({
+  precision: 78,
+});
+
 @injectable()
 export class PrismaStateService implements AsyncStateService {
   private cache: StateEntry[] = [];
@@ -25,18 +31,16 @@ export class PrismaStateService implements AsyncStateService {
     const data = this.cache
       .filter((entry) => entry.value !== undefined)
       .map((entry) => ({
-        path: new Prisma.Decimal(entry.key.toString()),
-        values: entry.value!.map(
-          (field) => new Prisma.Decimal(field.toString())
-        ),
+        path: new Decimal(entry.key.toString()),
+        values: entry.value!.map((field) => new Decimal(field.toString())),
         mask: this.mask,
       }));
 
     await prismaClient.$transaction([
       prismaClient.state.deleteMany({
         where: {
           path: {
-            in: this.cache.map((x) => new Prisma.Decimal(x.key.toString())),
+            in: this.cache.map((x) => new Decimal(x.key.toString())),
           },
           mask: this.mask,
         },
@@ -55,7 +59,7 @@ export class PrismaStateService implements AsyncStateService {
         AND: [
           {
             path: {
-              in: keys.map((key) => new Prisma.Decimal(key.toString())),
+              in: keys.map((key) => new Decimal(key.toString())),
             },
           },
           {
@@ -65,8 +69,8 @@ export class PrismaStateService implements AsyncStateService {
       },
     });
     return records.map((record) => ({
-      key: Field(record.path.toNumber()),
-      value: record.values.map((x) => Field(x.toString())),
+      key: Field(record.path.toFixed()),
+      value: record.values.map((x) => Field(x.toFixed())),
     }));
   }