Prisma 6.14.0 upgrade (#2444)

* Initial work on upgrading to 6.14.0

Set the output to node_modules still to make it easier

* Use ./generated Prisma folder, update types to fix issues

* Docker compose restart Clickhouse

* Prisma instrumentation update

* Docker

* Removed database dockerignore file, add generated prisma client to the top-level one

* Delete v3-catalog package.json

* Resolved pnpm lock file

* Log errors for very slow queries

Author: matt-aitken

.cursor/rules/webapp.mdc

@@ -6,7 +6,7 @@ alwaysApply: false
 
 The main trigger.dev webapp, which powers it's API and dashboard and makes up the docker image that is produced as an OSS image, is a Remix 2.1.0 app that uses an express server, written in TypeScript. The following subsystems are either included in the webapp or are used by the webapp in another part of the monorepo:
 
-- `@trigger.dev/database` exports a Prisma 5.4.1 client that is used extensively in the webapp to access a PostgreSQL instance. The schema file is [schema.prisma](mdc:internal-packages/database/prisma/schema.prisma)
+- `@trigger.dev/database` exports a Prisma 6.14.0 client that is used extensively in the webapp to access a PostgreSQL instance. The schema file is [schema.prisma](mdc:internal-packages/database/prisma/schema.prisma)
 - `@trigger.dev/core` is a published package and is used to share code between the `@trigger.dev/sdk` and the webapp. It includes functionality but also a load of Zod schemas for data validation. When importing from `@trigger.dev/core` in the webapp, we never import the root `@trigger.dev/core` path, instead we favor one of the subpath exports that you can find in [package.json](mdc:packages/core/package.json)
 - `@internal/run-engine` has all the code needed to trigger a run and take it through it's lifecycle to completion.
 - `@trigger.dev/redis-worker` is a custom redis based background job/worker system that's used in the webapp and also used inside the run engine.
@@ -31,7 +31,10 @@ We originally the Trigger.dev "Run Engine" not as a single system, but just spre
 - The batch trigger API endpoint is [api.v1.tasks.batch.ts](mdc:apps/webapp/app/routes/api.v1.tasks.batch.ts)
 - Setup code for the prisma client is in [db.server.ts](mdc:apps/webapp/app/db.server.ts)
 - The run engine is configured in [runEngine.server.ts](mdc:apps/webapp/app/v3/runEngine.server.ts)
-- All the "services" that are found in app/v3/services/**/*.server.ts
+- All the "services" that are found in app/v3/services/\*_/_.server.ts
 - The code for the TaskEvent data, which is the otel data sent from tasks to our servers, is in both the [eventRepository.server.ts](mdc:apps/webapp/app/v3/eventRepository.server.ts) and also the [otlpExporter.server.ts](mdc:apps/webapp/app/v3/otlpExporter.server.ts). The otel endpoints which are hit from production and development otel exporters is [otel.v1.logs.ts](mdc:apps/webapp/app/routes/otel.v1.logs.ts) and [otel.v1.traces.ts](mdc:apps/webapp/app/routes/otel.v1.traces.ts)
-- We use "presenters" to move more complex loader code into a class, and you can find those are app/v3/presenters/**/*.server.ts
+- We use "presenters" to move more complex loader code into a class, and you can find those are app/v3/presenters/\*_/_.server.ts
 
+- All the "services" that are found in app/v3/services/\*_/_.server.ts
+- The code for the TaskEvent data, which is the otel data sent from tasks to our servers, is in both the [eventRepository.server.ts](mdc:apps/webapp/app/v3/eventRepository.server.ts) and also the [otlpExporter.server.ts](mdc:apps/webapp/app/v3/otlpExporter.server.ts). The otel endpoints which are hit from production and development otel exporters is [otel.v1.logs.ts](mdc:apps/webapp/app/routes/otel.v1.logs.ts) and [otel.v1.traces.ts](mdc:apps/webapp/app/routes/otel.v1.traces.ts)
+- We use "presenters" to move more complex loader code into a class, and you can find those are app/v3/presenters/\*_/_.server.ts

.dockerignore

@@ -16,6 +16,8 @@
 **/dist
 **/node_modules
 
+**/generated/prisma
+
 apps/webapp/build
 apps/webapp/public/build
 

apps/webapp/app/db.server.ts

@@ -1,20 +1,21 @@
 import {
   Prisma,
   PrismaClient,
-  PrismaClientOrTransaction,
-  PrismaReplicaClient,
-  PrismaTransactionClient,
-  PrismaTransactionOptions,
+  $transaction as transac,
+  type PrismaClientOrTransaction,
+  type PrismaReplicaClient,
+  type PrismaTransactionClient,
+  type PrismaTransactionOptions,
 } from "@trigger.dev/database";
 import invariant from "tiny-invariant";
 import { z } from "zod";
 import { env } from "./env.server";
 import { logger } from "./services/logger.server";
 import { isValidDatabaseUrl } from "./utils/db";
 import { singleton } from "./utils/singleton";
-import { $transaction as transac } from "@trigger.dev/database";
 import { startActiveSpan } from "./v3/tracer.server";
 import { Span } from "@opentelemetry/api";
+import { queryPerformanceMonitor } from "./utils/queryPerformanceMonitor.server";
 
 export type {
   PrismaTransactionClient,
@@ -153,13 +154,19 @@ function getClient() {
             },
           ]
         : []) satisfies Prisma.LogDefinition[]),
-      // verbose
-      ...((process.env.VERBOSE_PRISMA_LOGS === "1"
+      // Query performance monitoring
+      ...((process.env.VERBOSE_PRISMA_LOGS === "1" ||
+      process.env.VERY_SLOW_QUERY_THRESHOLD_MS !== undefined
         ? [
             {
               emit: "event",
               level: "query",
             },
+          ]
+        : []) satisfies Prisma.LogDefinition[]),
+      // verbose
+      ...((process.env.VERBOSE_PRISMA_LOGS === "1"
+        ? [
             {
               emit: "stdout",
               level: "query",
@@ -206,6 +213,11 @@ function getClient() {
     });
   }
 
+  // Add query performance monitoring
+  client.$on("query", (log) => {
+    queryPerformanceMonitor.onQuery("writer", log);
+  });
+
   // connect eagerly
   client.$connect();
 
@@ -265,13 +277,19 @@ function getReplicaClient() {
             },
           ]
         : []) satisfies Prisma.LogDefinition[]),
-      // verbose
-      ...((process.env.VERBOSE_PRISMA_LOGS === "1"
+      // Query performance monitoring
+      ...((process.env.VERBOSE_PRISMA_LOGS === "1" ||
+      process.env.VERY_SLOW_QUERY_THRESHOLD_MS !== undefined
         ? [
             {
               emit: "event",
               level: "query",
             },
+          ]
+        : []) satisfies Prisma.LogDefinition[]),
+      // verbose
+      ...((process.env.VERBOSE_PRISMA_LOGS === "1"
+        ? [
             {
               emit: "stdout",
               level: "query",
@@ -317,6 +335,11 @@ function getReplicaClient() {
     });
   }
 
+  // Add query performance monitoring for replica client
+  replicaClient.$on("query", (log) => {
+    queryPerformanceMonitor.onQuery("replica", log);
+  });
+
   // connect eagerly
   replicaClient.$connect();
 

apps/webapp/app/env.server.ts

@@ -1080,6 +1080,8 @@ const EnvironmentSchema = z.object({
   AI_RUN_FILTER_MODEL: z.string().optional(),
 
   EVENT_LOOP_MONITOR_THRESHOLD_MS: z.coerce.number().int().default(100),
+
+  VERY_SLOW_QUERY_THRESHOLD_MS: z.coerce.number().int().optional(),
 });
 
 export type Environment = z.infer<typeof EnvironmentSchema>;

apps/webapp/app/routes/api.v1.projects.$projectRef.background-workers.$envSlug.$version.ts

@@ -82,9 +82,9 @@ export async function loader({ params, request }: LoaderFunctionArgs) {
   });
 }
 
-function decompressContent(compressedBuffer: Buffer): string {
-  // First, we need to decode the base64 Buffer to get the actual compressed data
-  const decodedBuffer = Buffer.from(compressedBuffer.toString("utf-8"), "base64");
+function decompressContent(compressedBuffer: Uint8Array): string {
+  // Convert Uint8Array to Buffer and decode base64 in one step
+  const decodedBuffer = Buffer.from(Buffer.from(compressedBuffer).toString("utf-8"), "base64");
 
   // Decompress the data
   const decompressedData = zlib.inflateSync(decodedBuffer);

apps/webapp/app/utils/queryPerformanceMonitor.server.ts

@@ -0,0 +1,64 @@
+import { env } from "~/env.server";
+import { logger } from "~/services/logger.server";
+
+export interface QueryPerformanceConfig {
+  verySlowQueryThreshold?: number; // ms
+  maxQueryLogLength: number;
+}
+
+export class QueryPerformanceMonitor {
+  private config: QueryPerformanceConfig;
+
+  constructor(config: Partial<QueryPerformanceConfig> = {}) {
+    this.config = {
+      maxQueryLogLength: 1000,
+      ...config,
+    };
+  }
+
+  onQuery(
+    clientType: "writer" | "replica",
+    log: {
+      duration: number;
+      query: string;
+      params: string;
+      target: string;
+      timestamp: Date;
+    }
+  ) {
+    if (this.config.verySlowQueryThreshold === undefined) {
+      return;
+    }
+
+    const { duration, query, params, target, timestamp } = log;
+
+    // Only log very slow queries as errors
+    if (duration > this.config.verySlowQueryThreshold) {
+      // Truncate long queries for readability
+      const truncatedQuery =
+        query.length > this.config.maxQueryLogLength
+          ? query.substring(0, this.config.maxQueryLogLength) + "..."
+          : query;
+
+      logger.error("Prisma: very slow database query", {
+        clientType,
+        durationMs: duration,
+        query: truncatedQuery,
+        target,
+        timestamp,
+        paramCount: this.countParams(query),
+        hasParams: params !== "[]" && params !== "",
+      });
+    }
+  }
+
+  private countParams(query: string): number {
+    // Count the number of $1, $2, etc. parameters in the query
+    const paramMatches = query.match(/\$\d+/g);
+    return paramMatches ? paramMatches.length : 0;
+  }
+}
+
+export const queryPerformanceMonitor = new QueryPerformanceMonitor({
+  verySlowQueryThreshold: env.VERY_SLOW_QUERY_THRESHOLD_MS,
+});

apps/webapp/package.json

@@ -78,7 +78,7 @@
     "@opentelemetry/sdk-trace-node": "1.25.1",
     "@opentelemetry/semantic-conventions": "1.25.1",
     "@popperjs/core": "^2.11.8",
-    "@prisma/instrumentation": "^5.11.0",
+    "@prisma/instrumentation": "^6.14.0",
     "@radix-ui/react-accordion": "^1.2.11",
     "@radix-ui/react-alert-dialog": "^1.0.4",
     "@radix-ui/react-dialog": "^1.0.3",
@@ -137,6 +137,7 @@
     "dotenv": "^16.4.5",
     "effect": "^3.11.7",
     "emails": "workspace:*",
+    "eventsource": "^4.0.0",
     "evt": "^2.4.13",
     "express": "4.20.0",
     "framer-motion": "^10.12.11",
@@ -214,9 +215,9 @@
     "@remix-run/dev": "2.1.0",
     "@remix-run/eslint-config": "2.1.0",
     "@remix-run/testing": "^2.1.0",
+    "@sentry/cli": "2.50.2",
     "@swc/core": "^1.3.4",
     "@swc/helpers": "^0.4.11",
-    "@sentry/cli": "2.50.2",
     "@tailwindcss/forms": "^0.5.3",
     "@tailwindcss/typography": "^0.5.9",
     "@total-typescript/ts-reset": "^0.4.2",

docker/Dockerfile

@@ -36,9 +36,9 @@ RUN corepack enable
 ENV NODE_ENV production
 RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --prod --no-frozen-lockfile
 COPY --from=pruner --chown=node:node /triggerdotdev/internal-packages/database/prisma/schema.prisma /triggerdotdev/internal-packages/database/prisma/schema.prisma
-# RUN pnpm add @prisma/client@5.1.1 -w
+# RUN pnpm add @prisma/client@6.14.0 -w
 ENV NPM_CONFIG_IGNORE_WORKSPACE_ROOT_CHECK true
-RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpx prisma@5.4.1 generate --schema /triggerdotdev/internal-packages/database/prisma/schema.prisma
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpx prisma@6.14.0 generate --schema /triggerdotdev/internal-packages/database/prisma/schema.prisma
 
 ## Builder (builds the webapp)
 FROM base AS builder

docker/docker-compose.yml

@@ -77,6 +77,7 @@ services:
 
   clickhouse:
     image: bitnami/clickhouse:latest
+    restart: always
     container_name: clickhouse
     environment:
       CLICKHOUSE_ADMIN_USER: default

internal-packages/database/.gitignore

@@ -1,3 +1,5 @@
 node_modules
 # Ensure the .env symlink is not removed by accident
 !.env
+
+generated/prisma