feat(webapp): add an emergency span cap for trace summary queries

A new optional TRACE_VIEW_EMERGENCY_SPAN_CAP env var clamps the trace
summary and detailed trace summary span limits on both event store
paths (ClickHouse and Postgres), covering the dashboard trace view and
the public run trace endpoint. Unset by default, so nothing changes
unless an operator sets it.

Author: ericallam

apps/webapp/app/env.server.ts

@@ -726,6 +726,9 @@ const EnvironmentSchema = z
     MAXIMUM_LIVE_RELOADING_EVENTS: z.coerce.number().int().default(1000),
     MAXIMUM_TRACE_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(25_000),
     MAXIMUM_TRACE_DETAILED_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(10_000),
+    // Emergency circuit breaker: when set, clamps the trace summary and detailed
+    // summary span limits on both event store paths to this value. Unset = disabled.
+    TRACE_VIEW_EMERGENCY_SPAN_CAP: z.coerce.number().int().positive().optional(),
     TASK_PAYLOAD_OFFLOAD_THRESHOLD: z.coerce.number().int().default(524_288), // 512KB
     BATCH_PAYLOAD_OFFLOAD_THRESHOLD: z.coerce.number().int().optional(), // Defaults to TASK_PAYLOAD_OFFLOAD_THRESHOLD if not set
     TASK_PAYLOAD_MAXIMUM_SIZE: z.coerce.number().int().default(3_145_728), // 3MB

apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts

@@ -2,6 +2,7 @@ import { ClickHouse } from "@internal/clickhouse";
 import { createHash } from "crypto";
 import { ClickhouseEventRepository } from "~/v3/eventRepository/clickhouseEventRepository.server";
 import { env } from "~/env.server";
+import { clampToEmergencySpanCap } from "~/v3/eventRepository/emergencySpanCap.server";
 import { singleton } from "~/utils/singleton";
 import type { OrganizationDataStoresRegistry } from "~/services/dataStores/organizationDataStoresRegistry.server";
 import { type IEventRepository } from "~/v3/eventRepository/eventRepository.types";
@@ -533,9 +534,12 @@ function buildEventRepository(store: string, clickhouse: ClickHouse): Clickhouse
         clickhouse,
         batchSize: env.EVENTS_CLICKHOUSE_BATCH_SIZE,
         flushInterval: env.EVENTS_CLICKHOUSE_FLUSH_INTERVAL_MS,
-        maximumTraceSummaryViewCount: env.EVENTS_CLICKHOUSE_MAX_TRACE_SUMMARY_VIEW_COUNT,
-        maximumTraceDetailedSummaryViewCount:
-          env.EVENTS_CLICKHOUSE_MAX_TRACE_DETAILED_SUMMARY_VIEW_COUNT,
+        maximumTraceSummaryViewCount: clampToEmergencySpanCap(
+          env.EVENTS_CLICKHOUSE_MAX_TRACE_SUMMARY_VIEW_COUNT
+        ),
+        maximumTraceDetailedSummaryViewCount: clampToEmergencySpanCap(
+          env.EVENTS_CLICKHOUSE_MAX_TRACE_DETAILED_SUMMARY_VIEW_COUNT
+        ),
         maximumLiveReloadingSetting: env.EVENTS_CLICKHOUSE_MAX_LIVE_RELOADING_SETTING,
         insertStrategy: env.EVENTS_CLICKHOUSE_INSERT_STRATEGY,
         waitForAsyncInsert: env.EVENTS_CLICKHOUSE_WAIT_FOR_ASYNC_INSERT === "1",
@@ -557,9 +561,12 @@ function buildEventRepository(store: string, clickhouse: ClickHouse): Clickhouse
         clickhouse: clickhouse,
         batchSize: env.EVENTS_CLICKHOUSE_BATCH_SIZE,
         flushInterval: env.EVENTS_CLICKHOUSE_FLUSH_INTERVAL_MS,
-        maximumTraceSummaryViewCount: env.EVENTS_CLICKHOUSE_MAX_TRACE_SUMMARY_VIEW_COUNT,
-        maximumTraceDetailedSummaryViewCount:
-          env.EVENTS_CLICKHOUSE_MAX_TRACE_DETAILED_SUMMARY_VIEW_COUNT,
+        maximumTraceSummaryViewCount: clampToEmergencySpanCap(
+          env.EVENTS_CLICKHOUSE_MAX_TRACE_SUMMARY_VIEW_COUNT
+        ),
+        maximumTraceDetailedSummaryViewCount: clampToEmergencySpanCap(
+          env.EVENTS_CLICKHOUSE_MAX_TRACE_DETAILED_SUMMARY_VIEW_COUNT
+        ),
         maximumLiveReloadingSetting: env.EVENTS_CLICKHOUSE_MAX_LIVE_RELOADING_SETTING,
         insertStrategy: env.EVENTS_CLICKHOUSE_INSERT_STRATEGY,
         waitForAsyncInsert: env.EVENTS_CLICKHOUSE_WAIT_FOR_ASYNC_INSERT === "1",

apps/webapp/app/v3/eventRepository/emergencySpanCap.server.ts

@@ -0,0 +1,8 @@
+import { env } from "~/env.server";
+
+// Emergency circuit breaker for trace views: when TRACE_VIEW_EMERGENCY_SPAN_CAP
+// is set, clamp a trace summary span limit to it. Unset = no clamping.
+export function clampToEmergencySpanCap(limit: number): number {
+  const cap = env.TRACE_VIEW_EMERGENCY_SPAN_CAP;
+  return cap === undefined ? limit : Math.min(limit, cap);
+}

apps/webapp/app/v3/taskEventStore.server.ts

@@ -2,6 +2,7 @@
 import { Prisma, TaskEvent } from "@trigger.dev/database";
 import type { PrismaClient, PrismaReplicaClient } from "~/db.server";
 import { env } from "~/env.server";
+import { clampToEmergencySpanCap } from "~/v3/eventRepository/emergencySpanCap.server";
 
 export type CommonTaskEvent = Omit<TaskEvent, "id">;
 export type TraceEvent = Pick<
@@ -192,7 +193,7 @@ export class TaskEventStore {
               : Prisma.empty
           }
         ORDER BY "startTime" ASC
-        LIMIT ${env.MAXIMUM_TRACE_SUMMARY_VIEW_COUNT}
+        LIMIT ${clampToEmergencySpanCap(env.MAXIMUM_TRACE_SUMMARY_VIEW_COUNT)}
       `;
     } else {
       return await this.readReplica.$queryRaw<TraceEvent[]>`
@@ -220,7 +221,7 @@ export class TaskEventStore {
               : Prisma.empty
           }
         ORDER BY "startTime" ASC
-        LIMIT ${env.MAXIMUM_TRACE_SUMMARY_VIEW_COUNT}
+        LIMIT ${clampToEmergencySpanCap(env.MAXIMUM_TRACE_SUMMARY_VIEW_COUNT)}
       `;
     }
   }
@@ -270,7 +271,7 @@ export class TaskEventStore {
               : Prisma.empty
           }
         ORDER BY "startTime" ASC
-        LIMIT ${env.MAXIMUM_TRACE_DETAILED_SUMMARY_VIEW_COUNT}
+        LIMIT ${clampToEmergencySpanCap(env.MAXIMUM_TRACE_DETAILED_SUMMARY_VIEW_COUNT)}
       `;
     } else {
       return await this.readReplica.$queryRaw<DetailedTraceEvent[]>`
@@ -299,7 +300,7 @@ export class TaskEventStore {
               : Prisma.empty
           }
         ORDER BY "startTime" ASC
-        LIMIT ${env.MAXIMUM_TRACE_DETAILED_SUMMARY_VIEW_COUNT}
+        LIMIT ${clampToEmergencySpanCap(env.MAXIMUM_TRACE_DETAILED_SUMMARY_VIEW_COUNT)}
       `;
     }
   }