diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index da99b394a8..c225b46588 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import { isValidDatabaseUrl } from "./utils/db"; import { isValidRegex } from "./utils/regex"; import { BoolEnv } from "./utils/boolEnv"; +import { OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT, OTEL_LINK_COUNT_LIMIT } from "@trigger.dev/core/v3"; const EnvironmentSchema = z.object({ NODE_ENV: z.union([z.literal("development"), z.literal("production"), z.literal("test")]), @@ -276,6 +277,15 @@ const EnvironmentSchema = z.object({ PROD_OTEL_LOG_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"), PROD_OTEL_LOG_MAX_QUEUE_SIZE: z.string().default("512"), + TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT: z.string().default("256"), + TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT: z.string().default("256"), + TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT: z.string().default("131072"), + TRIGGER_OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT: z.string().default("131072"), + TRIGGER_OTEL_SPAN_EVENT_COUNT_LIMIT: z.string().default("10"), + TRIGGER_OTEL_LINK_COUNT_LIMIT: z.string().default("2"), + TRIGGER_OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT: z.string().default("10"), + TRIGGER_OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT: z.string().default("10"), + CHECKPOINT_THRESHOLD_IN_MS: z.coerce.number().int().default(30000), // Internal OTEL environment variables diff --git a/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts b/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts index 66c1ddd9d1..ea6243f4a5 100644 --- a/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts +++ b/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts @@ -810,6 +810,7 @@ export const RuntimeEnvironmentForEnvRepoPayload = { apiKey: true, organizationId: true, branchName: true, + builtInEnvironmentVariableOverrides: true, }, } as const; @@ -1025,5 +1026,93 @@ async function resolveBuiltInProdVariables( async function resolveCommonBuiltInVariables( runtimeEnvironment: RuntimeEnvironmentForEnvRepo ): Promise> { - return []; + return [ + { + key: "TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT", + value: resolveBuiltInEnvironmentVariableOverrides( + "TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT", + runtimeEnvironment, + String(env.TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT) + ), + }, + { + key: "TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT", + value: resolveBuiltInEnvironmentVariableOverrides( + "TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT", + runtimeEnvironment, + String(env.TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT) + ), + }, + { + key: "TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT", + value: resolveBuiltInEnvironmentVariableOverrides( + "TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT", + runtimeEnvironment, + String(env.TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT) + ), + }, + { + key: "TRIGGER_OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT", + value: resolveBuiltInEnvironmentVariableOverrides( + "TRIGGER_OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT", + runtimeEnvironment, + String(env.TRIGGER_OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT) + ), + }, + { + key: "TRIGGER_OTEL_SPAN_EVENT_COUNT_LIMIT", + value: resolveBuiltInEnvironmentVariableOverrides( + "TRIGGER_OTEL_SPAN_EVENT_COUNT_LIMIT", + runtimeEnvironment, + String(env.TRIGGER_OTEL_SPAN_EVENT_COUNT_LIMIT) + ), + }, + { + key: "TRIGGER_OTEL_LINK_COUNT_LIMIT", + value: resolveBuiltInEnvironmentVariableOverrides( + "TRIGGER_OTEL_LINK_COUNT_LIMIT", + runtimeEnvironment, + String(env.TRIGGER_OTEL_LINK_COUNT_LIMIT) + ), + }, + { + key: "TRIGGER_OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT", + value: resolveBuiltInEnvironmentVariableOverrides( + "TRIGGER_OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT", + runtimeEnvironment, + String(env.TRIGGER_OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT) + ), + }, + { + key: "TRIGGER_OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT", + value: resolveBuiltInEnvironmentVariableOverrides( + "TRIGGER_OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT", + runtimeEnvironment, + String(env.TRIGGER_OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT) + ), + }, + ]; +} + +function resolveBuiltInEnvironmentVariableOverrides( + key: string, + runtimeEnvironment: RuntimeEnvironmentForEnvRepo, + defaultValue: string +) { + const overrides = runtimeEnvironment.builtInEnvironmentVariableOverrides; + + if (!overrides) { + return defaultValue; + } + + if ( + !Array.isArray(overrides) && + typeof overrides === "object" && + key in overrides && + typeof overrides[key] === "string" + ) { + return overrides[key]; + } + + return defaultValue; } diff --git a/apps/webapp/app/v3/otlpExporter.server.ts b/apps/webapp/app/v3/otlpExporter.server.ts index b87718f86e..888a6f46bb 100644 --- a/apps/webapp/app/v3/otlpExporter.server.ts +++ b/apps/webapp/app/v3/otlpExporter.server.ts @@ -39,7 +39,8 @@ class OTLPExporter { constructor( private readonly _eventRepository: EventRepository, - private readonly _verbose: boolean + private readonly _verbose: boolean, + private readonly _spanAttributeValueLengthLimit: number ) { this._tracer = trace.getTracer("otlp-exporter"); } @@ -52,7 +53,7 @@ class OTLPExporter { this.#logExportTracesVerbose(request); const events = this.#filterResourceSpans(request.resourceSpans).flatMap((resourceSpan) => { - return convertSpansToCreateableEvents(resourceSpan); + return convertSpansToCreateableEvents(resourceSpan, this._spanAttributeValueLengthLimit); }); const enrichedEvents = enrichCreatableEvents(events); @@ -79,7 +80,7 @@ class OTLPExporter { this.#logExportLogsVerbose(request); const events = this.#filterResourceLogs(request.resourceLogs).flatMap((resourceLog) => { - return convertLogsToCreateableEvents(resourceLog); + return convertLogsToCreateableEvents(resourceLog, this._spanAttributeValueLengthLimit); }); const enrichedEvents = enrichCreatableEvents(events); @@ -180,7 +181,10 @@ class OTLPExporter { } } -function convertLogsToCreateableEvents(resourceLog: ResourceLogs): Array { +function convertLogsToCreateableEvents( + resourceLog: ResourceLogs, + spanAttributeValueLengthLimit: number +): Array { const resourceAttributes = resourceLog.resource?.attributes ?? []; const resourceProperties = extractEventProperties(resourceAttributes); @@ -213,10 +217,10 @@ function convertLogsToCreateableEvents(resourceLog: ResourceLogs): Array { +function convertSpansToCreateableEvents( + resourceSpan: ResourceSpans, + spanAttributeValueLengthLimit: number +): Array { const resourceAttributes = resourceSpan.resource?.attributes ?? []; const resourceProperties = extractEventProperties(resourceAttributes); @@ -323,10 +330,10 @@ function convertSpansToCreateableEvents(resourceSpan: ResourceSpans): Array { + return isStringValue(attribute.value) + ? { + key: attribute.key, + value: { + stringValue: attribute.value.stringValue.slice(0, maximumLength), + }, + } + : attribute; + }); +} + export const otlpExporter = new OTLPExporter( eventRepository, - process.env.OTLP_EXPORTER_VERBOSE === "1" + process.env.OTLP_EXPORTER_VERBOSE === "1", + process.env.SERVER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT + ? parseInt(process.env.SERVER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT, 10) + : 8192 ); diff --git a/internal-packages/database/prisma/migrations/20250620144525_add_built_in_overrides_to_runtime_environment/migration.sql b/internal-packages/database/prisma/migrations/20250620144525_add_built_in_overrides_to_runtime_environment/migration.sql new file mode 100644 index 0000000000..04ee6a77a8 --- /dev/null +++ b/internal-packages/database/prisma/migrations/20250620144525_add_built_in_overrides_to_runtime_environment/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "RuntimeEnvironment" ADD COLUMN "builtInEnvironmentVariableOverrides" JSONB; \ No newline at end of file diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index c83bf6947b..3143b089f2 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -241,6 +241,9 @@ model RuntimeEnvironment { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt + /// Allows us to customize the built-in environment variables for a specific environment, like TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT + builtInEnvironmentVariableOverrides Json? + tunnelId String? backgroundWorkers BackgroundWorker[] diff --git a/packages/core/src/v3/limits.ts b/packages/core/src/v3/limits.ts index 212446faad..4cead21454 100644 --- a/packages/core/src/v3/limits.ts +++ b/packages/core/src/v3/limits.ts @@ -1,13 +1,46 @@ import { AttributeValue, Attributes } from "@opentelemetry/api"; +import { getEnvVar } from "./utils/getEnv.js"; + +function getOtelEnvVarLimit(key: string, defaultValue: number) { + const value = getEnvVar(key); + + if (!value) { + return defaultValue; + } + + return parseInt(value, 10); +} + +export const OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT = getOtelEnvVarLimit( + "TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT", + 256 +); +export const OTEL_LOG_ATTRIBUTE_COUNT_LIMIT = getOtelEnvVarLimit( + "TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT", + 256 +); +export const OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT = getOtelEnvVarLimit( + "TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT", + 131072 +); +export const OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT = getOtelEnvVarLimit( + "TRIGGER_OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT", + 131072 +); +export const OTEL_SPAN_EVENT_COUNT_LIMIT = getOtelEnvVarLimit( + "TRIGGER_OTEL_SPAN_EVENT_COUNT_LIMIT", + 10 +); +export const OTEL_LINK_COUNT_LIMIT = getOtelEnvVarLimit("TRIGGER_OTEL_LINK_COUNT_LIMIT", 2); +export const OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT = getOtelEnvVarLimit( + "TRIGGER_OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT", + 10 +); +export const OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT = getOtelEnvVarLimit( + "TRIGGER_OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT", + 10 +); -export const OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT = 256; -export const OTEL_LOG_ATTRIBUTE_COUNT_LIMIT = 256; -export const OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT = 1028; -export const OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT = 1028; -export const OTEL_SPAN_EVENT_COUNT_LIMIT = 10; -export const OTEL_LINK_COUNT_LIMIT = 2; -export const OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT = 10; -export const OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT = 10; export const OFFLOAD_IO_PACKET_LENGTH_LIMIT = 128 * 1024; export function imposeAttributeLimits(attributes: Attributes): Attributes {