feat(observability): export Trigger.dev telemetry to Grafana Cloud OTLP (#4583)

* feat(observability): export Trigger.dev telemetry to Grafana Cloud OTLP

Wire OTLP HTTP exporters for traces, logs, and metrics from the
Trigger.dev runtime to Grafana Cloud. Auth uses Basic with instance ID
and API token. Gated behind GRAFANA_OTLP_ENDPOINT, GRAFANA_INSTANCE_ID,
and GRAFANA_API_TOKEN — all three must be set together or all unset;
partial config throws at startup.

* improvement(observability): use OTLP HTTP/JSON for metrics for consistency with traces and logs

* feat(observability): tag Trigger.dev telemetry with deployment.environment.name

* improvement(observability): switch Grafana telemetry vars to OTLP-shaped trio

Author: TheodoreSpeaks

apps/sim/lib/core/config/env.ts

@@ -157,6 +157,9 @@ export const env = createEnv({
     LOG_LEVEL:                             z.enum(['DEBUG', 'INFO', 'WARN', 'ERROR']).optional(), // Minimum log level to display (defaults to ERROR in production, DEBUG in development)
     PROFOUND_API_KEY:                      z.string().min(1).optional(),           // Profound analytics API key
     PROFOUND_ENDPOINT:                     z.string().url().optional(),            // Profound analytics endpoint
+    GRAFANA_OTLP_ENDPOINT:                 z.string().url().optional(),            // Grafana Cloud OTLP HTTP gateway base URL (e.g., https://otlp-gateway-prod-us-east-0.grafana.net/otlp). Trigger.dev exporters append /v1/traces, /v1/logs, /v1/metrics.
+    GRAFANA_OTLP_HEADERS:                  z.string().min(1).optional(),           // Comma-separated key=value headers for OTLP requests (e.g., "Authorization=Basic <base64(instanceId:token)>"). Same format as the OTEL_EXPORTER_OTLP_HEADERS spec.
+    GRAFANA_DEPLOYMENT_ENVIRONMENT:        z.string().min(1).optional(),           // Deployment tier label (e.g., "production", "staging", "development"). Emitted as the stable `deployment.environment.name` resource attribute on Trigger.dev telemetry to match the rest of the Sim OTEL stack.
 
     // External Services
     BROWSERBASE_API_KEY:                   z.string().min(1).optional(),           // Browserbase API key for browser automation

apps/sim/package.json

@@ -65,6 +65,8 @@
     "@modelcontextprotocol/sdk": "1.29.0",
     "@monaco-editor/react": "4.7.0",
     "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/exporter-logs-otlp-http": "^0.217.0",
+    "@opentelemetry/exporter-metrics-otlp-http": "^0.217.0",
     "@opentelemetry/exporter-trace-otlp-http": "^0.217.0",
     "@opentelemetry/resources": "^2.7.0",
     "@opentelemetry/sdk-node": "^0.217.0",

apps/sim/trigger.config.ts

@@ -1,7 +1,64 @@
+import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http'
+import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'
+import { resourceFromAttributes } from '@opentelemetry/resources'
 import { additionalFiles, additionalPackages } from '@trigger.dev/build/extensions/core'
 import { defineConfig } from '@trigger.dev/sdk'
 import { env } from './lib/core/config/env'
 
+const grafanaEndpoint = env.GRAFANA_OTLP_ENDPOINT
+const grafanaHeaders = env.GRAFANA_OTLP_HEADERS
+const grafanaDeploymentEnvironment = env.GRAFANA_DEPLOYMENT_ENVIRONMENT
+const grafanaConfigured = Boolean(grafanaEndpoint || grafanaHeaders || grafanaDeploymentEnvironment)
+const grafanaFullyConfigured = Boolean(
+  grafanaEndpoint && grafanaHeaders && grafanaDeploymentEnvironment
+)
+
+if (grafanaConfigured && !grafanaFullyConfigured) {
+  throw new Error(
+    'Grafana OTLP telemetry is partially configured. Set GRAFANA_OTLP_ENDPOINT, GRAFANA_OTLP_HEADERS, and GRAFANA_DEPLOYMENT_ENVIRONMENT together, or leave all three unset.'
+  )
+}
+
+/**
+ * Parse OTLP headers per the OTEL spec format `key1=value1,key2=value2`.
+ * Values are URL-decoded; keys/values are trimmed; empty entries are skipped.
+ * @see https://opentelemetry.io/docs/specs/otel/protocol/exporter/
+ */
+function parseOtlpHeaders(raw: string): Record<string, string> {
+  const out: Record<string, string> = {}
+  for (const pair of raw.split(',')) {
+    const eq = pair.indexOf('=')
+    if (eq === -1) continue
+    const key = pair.slice(0, eq).trim()
+    const value = pair.slice(eq + 1).trim()
+    if (!key) continue
+    out[key] = decodeURIComponent(value)
+  }
+  return out
+}
+
+const grafanaTelemetry = grafanaFullyConfigured
+  ? (() => {
+      const baseUrl = grafanaEndpoint!.replace(/\/+$/, '')
+      const headers = parseOtlpHeaders(grafanaHeaders!)
+      if (Object.keys(headers).length === 0) {
+        throw new Error(
+          'GRAFANA_OTLP_HEADERS is set but yielded no valid key=value pairs. Expected format: "key1=value1,key2=value2".'
+        )
+      }
+      const resource = resourceFromAttributes({
+        'deployment.environment.name': grafanaDeploymentEnvironment!,
+      })
+      return {
+        exporters: [new OTLPTraceExporter({ url: `${baseUrl}/v1/traces`, headers })],
+        logExporters: [new OTLPLogExporter({ url: `${baseUrl}/v1/logs`, headers })],
+        metricExporters: [new OTLPMetricExporter({ url: `${baseUrl}/v1/metrics`, headers })],
+        resource,
+      }
+    })()
+  : undefined
+
 export default defineConfig({
   project: env.TRIGGER_PROJECT_ID!,
   runtime: 'node-22',
@@ -14,6 +71,7 @@ export default defineConfig({
     },
   },
   dirs: ['./background'],
+  ...(grafanaTelemetry ? { telemetry: grafanaTelemetry } : {}),
   build: {
     external: ['isolated-vm'],
     extensions: [