Generate the external traceID correctly and exporter 3rd party logs with the external traceID as well

Author: ericallam

packages/cli-v3/src/entryPoints/dev-run-worker.ts

@@ -175,7 +175,8 @@ async function bootstrap() {
   const tracer = new TriggerTracer({ tracer: otelTracer, logger: otelLogger });
   const consoleInterceptor = new ConsoleInterceptor(
     otelLogger,
-    typeof config.enableConsoleLogging === "boolean" ? config.enableConsoleLogging : true
+    typeof config.enableConsoleLogging === "boolean" ? config.enableConsoleLogging : true,
+    typeof config.disableConsoleInterceptor === "boolean" ? config.disableConsoleInterceptor : false
   );
 
   const configLogLevel = triggerLogLevel ?? config.logLevel ?? "info";

packages/cli-v3/src/entryPoints/managed-run-worker.ts

@@ -173,7 +173,8 @@ async function bootstrap() {
   const tracer = new TriggerTracer({ tracer: otelTracer, logger: otelLogger });
   const consoleInterceptor = new ConsoleInterceptor(
     otelLogger,
-    typeof config.enableConsoleLogging === "boolean" ? config.enableConsoleLogging : true
+    typeof config.enableConsoleLogging === "boolean" ? config.enableConsoleLogging : true,
+    typeof config.disableConsoleInterceptor === "boolean" ? config.disableConsoleInterceptor : false
   );
 
   const configLogLevel = triggerLogLevel ?? config.logLevel ?? "info";

packages/core/src/v3/config.ts

@@ -139,6 +139,11 @@ export type TriggerConfig = {
    */
   enableConsoleLogging?: boolean;
 
+  /**
+   * Disable the console interceptor. This will prevent logs from being sent to the trigger.dev backend.
+   */
+  disableConsoleInterceptor?: boolean;
+
   build?: {
     /**
      * Add custom conditions to the esbuild build. For example, if you are importing `ai/rsc`, you'll need to add "react-server" condition.

packages/core/src/v3/consoleInterceptor.ts

@@ -10,12 +10,17 @@ import { clock } from "./clock-api.js";
 export class ConsoleInterceptor {
   constructor(
     private readonly logger: logsAPI.Logger,
-    private readonly sendToStdIO: boolean
+    private readonly sendToStdIO: boolean,
+    private readonly interceptingDisabled: boolean
   ) {}
 
   // Intercept the console and send logs to the OpenTelemetry logger
   // during the execution of the callback
   async intercept<T>(console: Console, callback: () => Promise<T>): Promise<T> {
+    if (this.interceptingDisabled) {
+      return await callback();
+    }
+
     // Save the original console methods
     const originalConsole = {
       log: console.log,

packages/core/src/v3/otel/tracingSDK.ts

@@ -1,4 +1,5 @@
 import { DiagConsoleLogger, DiagLogLevel, TracerProvider, diag } from "@opentelemetry/api";
+import { RandomIdGenerator } from "@opentelemetry/sdk-trace-base";
 import { logs } from "@opentelemetry/api-logs";
 import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http";
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
@@ -16,6 +17,7 @@ import {
   BatchLogRecordProcessor,
   LoggerProvider,
   LogRecordExporter,
+  ReadableLogRecord,
   SimpleLogRecordProcessor,
 } from "@opentelemetry/sdk-logs";
 import {
@@ -92,6 +94,8 @@ export type TracingSDKConfig = {
   diagLogLevel?: TracingDiagnosticLogLevel;
 };
 
+const idGenerator = new RandomIdGenerator();
+
 export class TracingSDK {
   public readonly asyncResourceDetector = new AsyncResourceDetector();
   private readonly _logProvider: LoggerProvider;
@@ -160,7 +164,7 @@ export class TracingSDK {
       )
     );
 
-    const externalTraceId = crypto.randomUUID();
+    const externalTraceId = idGenerator.generateTraceId();
 
     for (const exporter of config.exporters ?? []) {
       traceProvider.addSpanProcessor(
@@ -215,13 +219,22 @@ export class TracingSDK {
     for (const externalLogExporter of config.logExporters ?? []) {
       loggerProvider.addLogRecordProcessor(
         getEnvVar("OTEL_BATCH_PROCESSING_ENABLED") === "1"
-          ? new BatchLogRecordProcessor(externalLogExporter, {
-              maxExportBatchSize: parseInt(getEnvVar("OTEL_LOG_MAX_EXPORT_BATCH_SIZE") ?? "64"),
-              scheduledDelayMillis: parseInt(getEnvVar("OTEL_LOG_SCHEDULED_DELAY_MILLIS") ?? "200"),
-              exportTimeoutMillis: parseInt(getEnvVar("OTEL_LOG_EXPORT_TIMEOUT_MILLIS") ?? "30000"),
-              maxQueueSize: parseInt(getEnvVar("OTEL_LOG_MAX_QUEUE_SIZE") ?? "512"),
-            })
-          : new SimpleLogRecordProcessor(externalLogExporter)
+          ? new BatchLogRecordProcessor(
+              new ExternalLogRecordExporterWrapper(externalLogExporter, externalTraceId),
+              {
+                maxExportBatchSize: parseInt(getEnvVar("OTEL_LOG_MAX_EXPORT_BATCH_SIZE") ?? "64"),
+                scheduledDelayMillis: parseInt(
+                  getEnvVar("OTEL_LOG_SCHEDULED_DELAY_MILLIS") ?? "200"
+                ),
+                exportTimeoutMillis: parseInt(
+                  getEnvVar("OTEL_LOG_EXPORT_TIMEOUT_MILLIS") ?? "30000"
+                ),
+                maxQueueSize: parseInt(getEnvVar("OTEL_LOG_MAX_QUEUE_SIZE") ?? "512"),
+              }
+            )
+          : new SimpleLogRecordProcessor(
+              new ExternalLogRecordExporterWrapper(externalLogExporter, externalTraceId)
+            )
       );
     }
 
@@ -321,3 +334,50 @@ class ExternalSpanExporterWrapper {
       : Promise.resolve();
   }
 }
+
+class ExternalLogRecordExporterWrapper {
+  constructor(
+    private underlyingExporter: LogRecordExporter,
+    private externalTraceId: string
+  ) {}
+
+  export(logs: any[], resultCallback: (result: any) => void): void {
+    const modifiedLogs = logs.map(this.transformLogRecord.bind(this));
+
+    this.underlyingExporter.export(modifiedLogs, resultCallback);
+  }
+
+  shutdown(): Promise<void> {
+    return this.underlyingExporter.shutdown();
+  }
+
+  transformLogRecord(logRecord: ReadableLogRecord): ReadableLogRecord {
+    // If there's no spanContext, or if the externalTraceId is not set, return the original logRecord.
+    if (!logRecord.spanContext || !this.externalTraceId) {
+      return logRecord;
+    }
+
+    // Capture externalTraceId for use within the proxy's scope.
+    const { externalTraceId } = this;
+
+    return new Proxy(logRecord, {
+      get(target, prop, receiver) {
+        if (prop === "spanContext") {
+          // Intercept access to spanContext.
+          const originalSpanContext = target.spanContext;
+          // Ensure originalSpanContext exists (it should, due to the check above, but good for safety).
+          if (originalSpanContext) {
+            return {
+              ...originalSpanContext,
+              traceId: externalTraceId, // Override traceId.
+            };
+          }
+          // Fallback if, for some reason, originalSpanContext is undefined here.
+          return undefined;
+        }
+        // For all other properties, defer to the original object.
+        return Reflect.get(target, prop, receiver);
+      },
+    });
+  }
+}

pnpm-lock.yaml

@@ -28,6 +28,7 @@
     "@trigger.dev/react-hooks": "workspace:*",
     "@trigger.dev/sdk": "workspace:*",
     "@opentelemetry/exporter-logs-otlp-http": "0.52.1",
+    "@opentelemetry/exporter-trace-otlp-http": "0.52.1",
     "@vercel/postgres": "^0.10.0",
     "ai": "4.2.5",
     "class-variance-authority": "^0.7.1",

references/d3-chat/package.json

@@ -2,6 +2,7 @@ import { defineConfig } from "@trigger.dev/sdk";
 import { pythonExtension } from "@trigger.dev/python/extension";
 import { installPlaywrightChromium } from "./src/extensions/playwright";
 import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
 
 export default defineConfig({
   project: "proj_cdmymsrobxmcgjqzhdkq",
@@ -16,6 +17,15 @@ export default defineConfig({
         },
       }),
     ],
+    exporters: [
+      new OTLPTraceExporter({
+        url: "https://api.axiom.co/v1/traces",
+        headers: {
+          Authorization: `Bearer ${process.env.AXIOM_TOKEN}`,
+          "X-Axiom-Dataset": "d3-chat-tester",
+        },
+      }),
+    ],
   },
   maxDuration: 3600,
   build: {