fix: external traces now respect parent sampling, and prevent broken traces when there is no external trace context

Author: ericallam

apps/webapp/app/runEngine/services/triggerTask.server.ts

@@ -385,7 +385,8 @@ export class RunEngineTriggerTaskService {
 
     const newExternalTraceparent = serializeTraceparent(
       parsedTraceparent.traceId,
-      parentSpanId ?? parsedTraceparent.spanId
+      parentSpanId ?? parsedTraceparent.spanId,
+      parsedTraceparent.traceFlags
     );
 
     return {

packages/core/src/v3/isomorphic/traceContext.ts

@@ -1,6 +1,6 @@
 export function parseTraceparent(
   traceparent?: string
-): { traceId: string; spanId: string } | undefined {
+): { traceId: string; spanId: string; traceFlags?: string } | undefined {
   if (!traceparent) {
     return undefined;
   }
@@ -11,7 +11,7 @@ export function parseTraceparent(
     return undefined;
   }
 
-  const [version, traceId, spanId] = parts;
+  const [version, traceId, spanId, traceFlags] = parts;
 
   if (version !== "00") {
     return undefined;
@@ -21,9 +21,9 @@ export function parseTraceparent(
     return undefined;
   }
 
-  return { traceId, spanId };
+  return { traceId, spanId, traceFlags };
 }
 
-export function serializeTraceparent(traceId: string, spanId: string) {
-  return `00-${traceId}-${spanId}-01`;
+export function serializeTraceparent(traceId: string, spanId: string, traceFlags?: string) {
+  return `00-${traceId}-${spanId}-${traceFlags ?? "01"}`;
 }

packages/core/src/v3/otel/tracingSDK.ts

@@ -21,7 +21,7 @@ import {
   SimpleSpanProcessor,
   SpanExporter,
 } from "@opentelemetry/sdk-trace-node";
-import { SemanticResourceAttributes } from "@opentelemetry/semantic-conventions";
+import { SemanticResourceAttributes, SEMATTRS_HTTP_URL } from "@opentelemetry/semantic-conventions";
 import { VERSION } from "../../version.js";
 import {
   OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT,
@@ -287,17 +287,27 @@ function setLogLevel(level: TracingDiagnosticLogLevel) {
 }
 
 class ExternalSpanExporterWrapper {
+  private readonly _isExternallySampled: boolean;
+
   constructor(
     private underlyingExporter: SpanExporter,
     private externalTraceId: string,
     private externalTraceContext:
-      | { traceId: string; spanId: string; tracestate?: string }
+      | { traceId: string; spanId: string; tracestate?: string; traceFlags?: string }
       | undefined
-  ) {}
+  ) {
+    this._isExternallySampled =
+      typeof externalTraceContext?.traceFlags === "string"
+        ? externalTraceContext.traceFlags === "01"
+        : true;
+  }
 
   private transformSpan(span: ReadableSpan): ReadableSpan | undefined {
-    if (span.attributes[SemanticInternalAttributes.SPAN_PARTIAL]) {
-      // Skip partial spans
+    if (!this._isExternallySampled) {
+      return;
+    }
+
+    if (isSpanInternalOnly(span)) {
       return;
     }
 
@@ -325,8 +335,15 @@ class ExternalSpanExporterWrapper {
         traceState: this.externalTraceContext.tracestate
           ? new TraceState(this.externalTraceContext.tracestate)
           : undefined,
-        traceFlags: parentSpanContext?.traceFlags ?? 0,
+        traceFlags:
+          typeof this.externalTraceContext.traceFlags === "string"
+            ? this.externalTraceContext.traceFlags === "01"
+              ? 1
+              : 0
+            : parentSpanContext?.traceFlags ?? 0,
       };
+    } else if (isAttemptSpan) {
+      parentSpanContext = undefined;
     }
 
     return {
@@ -360,15 +377,28 @@ class ExternalSpanExporterWrapper {
 }
 
 class ExternalLogRecordExporterWrapper {
+  private readonly _isExternallySampled: boolean;
+
   constructor(
     private underlyingExporter: LogRecordExporter,
     private externalTraceId: string,
     private externalTraceContext:
-      | { traceId: string; spanId: string; tracestate?: string }
+      | { traceId: string; spanId: string; tracestate?: string; traceFlags?: string }
       | undefined
-  ) {}
+  ) {
+    this._isExternallySampled =
+      typeof externalTraceContext?.traceFlags === "string"
+        ? externalTraceContext.traceFlags === "01"
+        : true;
+  }
 
   export(logs: any[], resultCallback: (result: any) => void): void {
+    if (!this._isExternallySampled) {
+      this.underlyingExporter.export([], resultCallback);
+
+      return;
+    }
+
     const modifiedLogs = logs.map(this.transformLogRecord.bind(this));
 
     this.underlyingExporter.export(modifiedLogs, resultCallback);
@@ -410,3 +440,40 @@ class ExternalLogRecordExporterWrapper {
     });
   }
 }
+
+function isSpanInternalOnly(span: ReadableSpan): boolean {
+  if (span.attributes[SemanticInternalAttributes.SPAN_PARTIAL]) {
+    // Skip partial spans
+    return true;
+  }
+
+  const urlPath = span.attributes["url.path"];
+
+  if (typeof urlPath === "string" && urlPath === "/api/v1/usage/ingest") {
+    return true;
+  }
+
+  const httpUrl = span.attributes[SEMATTRS_HTTP_URL] ?? span.attributes["url.full"];
+
+  if (typeof httpUrl === "string" && httpUrl.includes("https://api.trigger.dev")) {
+    return true;
+  }
+
+  if (typeof httpUrl === "string" && httpUrl.includes("https://billing.trigger.dev")) {
+    return true;
+  }
+
+  if (typeof httpUrl === "string" && httpUrl.includes("https://cloud.trigger.dev")) {
+    return true;
+  }
+
+  if (typeof httpUrl === "string" && httpUrl.includes("https://engine.trigger.dev")) {
+    return true;
+  }
+
+  if (typeof httpUrl === "string" && httpUrl.includes("/api/v1/usage/ingest")) {
+    return true;
+  }
+
+  return false;
+}

packages/core/src/v3/traceContext/manager.ts

@@ -39,7 +39,12 @@ export class StandardTraceContextManager implements TraceContextManager {
     const spanContext = {
       traceId: externalTraceContext.traceId,
       spanId: currentSpanContext.spanId,
-      traceFlags: TraceFlags.SAMPLED,
+      traceFlags:
+        typeof externalTraceContext.traceFlags === "string"
+          ? externalTraceContext.traceFlags === "01"
+            ? TraceFlags.SAMPLED
+            : TraceFlags.NONE
+          : TraceFlags.SAMPLED,
       isRemote: true,
     };
 
@@ -60,7 +65,7 @@ function extractExternalTraceContext(traceContext: unknown) {
       : undefined;
 
   if ("traceparent" in traceContext && typeof traceContext.traceparent === "string") {
-    const [version, traceId, spanId] = traceContext.traceparent.split("-");
+    const [version, traceId, spanId, traceFlags] = traceContext.traceparent.split("-");
 
     if (!traceId || !spanId) {
       return undefined;
@@ -69,6 +74,7 @@ function extractExternalTraceContext(traceContext: unknown) {
     return {
       traceId,
       spanId,
+      traceFlags,
       tracestate: tracestate,
     };
   }

packages/core/src/v3/traceContext/types.ts

@@ -8,6 +8,7 @@ export interface TraceContextManager {
     | {
         traceId: string;
         spanId: string;
+        traceFlags?: string;
         tracestate?: string;
       }
     | undefined;

pnpm-lock.yaml

@@ -26,6 +26,8 @@
     "@opentelemetry/api-logs": "^0.203.0",
     "@opentelemetry/exporter-logs-otlp-http": "0.203.0",
     "@opentelemetry/exporter-trace-otlp-http": "0.203.0",
+    "@opentelemetry/instrumentation-http": "0.203.0",
+    "@opentelemetry/instrumentation-undici": "0.14.0",
     "@opentelemetry/instrumentation": "^0.203.0",
     "@opentelemetry/sdk-logs": "^0.203.0",
     "@radix-ui/react-avatar": "^1.1.3",

references/d3-chat/package.json

@@ -1,5 +1,5 @@
 import { registerOTel } from "@vercel/otel";
 
 export function register() {
-  registerOTel({ serviceName: "d3-chat" });
+  registerOTel({ serviceName: "d3-chat", traceSampler: "traceidratio" });
 }

references/d3-chat/src/instrumentation.ts

@@ -3,11 +3,14 @@ import { pythonExtension } from "@trigger.dev/python/extension";
 import { installPlaywrightChromium } from "./src/extensions/playwright";
 import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http";
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
+import { HttpInstrumentation } from "@opentelemetry/instrumentation-http";
+import { UndiciInstrumentation } from "@opentelemetry/instrumentation-undici";
 
 export default defineConfig({
   project: process.env.TRIGGER_PROJECT_REF!,
   dirs: ["./src/trigger"],
   telemetry: {
+    instrumentations: [new HttpInstrumentation(), new UndiciInstrumentation()],
     logExporters: [
       new OTLPLogExporter({
         url: "https://api.axiom.co/v1/logs",