diff --git a/.changeset/violet-llamas-roll.md b/.changeset/violet-llamas-roll.md new file mode 100644 index 0000000000..a2d90ee049 --- /dev/null +++ b/.changeset/violet-llamas-roll.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +fix: external traces now respect parent sampling, and prevent broken traces when there is no external trace context diff --git a/apps/webapp/app/runEngine/services/triggerTask.server.ts b/apps/webapp/app/runEngine/services/triggerTask.server.ts index 5ec3f29dd8..7db8449c33 100644 --- a/apps/webapp/app/runEngine/services/triggerTask.server.ts +++ b/apps/webapp/app/runEngine/services/triggerTask.server.ts @@ -385,7 +385,8 @@ export class RunEngineTriggerTaskService { const newExternalTraceparent = serializeTraceparent( parsedTraceparent.traceId, - parentSpanId ?? parsedTraceparent.spanId + parentSpanId ?? parsedTraceparent.spanId, + parsedTraceparent.traceFlags ); return { diff --git a/packages/core/src/v3/isomorphic/traceContext.ts b/packages/core/src/v3/isomorphic/traceContext.ts index b6754c4c6f..b91b661924 100644 --- a/packages/core/src/v3/isomorphic/traceContext.ts +++ b/packages/core/src/v3/isomorphic/traceContext.ts @@ -1,6 +1,6 @@ export function parseTraceparent( traceparent?: string -): { traceId: string; spanId: string } | undefined { +): { traceId: string; spanId: string; traceFlags?: string } | undefined { if (!traceparent) { return undefined; } @@ -11,7 +11,7 @@ export function parseTraceparent( return undefined; } - const [version, traceId, spanId] = parts; + const [version, traceId, spanId, traceFlags] = parts; if (version !== "00") { return undefined; @@ -21,9 +21,9 @@ export function parseTraceparent( return undefined; } - return { traceId, spanId }; + return { traceId, spanId, traceFlags }; } -export function serializeTraceparent(traceId: string, spanId: string) { - return `00-${traceId}-${spanId}-01`; +export function serializeTraceparent(traceId: string, spanId: string, traceFlags?: string) { + return `00-${traceId}-${spanId}-${traceFlags ?? "01"}`; } diff --git a/packages/core/src/v3/otel/tracingSDK.ts b/packages/core/src/v3/otel/tracingSDK.ts index 7fded903cd..7fcb82450c 100644 --- a/packages/core/src/v3/otel/tracingSDK.ts +++ b/packages/core/src/v3/otel/tracingSDK.ts @@ -1,4 +1,10 @@ -import { DiagConsoleLogger, DiagLogLevel, TracerProvider, diag } from "@opentelemetry/api"; +import { + DiagConsoleLogger, + DiagLogLevel, + TraceFlags, + TracerProvider, + diag, +} from "@opentelemetry/api"; import { logs } from "@opentelemetry/api-logs"; import { TraceState } from "@opentelemetry/core"; import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http"; @@ -21,7 +27,7 @@ import { SimpleSpanProcessor, SpanExporter, } from "@opentelemetry/sdk-trace-node"; -import { SemanticResourceAttributes } from "@opentelemetry/semantic-conventions"; +import { SemanticResourceAttributes, SEMATTRS_HTTP_URL } from "@opentelemetry/semantic-conventions"; import { VERSION } from "../../version.js"; import { OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT, @@ -287,17 +293,24 @@ function setLogLevel(level: TracingDiagnosticLogLevel) { } class ExternalSpanExporterWrapper { + private readonly _isExternallySampled: boolean; + constructor( private underlyingExporter: SpanExporter, private externalTraceId: string, private externalTraceContext: - | { traceId: string; spanId: string; tracestate?: string } + | { traceId: string; spanId: string; traceFlags: number; tracestate?: string } | undefined - ) {} + ) { + this._isExternallySampled = isTraceFlagSampled(externalTraceContext?.traceFlags); + } private transformSpan(span: ReadableSpan): ReadableSpan | undefined { - if (span.attributes[SemanticInternalAttributes.SPAN_PARTIAL]) { - // Skip partial spans + if (!this._isExternallySampled) { + return; + } + + if (isSpanInternalOnly(span)) { return; } @@ -325,8 +338,10 @@ class ExternalSpanExporterWrapper { traceState: this.externalTraceContext.tracestate ? new TraceState(this.externalTraceContext.tracestate) : undefined, - traceFlags: parentSpanContext?.traceFlags ?? 0, + traceFlags: this.externalTraceContext.traceFlags, }; + } else if (isAttemptSpan) { + parentSpanContext = undefined; } return { @@ -360,15 +375,25 @@ class ExternalSpanExporterWrapper { } class ExternalLogRecordExporterWrapper { + private readonly _isExternallySampled: boolean; + constructor( private underlyingExporter: LogRecordExporter, private externalTraceId: string, private externalTraceContext: - | { traceId: string; spanId: string; tracestate?: string } + | { traceId: string; spanId: string; tracestate?: string; traceFlags: number } | undefined - ) {} + ) { + this._isExternallySampled = isTraceFlagSampled(externalTraceContext?.traceFlags); + } export(logs: any[], resultCallback: (result: any) => void): void { + if (!this._isExternallySampled) { + this.underlyingExporter.export([], resultCallback); + + return; + } + const modifiedLogs = logs.map(this.transformLogRecord.bind(this)); this.underlyingExporter.export(modifiedLogs, resultCallback); @@ -410,3 +435,57 @@ class ExternalLogRecordExporterWrapper { }); } } + +function isSpanInternalOnly(span: ReadableSpan): boolean { + if (span.attributes[SemanticInternalAttributes.SPAN_PARTIAL]) { + // Skip partial spans + return true; + } + + const urlPath = span.attributes["url.path"]; + + if (typeof urlPath === "string" && urlPath === "/api/v1/usage/ingest") { + return true; + } + + const httpUrl = span.attributes[SEMATTRS_HTTP_URL] ?? span.attributes["url.full"]; + + const url = safeParseUrl(httpUrl); + + if (!url) { + return false; + } + + const internalHosts = [ + "api.trigger.dev", + "billing.trigger.dev", + "cloud.trigger.dev", + "engine.trigger.dev", + "platform.trigger.dev", + ]; + + return ( + internalHosts.some((host) => url.hostname.includes(host)) || + url.pathname.includes("/api/v1/usage/ingest") + ); +} + +function safeParseUrl(url: unknown): URL | undefined { + if (typeof url !== "string") { + return undefined; + } + + try { + return new URL(url); + } catch (e) { + return undefined; + } +} + +function isTraceFlagSampled(traceFlags?: number): boolean { + if (typeof traceFlags !== "number") { + return true; + } + + return (traceFlags & TraceFlags.SAMPLED) === TraceFlags.SAMPLED; +} diff --git a/packages/core/src/v3/traceContext/manager.ts b/packages/core/src/v3/traceContext/manager.ts index aa7f92b748..f61b7c7af7 100644 --- a/packages/core/src/v3/traceContext/manager.ts +++ b/packages/core/src/v3/traceContext/manager.ts @@ -1,5 +1,6 @@ import { Context, context, propagation, trace, TraceFlags } from "@opentelemetry/api"; import { TraceContextManager } from "./types.js"; +import { parseTraceParent } from "@opentelemetry/core"; export class StandardTraceContextManager implements TraceContextManager { public traceContext: Record = {}; @@ -39,7 +40,7 @@ export class StandardTraceContextManager implements TraceContextManager { const spanContext = { traceId: externalTraceContext.traceId, spanId: currentSpanContext.spanId, - traceFlags: TraceFlags.SAMPLED, + traceFlags: externalTraceContext.traceFlags, isRemote: true, }; @@ -60,15 +61,16 @@ function extractExternalTraceContext(traceContext: unknown) { : undefined; if ("traceparent" in traceContext && typeof traceContext.traceparent === "string") { - const [version, traceId, spanId] = traceContext.traceparent.split("-"); + const externalSpanContext = parseTraceParent(traceContext.traceparent); - if (!traceId || !spanId) { + if (!externalSpanContext) { return undefined; } return { - traceId, - spanId, + traceId: externalSpanContext.traceId, + spanId: externalSpanContext.spanId, + traceFlags: externalSpanContext.traceFlags, tracestate: tracestate, }; } diff --git a/packages/core/src/v3/traceContext/types.ts b/packages/core/src/v3/traceContext/types.ts index a1130cdf56..dc30cf6f47 100644 --- a/packages/core/src/v3/traceContext/types.ts +++ b/packages/core/src/v3/traceContext/types.ts @@ -8,6 +8,7 @@ export interface TraceContextManager { | { traceId: string; spanId: string; + traceFlags: number; tracestate?: string; } | undefined; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 15490dbedd..5c7af2cf01 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1962,6 +1962,12 @@ importers: '@opentelemetry/instrumentation': specifier: ^0.203.0 version: 0.203.0(@opentelemetry/api@1.9.0)(supports-color@10.0.0) + '@opentelemetry/instrumentation-http': + specifier: 0.203.0 + version: 0.203.0(@opentelemetry/api@1.9.0) + '@opentelemetry/instrumentation-undici': + specifier: 0.14.0 + version: 0.14.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-logs': specifier: ^0.203.0 version: 0.203.0(@opentelemetry/api@1.9.0) @@ -10340,6 +10346,21 @@ packages: - supports-color dev: false + /@opentelemetry/instrumentation-http@0.203.0(@opentelemetry/api@1.9.0): + resolution: {integrity: sha512-y3uQAcCOAwnO6vEuNVocmpVzG3PER6/YZqbPbbffDdJ9te5NkHEkfSMNzlC3+v7KlE+WinPGc3N7MR30G1HY2g==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.0.1(@opentelemetry/api@1.9.0) + '@opentelemetry/instrumentation': 0.203.0(@opentelemetry/api@1.9.0)(supports-color@10.0.0) + '@opentelemetry/semantic-conventions': 1.36.0 + forwarded-parse: 2.1.2 + transitivePeerDependencies: + - supports-color + dev: false + /@opentelemetry/instrumentation-http@0.49.1(@opentelemetry/api@1.4.1): resolution: {integrity: sha512-Yib5zrW2s0V8wTeUK/B3ZtpyP4ldgXj9L3Ws/axXrW1dW0/mEFKifK50MxMQK9g5NNJQS9dWH7rvcEGZdWdQDA==} engines: {node: '>=14'} @@ -10565,6 +10586,19 @@ packages: - supports-color dev: false + /@opentelemetry/instrumentation-undici@0.14.0(@opentelemetry/api@1.9.0): + resolution: {integrity: sha512-2HN+7ztxAReXuxzrtA3WboAKlfP5OsPA57KQn2AdYZbJ3zeRPcLXyW4uO/jpLE6PLm0QRtmeGCmfYpqRlwgSwg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.7.0 + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.0.1(@opentelemetry/api@1.9.0) + '@opentelemetry/instrumentation': 0.203.0(@opentelemetry/api@1.9.0)(supports-color@10.0.0) + transitivePeerDependencies: + - supports-color + dev: false + /@opentelemetry/instrumentation-undici@0.2.0(@opentelemetry/api@1.4.1): resolution: {integrity: sha512-RH9WdVRtpnyp8kvya2RYqKsJouPxvHl7jKPsIfrbL8u2QCKloAGi0uEqDHoOS15ZRYPQTDXZ7d8jSpUgSQmvpA==} engines: {node: '>=14'} diff --git a/references/d3-chat/package.json b/references/d3-chat/package.json index 22e5a9dae9..3b78c978e8 100644 --- a/references/d3-chat/package.json +++ b/references/d3-chat/package.json @@ -26,6 +26,8 @@ "@opentelemetry/api-logs": "^0.203.0", "@opentelemetry/exporter-logs-otlp-http": "0.203.0", "@opentelemetry/exporter-trace-otlp-http": "0.203.0", + "@opentelemetry/instrumentation-http": "0.203.0", + "@opentelemetry/instrumentation-undici": "0.14.0", "@opentelemetry/instrumentation": "^0.203.0", "@opentelemetry/sdk-logs": "^0.203.0", "@radix-ui/react-avatar": "^1.1.3", diff --git a/references/d3-chat/src/instrumentation.ts b/references/d3-chat/src/instrumentation.ts index ad3e547447..b4ca035bf7 100644 --- a/references/d3-chat/src/instrumentation.ts +++ b/references/d3-chat/src/instrumentation.ts @@ -1,5 +1,5 @@ import { registerOTel } from "@vercel/otel"; export function register() { - registerOTel({ serviceName: "d3-chat" }); + registerOTel({ serviceName: "d3-chat", traceSampler: "traceidratio" }); } diff --git a/references/d3-chat/trigger.config.ts b/references/d3-chat/trigger.config.ts index ac84181392..528215ac7d 100644 --- a/references/d3-chat/trigger.config.ts +++ b/references/d3-chat/trigger.config.ts @@ -3,11 +3,14 @@ import { pythonExtension } from "@trigger.dev/python/extension"; import { installPlaywrightChromium } from "./src/extensions/playwright"; import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; +import { HttpInstrumentation } from "@opentelemetry/instrumentation-http"; +import { UndiciInstrumentation } from "@opentelemetry/instrumentation-undici"; export default defineConfig({ project: process.env.TRIGGER_PROJECT_REF!, dirs: ["./src/trigger"], telemetry: { + instrumentations: [new HttpInstrumentation(), new UndiciInstrumentation()], logExporters: [ new OTLPLogExporter({ url: "https://api.axiom.co/v1/logs",