Skip to content

Commit 12e9027

Browse files
mydeamydea
committed
fix(vercel-ai): Ensure tool errors have correct trace connected
1 parent 27e60b7 commit 12e9027

File tree

6 files changed

+195
-18
lines changed

6 files changed

+195
-18
lines changed

‎dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-with-pii.mjs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,5 @@ Sentry.init({
77
tracesSampleRate: 1.0,
88
sendDefaultPii: true,
99
transport: loggingTransport,
10-
integrations: [Sentry.vercelAIIntegration({ force: true })],
10+
integrations: [Sentry.vercelAIIntegration()],
1111
});

‎dev-packages/node-integration-tests/suites/tracing/vercelai/instrument.mjs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,5 @@ Sentry.init({
66
release: '1.0',
77
tracesSampleRate: 1.0,
88
transport: loggingTransport,
9-
integrations: [Sentry.vercelAIIntegration({ force: true })],
9+
integrations: [Sentry.vercelAIIntegration()],
1010
});

‎dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error-in-tool.mjs

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
import * as Sentry from '@sentry/node';
2+
import { generateText } from 'ai';
3+
import { MockLanguageModelV1 } from 'ai/test';
4+
import { z } from 'zod';
5+
6+
async function run() {
7+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
8+
await generateText({
9+
model: new MockLanguageModelV1({
10+
doGenerate: async () => ({
11+
rawCall: { rawPrompt: null, rawSettings: {} },
12+
finishReason: 'tool-calls',
13+
usage: { promptTokens: 15, completionTokens: 25 },
14+
text: 'Tool call completed!',
15+
toolCalls: [
16+
{
17+
toolCallType: 'function',
18+
toolCallId: 'call-1',
19+
toolName: 'getWeather',
20+
args: '{ "location": "San Francisco" }',
21+
},
22+
],
23+
}),
24+
}),
25+
tools: {
26+
getWeather: {
27+
parameters: z.object({ location: z.string() }),
28+
execute: async args => {
29+
throw new Error('Error in tool');
30+
},
31+
},
32+
},
33+
prompt: 'What is the weather in San Francisco?',
34+
});
35+
});
36+
}
37+
38+
run();

‎dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts

Lines changed: 111 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import { afterAll, describe, expect } from 'vitest';
22
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner';
33

4-
// `ai` SDK only support Node 18+
54
describe('Vercel AI integration', () => {
65
afterAll(() => {
76
cleanupChildProcesses();
@@ -416,4 +415,115 @@ describe('Vercel AI integration', () => {
416415
await createRunner().expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE }).start().completed();
417416
});
418417
});
418+
419+
createEsmAndCjsTests(__dirname, 'scenario-error-in-tool.mjs', 'instrument.mjs', (createRunner, test) => {
420+
test('captures error in tool', async () => {
421+
const expectedTransaction = {
422+
transaction: 'main',
423+
spans: expect.arrayContaining([
424+
expect.objectContaining({
425+
data: {
426+
'vercel.ai.model.id': 'mock-model-id',
427+
'vercel.ai.model.provider': 'mock-provider',
428+
'vercel.ai.operationId': 'ai.generateText',
429+
'vercel.ai.pipeline.name': 'generateText',
430+
'vercel.ai.settings.maxRetries': 2,
431+
'vercel.ai.settings.maxSteps': 1,
432+
'vercel.ai.streaming': false,
433+
'gen_ai.response.model': 'mock-model-id',
434+
'operation.name': 'ai.generateText',
435+
'sentry.op': 'gen_ai.invoke_agent',
436+
'sentry.origin': 'auto.vercelai.otel',
437+
},
438+
description: 'generateText',
439+
op: 'gen_ai.invoke_agent',
440+
origin: 'auto.vercelai.otel',
441+
status: 'unknown_error',
442+
}),
443+
expect.objectContaining({
444+
data: {
445+
'vercel.ai.model.id': 'mock-model-id',
446+
'vercel.ai.model.provider': 'mock-provider',
447+
'vercel.ai.operationId': 'ai.generateText.doGenerate',
448+
'vercel.ai.pipeline.name': 'generateText.doGenerate',
449+
'vercel.ai.response.finishReason': 'tool-calls',
450+
'vercel.ai.response.id': expect.any(String),
451+
'vercel.ai.response.model': 'mock-model-id',
452+
'vercel.ai.response.timestamp': expect.any(String),
453+
'vercel.ai.settings.maxRetries': 2,
454+
'vercel.ai.streaming': false,
455+
'gen_ai.request.model': 'mock-model-id',
456+
'gen_ai.response.finish_reasons': ['tool-calls'],
457+
'gen_ai.response.id': expect.any(String),
458+
'gen_ai.response.model': 'mock-model-id',
459+
'gen_ai.system': 'mock-provider',
460+
'gen_ai.usage.input_tokens': 15,
461+
'gen_ai.usage.output_tokens': 25,
462+
'gen_ai.usage.total_tokens': 40,
463+
'operation.name': 'ai.generateText.doGenerate',
464+
'sentry.op': 'gen_ai.generate_text',
465+
'sentry.origin': 'auto.vercelai.otel',
466+
},
467+
description: 'generate_text mock-model-id',
468+
op: 'gen_ai.generate_text',
469+
origin: 'auto.vercelai.otel',
470+
status: 'ok',
471+
}),
472+
expect.objectContaining({
473+
data: {
474+
'vercel.ai.operationId': 'ai.toolCall',
475+
'gen_ai.tool.call.id': 'call-1',
476+
'gen_ai.tool.name': 'getWeather',
477+
'gen_ai.tool.type': 'function',
478+
'operation.name': 'ai.toolCall',
479+
'sentry.op': 'gen_ai.execute_tool',
480+
'sentry.origin': 'auto.vercelai.otel',
481+
},
482+
description: 'execute_tool getWeather',
483+
op: 'gen_ai.execute_tool',
484+
origin: 'auto.vercelai.otel',
485+
status: 'unknown_error',
486+
}),
487+
]),
488+
};
489+
490+
let traceId: string = 'unset-trace-id';
491+
let spanId: string = 'unset-span-id';
492+
493+
const expectedError = {
494+
contexts: {
495+
trace: {
496+
span_id: expect.any(String),
497+
trace_id: expect.any(String),
498+
},
499+
},
500+
exception: {
501+
values: expect.arrayContaining([
502+
expect.objectContaining({
503+
type: 'AI_ToolExecutionError',
504+
value: 'Error executing tool getWeather: Error in tool',
505+
}),
506+
]),
507+
},
508+
};
509+
510+
await createRunner()
511+
.expect({
512+
transaction: transaction => {
513+
expect(transaction).toMatchObject(expectedTransaction);
514+
traceId = transaction.contexts!.trace!.trace_id;
515+
spanId = transaction.contexts!.trace!.span_id;
516+
},
517+
})
518+
.expect({
519+
event: event => {
520+
expect(event).toMatchObject(expectedError);
521+
expect(event.contexts!.trace!.trace_id).toBe(traceId);
522+
expect(event.contexts!.trace!.span_id).toBe(spanId);
523+
},
524+
})
525+
.start()
526+
.completed();
527+
});
528+
});
419529
});

‎packages/node-core/src/integrations/onunhandledrejection.ts

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import type { Client, IntegrationFn, SeverityLevel } from '@sentry/core';
2-
import { captureException, consoleSandbox, defineIntegration, getClient } from '@sentry/core';
1+
import type { Client, IntegrationFn, SeverityLevel, Span } from '@sentry/core';
2+
import { captureException, consoleSandbox, defineIntegration, getClient, withActiveSpan } from '@sentry/core';
33
import { logAndExitProcess } from '../utils/errorhandling';
44

55
type UnhandledRejectionMode = 'none' | 'warn' | 'strict';
@@ -51,16 +51,26 @@ export function makeUnhandledPromiseHandler(
5151

5252
const level: SeverityLevel = options.mode === 'strict' ? 'fatal' : 'error';
5353

54-
captureException(reason, {
55-
originalException: promise,
56-
captureContext: {
57-
extra: { unhandledPromiseRejection: true },
58-
level,
59-
},
60-
mechanism: {
61-
handled: false,
62-
type: 'onunhandledrejection',
63-
},
54+
// this can be set in places where we cannot reliably get access to the active span/error
55+
// when the error bubbles up to this handler, we can use this to set the active span
56+
const activeSpanForError = (reason as { _sentry_active_span?: Span })._sentry_active_span;
57+
58+
const activeSpanWrapper = activeSpanForError
59+
? (fn: () => void) => withActiveSpan(activeSpanForError, fn)
60+
: (fn: () => void) => fn();
61+
62+
activeSpanWrapper(() => {
63+
captureException(reason, {
64+
originalException: promise,
65+
captureContext: {
66+
extra: { unhandledPromiseRejection: true },
67+
level,
68+
},
69+
mechanism: {
70+
handled: false,
71+
type: 'onunhandledrejection',
72+
},
73+
});
6474
});
6575

6676
handleRejection(reason, options.mode);

‎packages/node/src/integrations/tracing/vercelai/instrumentation.ts

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
import type { InstrumentationConfig, InstrumentationModuleDefinition } from '@opentelemetry/instrumentation';
22
import { InstrumentationBase, InstrumentationNodeModuleDefinition } from '@opentelemetry/instrumentation';
3-
import { getCurrentScope, SDK_VERSION } from '@sentry/core';
3+
import {
4+
addNonEnumerableProperty,
5+
getActiveSpan,
6+
getCurrentScope,
7+
handleCallbackErrors,
8+
SDK_VERSION,
9+
} from '@sentry/core';
410
import { INTEGRATION_NAME } from './constants';
511
import type { TelemetrySettings, VercelAiIntegration } from './types';
612

@@ -132,8 +138,21 @@ export class SentryVercelAiInstrumentation extends InstrumentationBase {
132138
recordOutputs,
133139
};
134140

135-
// @ts-expect-error we know that the method exists
136-
return originalMethod.apply(this, args);
141+
return handleCallbackErrors(
142+
() => {
143+
// @ts-expect-error we know that the method exists
144+
return originalMethod.apply(this, args);
145+
},
146+
error => {
147+
// This error bubbles up to unhandledrejection handler (if not handled before),
148+
// where we do not know the active span anymore
149+
// So to circumvent this, we set the active span on the error object
150+
// which is picked up by the unhandledrejection handler
151+
if (error && typeof error === 'object') {
152+
addNonEnumerableProperty(error, '_sentry_active_span', getActiveSpan());
153+
}
154+
},
155+
);
137156
};
138157
}
139158

0 commit comments

Comments
 (0)