diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/instrument-with-pii.mjs new file mode 100644 index 000000000000..b798e21228f5 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/instrument-with-pii.mjs @@ -0,0 +1,11 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + sendDefaultPii: true, + transport: loggingTransport, + integrations: [Sentry.vercelAIIntegration()], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/instrument.mjs new file mode 100644 index 000000000000..5e898ee1949d --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/instrument.mjs @@ -0,0 +1,10 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + integrations: [Sentry.vercelAIIntegration()], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario-error-in-tool.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario-error-in-tool.mjs new file mode 100644 index 000000000000..9ea18401ac35 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario-error-in-tool.mjs @@ -0,0 +1,41 @@ +import * as Sentry from '@sentry/node'; +import { generateText, tool } from 'ai'; +import { MockLanguageModelV3 } from 'ai/test'; +import { z } from 'zod'; + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + await generateText({ + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, + usage: { + inputTokens: { total: 15, noCache: 15, cached: 0 }, + outputTokens: { total: 25, noCache: 25, cached: 0 }, + totalTokens: { total: 40, noCache: 40, cached: 0 }, + }, + content: [ + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'getWeather', + input: JSON.stringify({ location: 'San Francisco' }), + }, + ], + warnings: [], + }), + }), + tools: { + getWeather: tool({ + inputSchema: z.object({ location: z.string() }), + execute: async () => { + throw new Error('Error in tool'); + }, + }), + }, + prompt: 'What is the weather in San Francisco?', + }); + }); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario.mjs new file mode 100644 index 000000000000..66233d1dabe5 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario.mjs @@ -0,0 +1,92 @@ +import * as Sentry from '@sentry/node'; +import { generateText, tool } from 'ai'; +import { MockLanguageModelV3 } from 'ai/test'; +import { z } from 'zod'; + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + await generateText({ + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 10, noCache: 10, cached: 0 }, + outputTokens: { total: 20, noCache: 20, cached: 0 }, + totalTokens: { total: 30, noCache: 30, cached: 0 }, + }, + content: [{ type: 'text', text: 'First span here!' }], + warnings: [], + }), + }), + prompt: 'Where is the first span?', + }); + + // This span should have input and output prompts attached because telemetry is explicitly enabled. + await generateText({ + experimental_telemetry: { isEnabled: true }, + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 10, noCache: 10, cached: 0 }, + outputTokens: { total: 20, noCache: 20, cached: 0 }, + totalTokens: { total: 30, noCache: 30, cached: 0 }, + }, + content: [{ type: 'text', text: 'Second span here!' }], + warnings: [], + }), + }), + prompt: 'Where is the second span?', + }); + + // This span should include tool calls and tool results + await generateText({ + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, + usage: { + inputTokens: { total: 15, noCache: 15, cached: 0 }, + outputTokens: { total: 25, noCache: 25, cached: 0 }, + totalTokens: { total: 40, noCache: 40, cached: 0 }, + }, + content: [ + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'getWeather', + input: JSON.stringify({ location: 'San Francisco' }), + }, + ], + warnings: [], + }), + }), + tools: { + getWeather: tool({ + inputSchema: z.object({ location: z.string() }), + execute: async ({ location }) => `Weather in ${location}: Sunny, 72°F`, + }), + }, + prompt: 'What is the weather in San Francisco?', + }); + + // This span should not be captured because we've disabled telemetry + await generateText({ + experimental_telemetry: { isEnabled: false }, + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 10, noCache: 10, cached: 0 }, + outputTokens: { total: 20, noCache: 20, cached: 0 }, + totalTokens: { total: 30, noCache: 30, cached: 0 }, + }, + content: [{ type: 'text', text: 'Third span here!' }], + warnings: [], + }), + }), + prompt: 'Where is the third span?', + }); + }); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/test.ts new file mode 100644 index 000000000000..98a16618d77d --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/test.ts @@ -0,0 +1,577 @@ +import type { Event } from '@sentry/node'; +import { afterAll, describe, expect } from 'vitest'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; + +describe('Vercel AI integration (V6)', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE = { + transaction: 'main', + spans: expect.arrayContaining([ + // First span - no telemetry config, should enable telemetry but not record inputs/outputs when sendDefaultPii: false + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.request.model': 'mock-model-id', + 'vercel.ai.model.provider': 'mock-provider', + 'vercel.ai.operationId': 'ai.generateText', + 'vercel.ai.pipeline.name': 'generateText', + 'vercel.ai.request.headers.user-agent': expect.any(String), + 'vercel.ai.response.finishReason': 'stop', + 'vercel.ai.settings.maxRetries': 2, + 'vercel.ai.streaming': false, + 'gen_ai.response.model': 'mock-model-id', + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 20, + 'gen_ai.usage.total_tokens': 30, + 'gen_ai.operation.name': 'ai.generateText', + 'sentry.op': 'gen_ai.invoke_agent', + 'sentry.origin': 'auto.vercelai.otel', + }), + description: 'generateText', + op: 'gen_ai.invoke_agent', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + // Second span - explicitly enabled telemetry but recordInputs/recordOutputs not set, should not record when sendDefaultPii: false + expect.objectContaining({ + data: expect.objectContaining({ + 'sentry.origin': 'auto.vercelai.otel', + 'sentry.op': 'gen_ai.generate_text', + 'gen_ai.operation.name': 'ai.generateText.doGenerate', + 'vercel.ai.operationId': 'ai.generateText.doGenerate', + 'vercel.ai.model.provider': 'mock-provider', + 'vercel.ai.request.headers.user-agent': expect.any(String), + 'gen_ai.request.model': 'mock-model-id', + 'vercel.ai.settings.maxRetries': 2, + 'gen_ai.system': 'mock-provider', + 'vercel.ai.pipeline.name': 'generateText.doGenerate', + 'vercel.ai.streaming': false, + 'vercel.ai.response.finishReason': 'stop', + 'vercel.ai.response.model': 'mock-model-id', + 'vercel.ai.response.id': expect.any(String), + 'vercel.ai.response.timestamp': expect.any(String), + 'gen_ai.response.finish_reasons': ['stop'], + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 20, + 'gen_ai.response.id': expect.any(String), + 'gen_ai.response.model': 'mock-model-id', + 'gen_ai.usage.total_tokens': 30, + }), + description: 'generate_text mock-model-id', + op: 'gen_ai.generate_text', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + // Third span - explicit telemetry enabled, should record inputs/outputs regardless of sendDefaultPii + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.request.model': 'mock-model-id', + 'vercel.ai.model.provider': 'mock-provider', + 'vercel.ai.operationId': 'ai.generateText', + 'vercel.ai.pipeline.name': 'generateText', + 'vercel.ai.prompt': '{"prompt":"Where is the second span?"}', + 'vercel.ai.request.headers.user-agent': expect.any(String), + 'vercel.ai.response.finishReason': 'stop', + 'gen_ai.response.text': expect.any(String), + 'vercel.ai.settings.maxRetries': 2, + 'vercel.ai.streaming': false, + 'gen_ai.prompt': '{"prompt":"Where is the second span?"}', + 'gen_ai.request.messages': '[{"role":"user","content":"Where is the second span?"}]', + 'gen_ai.response.model': 'mock-model-id', + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 20, + 'gen_ai.usage.total_tokens': 30, + 'gen_ai.operation.name': 'ai.generateText', + 'sentry.op': 'gen_ai.invoke_agent', + 'sentry.origin': 'auto.vercelai.otel', + }), + description: 'generateText', + op: 'gen_ai.invoke_agent', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + // Fourth span - doGenerate for explicit telemetry enabled call + expect.objectContaining({ + data: expect.objectContaining({ + 'sentry.origin': 'auto.vercelai.otel', + 'sentry.op': 'gen_ai.generate_text', + 'gen_ai.operation.name': 'ai.generateText.doGenerate', + 'vercel.ai.operationId': 'ai.generateText.doGenerate', + 'vercel.ai.model.provider': 'mock-provider', + 'vercel.ai.request.headers.user-agent': expect.any(String), + 'gen_ai.request.model': 'mock-model-id', + 'vercel.ai.settings.maxRetries': 2, + 'gen_ai.system': 'mock-provider', + 'vercel.ai.pipeline.name': 'generateText.doGenerate', + 'vercel.ai.streaming': false, + 'vercel.ai.response.finishReason': 'stop', + 'vercel.ai.response.model': 'mock-model-id', + 'vercel.ai.response.id': expect.any(String), + 'gen_ai.response.text': expect.any(String), + 'vercel.ai.response.timestamp': expect.any(String), + 'gen_ai.request.messages': expect.any(String), + 'gen_ai.response.finish_reasons': ['stop'], + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 20, + 'gen_ai.response.id': expect.any(String), + 'gen_ai.response.model': 'mock-model-id', + 'gen_ai.usage.total_tokens': 30, + }), + description: 'generate_text mock-model-id', + op: 'gen_ai.generate_text', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + // Fifth span - tool call generateText span + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.request.model': 'mock-model-id', + 'vercel.ai.model.provider': 'mock-provider', + 'vercel.ai.operationId': 'ai.generateText', + 'vercel.ai.pipeline.name': 'generateText', + 'vercel.ai.request.headers.user-agent': expect.any(String), + 'vercel.ai.response.finishReason': 'tool-calls', + 'vercel.ai.settings.maxRetries': 2, + 'vercel.ai.streaming': false, + 'gen_ai.response.model': 'mock-model-id', + 'gen_ai.usage.input_tokens': 15, + 'gen_ai.usage.output_tokens': 25, + 'gen_ai.usage.total_tokens': 40, + 'gen_ai.operation.name': 'ai.generateText', + 'sentry.op': 'gen_ai.invoke_agent', + 'sentry.origin': 'auto.vercelai.otel', + }), + description: 'generateText', + op: 'gen_ai.invoke_agent', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + // Sixth span - tool call doGenerate span + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.request.model': 'mock-model-id', + 'vercel.ai.model.provider': 'mock-provider', + 'vercel.ai.operationId': 'ai.generateText.doGenerate', + 'vercel.ai.pipeline.name': 'generateText.doGenerate', + 'vercel.ai.request.headers.user-agent': expect.any(String), + 'vercel.ai.response.finishReason': 'tool-calls', + 'vercel.ai.response.id': expect.any(String), + 'vercel.ai.response.model': 'mock-model-id', + 'vercel.ai.response.timestamp': expect.any(String), + 'vercel.ai.settings.maxRetries': 2, + 'vercel.ai.streaming': false, + 'gen_ai.response.finish_reasons': ['tool-calls'], + 'gen_ai.response.id': expect.any(String), + 'gen_ai.response.model': 'mock-model-id', + 'gen_ai.system': 'mock-provider', + 'gen_ai.usage.input_tokens': 15, + 'gen_ai.usage.output_tokens': 25, + 'gen_ai.usage.total_tokens': 40, + 'gen_ai.operation.name': 'ai.generateText.doGenerate', + 'sentry.op': 'gen_ai.generate_text', + 'sentry.origin': 'auto.vercelai.otel', + }), + description: 'generate_text mock-model-id', + op: 'gen_ai.generate_text', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + // Seventh span - tool call execution span + expect.objectContaining({ + data: expect.objectContaining({ + 'vercel.ai.operationId': 'ai.toolCall', + 'gen_ai.tool.call.id': 'call-1', + 'gen_ai.tool.name': 'getWeather', + 'gen_ai.tool.type': 'function', + 'gen_ai.operation.name': 'ai.toolCall', + 'sentry.op': 'gen_ai.execute_tool', + 'sentry.origin': 'auto.vercelai.otel', + }), + description: 'execute_tool getWeather', + op: 'gen_ai.execute_tool', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + ]), + }; + + const EXPECTED_AVAILABLE_TOOLS_JSON = + '[{"type":"function","name":"getWeather","inputSchema":{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","properties":{"location":{"type":"string"}},"required":["location"],"additionalProperties":false}}]'; + + const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = { + transaction: 'main', + spans: expect.arrayContaining([ + // First span - no telemetry config, should enable telemetry AND record inputs/outputs when sendDefaultPii: true + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.request.model': 'mock-model-id', + 'vercel.ai.model.provider': 'mock-provider', + 'vercel.ai.operationId': 'ai.generateText', + 'vercel.ai.pipeline.name': 'generateText', + 'vercel.ai.prompt': '{"prompt":"Where is the first span?"}', + 'vercel.ai.request.headers.user-agent': expect.any(String), + 'gen_ai.request.messages': '[{"role":"user","content":"Where is the first span?"}]', + 'vercel.ai.response.finishReason': 'stop', + 'gen_ai.response.text': 'First span here!', + 'vercel.ai.settings.maxRetries': 2, + 'vercel.ai.streaming': false, + 'gen_ai.prompt': '{"prompt":"Where is the first span?"}', + 'gen_ai.response.model': 'mock-model-id', + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 20, + 'gen_ai.usage.total_tokens': 30, + 'gen_ai.operation.name': 'ai.generateText', + 'sentry.op': 'gen_ai.invoke_agent', + 'sentry.origin': 'auto.vercelai.otel', + }), + description: 'generateText', + op: 'gen_ai.invoke_agent', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + // Second span - doGenerate for first call, should also include input/output fields when sendDefaultPii: true + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.request.model': 'mock-model-id', + 'vercel.ai.model.provider': 'mock-provider', + 'vercel.ai.operationId': 'ai.generateText.doGenerate', + 'vercel.ai.pipeline.name': 'generateText.doGenerate', + 'vercel.ai.request.headers.user-agent': expect.any(String), + 'gen_ai.request.messages': '[{"role":"user","content":[{"type":"text","text":"Where is the first span?"}]}]', + 'vercel.ai.response.finishReason': 'stop', + 'vercel.ai.response.id': expect.any(String), + 'vercel.ai.response.model': 'mock-model-id', + 'gen_ai.response.text': 'First span here!', + 'vercel.ai.response.timestamp': expect.any(String), + 'vercel.ai.settings.maxRetries': 2, + 'vercel.ai.streaming': false, + 'gen_ai.response.finish_reasons': ['stop'], + 'gen_ai.response.id': expect.any(String), + 'gen_ai.response.model': 'mock-model-id', + 'gen_ai.system': 'mock-provider', + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 20, + 'gen_ai.usage.total_tokens': 30, + 'gen_ai.operation.name': 'ai.generateText.doGenerate', + 'sentry.op': 'gen_ai.generate_text', + 'sentry.origin': 'auto.vercelai.otel', + }), + description: 'generate_text mock-model-id', + op: 'gen_ai.generate_text', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + // Third span - explicitly enabled telemetry, should record inputs/outputs regardless of sendDefaultPii + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.request.model': 'mock-model-id', + 'vercel.ai.model.provider': 'mock-provider', + 'vercel.ai.operationId': 'ai.generateText', + 'vercel.ai.pipeline.name': 'generateText', + 'vercel.ai.prompt': '{"prompt":"Where is the second span?"}', + 'vercel.ai.request.headers.user-agent': expect.any(String), + 'gen_ai.request.messages': '[{"role":"user","content":"Where is the second span?"}]', + 'vercel.ai.response.finishReason': 'stop', + 'gen_ai.response.text': expect.any(String), + 'vercel.ai.settings.maxRetries': 2, + 'vercel.ai.streaming': false, + 'gen_ai.prompt': '{"prompt":"Where is the second span?"}', + 'gen_ai.response.model': 'mock-model-id', + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 20, + 'gen_ai.usage.total_tokens': 30, + 'gen_ai.operation.name': 'ai.generateText', + 'sentry.op': 'gen_ai.invoke_agent', + 'sentry.origin': 'auto.vercelai.otel', + }), + description: 'generateText', + op: 'gen_ai.invoke_agent', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + // Fourth span - doGenerate for explicitly enabled telemetry call + expect.objectContaining({ + data: expect.objectContaining({ + 'sentry.origin': 'auto.vercelai.otel', + 'sentry.op': 'gen_ai.generate_text', + 'gen_ai.operation.name': 'ai.generateText.doGenerate', + 'vercel.ai.operationId': 'ai.generateText.doGenerate', + 'vercel.ai.model.provider': 'mock-provider', + 'vercel.ai.request.headers.user-agent': expect.any(String), + 'gen_ai.request.model': 'mock-model-id', + 'vercel.ai.settings.maxRetries': 2, + 'gen_ai.system': 'mock-provider', + 'vercel.ai.pipeline.name': 'generateText.doGenerate', + 'vercel.ai.streaming': false, + 'vercel.ai.response.finishReason': 'stop', + 'vercel.ai.response.model': 'mock-model-id', + 'vercel.ai.response.id': expect.any(String), + 'gen_ai.response.text': expect.any(String), + 'vercel.ai.response.timestamp': expect.any(String), + 'gen_ai.request.messages': expect.any(String), + 'gen_ai.response.finish_reasons': ['stop'], + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 20, + 'gen_ai.response.id': expect.any(String), + 'gen_ai.response.model': 'mock-model-id', + 'gen_ai.usage.total_tokens': 30, + }), + description: 'generate_text mock-model-id', + op: 'gen_ai.generate_text', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + // Fifth span - tool call generateText span (should include prompts when sendDefaultPii: true) + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.request.model': 'mock-model-id', + 'vercel.ai.model.provider': 'mock-provider', + 'vercel.ai.operationId': 'ai.generateText', + 'vercel.ai.pipeline.name': 'generateText', + 'vercel.ai.prompt': '{"prompt":"What is the weather in San Francisco?"}', + 'vercel.ai.request.headers.user-agent': expect.any(String), + 'gen_ai.request.messages': '[{"role":"user","content":"What is the weather in San Francisco?"}]', + 'vercel.ai.response.finishReason': 'tool-calls', + 'gen_ai.response.tool_calls': expect.any(String), + 'vercel.ai.settings.maxRetries': 2, + 'vercel.ai.streaming': false, + 'gen_ai.prompt': '{"prompt":"What is the weather in San Francisco?"}', + 'gen_ai.response.model': 'mock-model-id', + 'gen_ai.usage.input_tokens': 15, + 'gen_ai.usage.output_tokens': 25, + 'gen_ai.usage.total_tokens': 40, + 'gen_ai.operation.name': 'ai.generateText', + 'sentry.op': 'gen_ai.invoke_agent', + 'sentry.origin': 'auto.vercelai.otel', + }), + description: 'generateText', + op: 'gen_ai.invoke_agent', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + // Sixth span - tool call doGenerate span (should include prompts when sendDefaultPii: true) + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.request.model': 'mock-model-id', + 'vercel.ai.model.provider': 'mock-provider', + 'vercel.ai.operationId': 'ai.generateText.doGenerate', + 'vercel.ai.pipeline.name': 'generateText.doGenerate', + 'vercel.ai.request.headers.user-agent': expect.any(String), + 'gen_ai.request.messages': expect.any(String), + 'vercel.ai.prompt.toolChoice': expect.any(String), + 'gen_ai.request.available_tools': EXPECTED_AVAILABLE_TOOLS_JSON, + 'vercel.ai.response.finishReason': 'tool-calls', + 'vercel.ai.response.id': expect.any(String), + 'vercel.ai.response.model': 'mock-model-id', + // 'gen_ai.response.text': 'Tool call completed!', // TODO: look into why this is not being set + 'vercel.ai.response.timestamp': expect.any(String), + 'gen_ai.response.tool_calls': expect.any(String), + 'vercel.ai.settings.maxRetries': 2, + 'vercel.ai.streaming': false, + 'gen_ai.response.finish_reasons': ['tool-calls'], + 'gen_ai.response.id': expect.any(String), + 'gen_ai.response.model': 'mock-model-id', + 'gen_ai.system': 'mock-provider', + 'gen_ai.usage.input_tokens': 15, + 'gen_ai.usage.output_tokens': 25, + 'gen_ai.usage.total_tokens': 40, + 'gen_ai.operation.name': 'ai.generateText.doGenerate', + 'sentry.op': 'gen_ai.generate_text', + 'sentry.origin': 'auto.vercelai.otel', + }), + description: 'generate_text mock-model-id', + op: 'gen_ai.generate_text', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + // Seventh span - tool call execution span + expect.objectContaining({ + data: expect.objectContaining({ + 'vercel.ai.operationId': 'ai.toolCall', + 'gen_ai.tool.call.id': 'call-1', + 'gen_ai.tool.name': 'getWeather', + 'gen_ai.tool.input': expect.any(String), + 'gen_ai.tool.output': expect.any(String), + 'gen_ai.tool.type': 'function', + 'gen_ai.operation.name': 'ai.toolCall', + 'sentry.op': 'gen_ai.execute_tool', + 'sentry.origin': 'auto.vercelai.otel', + }), + description: 'execute_tool getWeather', + op: 'gen_ai.execute_tool', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + ]), + }; + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('creates ai related spans with sendDefaultPii: false', async () => { + await createRunner().expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE }).start().completed(); + }); + }, + { + additionalDependencies: { + ai: '^6.0.0', + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('creates ai related spans with sendDefaultPii: true', async () => { + await createRunner().expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE }).start().completed(); + }); + }, + { + additionalDependencies: { + ai: '^6.0.0', + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-error-in-tool.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('captures error in tool', async () => { + const expectedTransaction = { + transaction: 'main', + spans: expect.arrayContaining([ + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.request.model': 'mock-model-id', + 'vercel.ai.model.provider': 'mock-provider', + 'vercel.ai.operationId': 'ai.generateText', + 'vercel.ai.pipeline.name': 'generateText', + 'vercel.ai.request.headers.user-agent': expect.any(String), + 'vercel.ai.settings.maxRetries': 2, + 'vercel.ai.streaming': false, + 'gen_ai.response.model': 'mock-model-id', + 'gen_ai.usage.input_tokens': 15, + 'gen_ai.usage.output_tokens': 25, + 'gen_ai.usage.total_tokens': 40, + 'gen_ai.operation.name': 'ai.generateText', + 'sentry.op': 'gen_ai.invoke_agent', + 'sentry.origin': 'auto.vercelai.otel', + 'vercel.ai.response.finishReason': 'tool-calls', + }), + description: 'generateText', + op: 'gen_ai.invoke_agent', + origin: 'auto.vercelai.otel', + }), + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.request.model': 'mock-model-id', + 'vercel.ai.model.provider': 'mock-provider', + 'vercel.ai.operationId': 'ai.generateText.doGenerate', + 'vercel.ai.pipeline.name': 'generateText.doGenerate', + 'vercel.ai.request.headers.user-agent': expect.any(String), + 'vercel.ai.response.finishReason': 'tool-calls', + 'vercel.ai.response.id': expect.any(String), + 'vercel.ai.response.model': 'mock-model-id', + 'vercel.ai.response.timestamp': expect.any(String), + 'vercel.ai.settings.maxRetries': 2, + 'vercel.ai.streaming': false, + 'gen_ai.response.finish_reasons': ['tool-calls'], + 'gen_ai.response.id': expect.any(String), + 'gen_ai.response.model': 'mock-model-id', + 'gen_ai.system': 'mock-provider', + 'gen_ai.usage.input_tokens': 15, + 'gen_ai.usage.output_tokens': 25, + 'gen_ai.usage.total_tokens': 40, + 'gen_ai.operation.name': 'ai.generateText.doGenerate', + 'sentry.op': 'gen_ai.generate_text', + 'sentry.origin': 'auto.vercelai.otel', + }), + description: 'generate_text mock-model-id', + op: 'gen_ai.generate_text', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + expect.objectContaining({ + data: expect.objectContaining({ + 'vercel.ai.operationId': 'ai.toolCall', + 'gen_ai.tool.call.id': 'call-1', + 'gen_ai.tool.name': 'getWeather', + 'gen_ai.tool.type': 'function', + 'gen_ai.operation.name': 'ai.toolCall', + 'sentry.op': 'gen_ai.execute_tool', + 'sentry.origin': 'auto.vercelai.otel', + }), + description: 'execute_tool getWeather', + op: 'gen_ai.execute_tool', + origin: 'auto.vercelai.otel', + status: 'internal_error', + }), + ]), + }; + + const expectedError = { + level: 'error', + tags: expect.objectContaining({ + 'vercel.ai.tool.name': 'getWeather', + 'vercel.ai.tool.callId': 'call-1', + }), + }; + + let transactionEvent: Event | undefined; + let errorEvent: Event | undefined; + + await createRunner() + .expect({ + transaction: transaction => { + transactionEvent = transaction; + }, + }) + .expect({ + event: event => { + errorEvent = event; + }, + }) + .start() + .completed(); + + expect(transactionEvent).toBeDefined(); + expect(transactionEvent).toMatchObject(expectedTransaction); + + expect(errorEvent).toBeDefined(); + expect(errorEvent).toMatchObject(expectedError); + + // Trace id should be the same for the transaction and error event + expect(transactionEvent!.contexts!.trace!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id); + }); + }, + { + additionalDependencies: { + ai: '^6.0.0', + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('creates ai related spans with v6', async () => { + await createRunner().expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE }).start().completed(); + }); + }, + { + additionalDependencies: { + ai: '^6.0.0', + }, + }, + ); +}); diff --git a/packages/core/src/tracing/vercel-ai/index.ts b/packages/core/src/tracing/vercel-ai/index.ts index 6b59feb7a0ec..3415852ac4f3 100644 --- a/packages/core/src/tracing/vercel-ai/index.ts +++ b/packages/core/src/tracing/vercel-ai/index.ts @@ -22,7 +22,7 @@ import { getSpanOpFromName, requestMessagesFromPrompt, } from './utils'; -import type { ProviderMetadata } from './vercel-ai-attributes'; +import type { OpenAiProviderMetadata, ProviderMetadata } from './vercel-ai-attributes'; import { AI_MODEL_ID_ATTRIBUTE, AI_PROMPT_MESSAGES_ATTRIBUTE, @@ -270,28 +270,28 @@ function addProviderMetadataToAttributes(attributes: SpanAttributes): void { if (providerMetadata) { try { const providerMetadataObject = JSON.parse(providerMetadata) as ProviderMetadata; - if (providerMetadataObject.openai) { + + // Handle OpenAI metadata (v5 uses 'openai', v6 Azure Responses API uses 'azure') + const openaiMetadata: OpenAiProviderMetadata | undefined = + providerMetadataObject.openai ?? providerMetadataObject.azure; + if (openaiMetadata) { setAttributeIfDefined( attributes, GEN_AI_USAGE_INPUT_TOKENS_CACHED_ATTRIBUTE, - providerMetadataObject.openai.cachedPromptTokens, - ); - setAttributeIfDefined( - attributes, - 'gen_ai.usage.output_tokens.reasoning', - providerMetadataObject.openai.reasoningTokens, + openaiMetadata.cachedPromptTokens, ); + setAttributeIfDefined(attributes, 'gen_ai.usage.output_tokens.reasoning', openaiMetadata.reasoningTokens); setAttributeIfDefined( attributes, 'gen_ai.usage.output_tokens.prediction_accepted', - providerMetadataObject.openai.acceptedPredictionTokens, + openaiMetadata.acceptedPredictionTokens, ); setAttributeIfDefined( attributes, 'gen_ai.usage.output_tokens.prediction_rejected', - providerMetadataObject.openai.rejectedPredictionTokens, + openaiMetadata.rejectedPredictionTokens, ); - setAttributeIfDefined(attributes, 'gen_ai.conversation.id', providerMetadataObject.openai.responseId); + setAttributeIfDefined(attributes, 'gen_ai.conversation.id', openaiMetadata.responseId); } if (providerMetadataObject.anthropic) { diff --git a/packages/core/src/tracing/vercel-ai/vercel-ai-attributes.ts b/packages/core/src/tracing/vercel-ai/vercel-ai-attributes.ts index 95052fc1265a..3bb37e6a429a 100644 --- a/packages/core/src/tracing/vercel-ai/vercel-ai-attributes.ts +++ b/packages/core/src/tracing/vercel-ai/vercel-ai-attributes.ts @@ -821,7 +821,7 @@ export const AI_TOOL_CALL_SPAN_ATTRIBUTES = { * @see https://github.com/vercel/ai/blob/65e042afde6aad4da9d7a62526ece839eb34f9a5/packages/openai/src/openai-chat-language-model.ts#L397-L416 * @see https://github.com/vercel/ai/blob/65e042afde6aad4da9d7a62526ece839eb34f9a5/packages/openai/src/responses/openai-responses-language-model.ts#L377C7-L384 */ -interface OpenAiProviderMetadata { +export interface OpenAiProviderMetadata { /** * The number of predicted output tokens that were accepted. * @see https://ai-sdk.dev/providers/ai-sdk-providers/openai#predicted-outputs @@ -1041,9 +1041,11 @@ interface PerplexityProviderMetadata { export interface ProviderMetadata { openai?: OpenAiProviderMetadata; + azure?: OpenAiProviderMetadata; // v6: Azure Responses API uses 'azure' key instead of 'openai' anthropic?: AnthropicProviderMetadata; bedrock?: AmazonBedrockProviderMetadata; google?: GoogleGenerativeAIProviderMetadata; + vertex?: GoogleGenerativeAIProviderMetadata; // v6: Google Vertex uses 'vertex' key instead of 'google' deepseek?: DeepSeekProviderMetadata; perplexity?: PerplexityProviderMetadata; } diff --git a/packages/node/src/integrations/tracing/vercelai/instrumentation.ts b/packages/node/src/integrations/tracing/vercelai/instrumentation.ts index 872e0153edba..792032a7314e 100644 --- a/packages/node/src/integrations/tracing/vercelai/instrumentation.ts +++ b/packages/node/src/integrations/tracing/vercelai/instrumentation.ts @@ -15,6 +15,8 @@ import { import { INTEGRATION_NAME } from './constants'; import type { TelemetrySettings, VercelAiIntegration } from './types'; +const SUPPORTED_VERSIONS = ['>=3.0.0 <7']; + // List of patched methods // From: https://sdk.vercel.ai/docs/ai-sdk-core/telemetry#collected-data const INSTRUMENTED_METHODS = [ @@ -186,7 +188,7 @@ export class SentryVercelAiInstrumentation extends InstrumentationBase { * Initializes the instrumentation by defining the modules to be patched. */ public init(): InstrumentationModuleDefinition { - const module = new InstrumentationNodeModuleDefinition('ai', ['>=3.0.0 <6'], this._patch.bind(this)); + const module = new InstrumentationNodeModuleDefinition('ai', SUPPORTED_VERSIONS, this._patch.bind(this)); return module; }