From beed714f8c239c17b9194cfac86a4308e643cf16 Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Wed, 23 Jul 2025 14:12:29 +0200 Subject: [PATCH 1/4] test(node): Test vercelai errors are linked to parent span --- .../tracing/vercelai/scenario-error.mjs | 53 +++++++++++ .../vercelai/scenario-express-error.mjs | 81 +++++++++++++++++ .../suites/tracing/vercelai/test.ts | 90 +++++++++++++++++++ 3 files changed, 224 insertions(+) create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-express-error.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error.mjs new file mode 100644 index 000000000000..e2658c17f801 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error.mjs @@ -0,0 +1,53 @@ +import * as Sentry from '@sentry/node'; +import { generateText, tool } from 'ai'; +import { MockLanguageModelV1 } from 'ai/test'; +import { z } from 'zod'; + +async function run() { + // Create a manual outer span (simulating the root span) + await Sentry.startSpan({ op: 'http.server', name: 'GET /api/test', description: 'HTTP server request' }, async () => { + // It is expected that the error will bubble up naturally to the outer span + await generateText({ + model: new MockLanguageModelV1({ + doGenerate: async () => ({ + rawCall: { rawPrompt: null, rawSettings: {} }, + finishReason: 'stop', + usage: { promptTokens: 10, completionTokens: 20 }, + text: 'First span here!', + toolCalls: [ + { + toolCallType: 'function', + toolCallId: 'call-1', + toolName: 'calculateTool', + args: '{ "a": 1, "b": 2 }', + }, + ], + }), + }), + experimental_telemetry: { + functionId: 'Simple Agent', + recordInputs: true, + recordOutputs: true, + isEnabled: true, + }, + tools: { + calculateTool: tool({ + description: 'Calculate the result of a math problem. Returns a number.', + parameters: z.object({ + a: z.number().describe('First number'), + b: z.number().describe('Second number'), + }), + type: 'function', + execute: async () => { + throw new Error('Not implemented'); + }, + }), + }, + maxSteps: 2, + system: 'You help users with their math problems.', + prompt: 'What is 1 + 1?', + }); + }); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-express-error.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-express-error.mjs new file mode 100644 index 000000000000..896ceceba6f4 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-express-error.mjs @@ -0,0 +1,81 @@ +import * as Sentry from '@sentry/node'; +import { generateText, tool } from 'ai'; +import { MockLanguageModelV1 } from 'ai/test'; +import express from 'express'; +import { createServer } from 'http'; +import { z } from 'zod'; + +async function run() { + const app = express(); + + app.get('/api/chat', async (req, res) => { + try { + await generateText({ + model: new MockLanguageModelV1({ + doGenerate: async () => ({ + rawCall: { rawPrompt: null, rawSettings: {} }, + finishReason: 'stop', + usage: { promptTokens: 10, completionTokens: 20 }, + text: 'Processing your request...', + toolCalls: [ + { + toolCallType: 'function', + toolCallId: 'call-1', + toolName: 'calculateTool', + args: '{ "a": 1, "b": 2 }', + }, + ], + }), + }), + experimental_telemetry: { + functionId: 'Chat Assistant', + recordInputs: true, + recordOutputs: true, + isEnabled: true, + }, + tools: { + calculateTool: tool({ + description: 'Calculate the result of a math problem. Returns a number.', + parameters: z.object({ + a: z.number().describe('First number'), + b: z.number().describe('Second number'), + }), + type: 'function', + execute: async () => { + throw new Error('Calculation service unavailable'); + }, + }), + }, + maxSteps: 2, + system: 'You are a helpful chat assistant.', + prompt: 'What is 1 + 1?', + }); + + res.json({ success: true }); + } catch (error) { + res.status(500).json({ error: error.message }); + } + }); + + Sentry.setupExpressErrorHandler(app); + + const server = createServer(app); + + // Start server and make request + server.listen(0, () => { + const port = server.address()?.port; + // eslint-disable-next-line no-console + console.log(JSON.stringify({ port })); + + // Make the request that will trigger the error + fetch(`http://localhost:${port}/api/chat`) + .then(() => { + server.close(); + }) + .catch(() => { + server.close(); + }); + }); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts index f9b853aa4946..17a01d2e61c7 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts @@ -416,4 +416,94 @@ describe('Vercel AI integration', () => { await createRunner().expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE }).start().completed(); }); }); + + createEsmAndCjsTests(__dirname, 'scenario-error.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { + test('Vercel AI errors should inherit parent trace context from manually created outer span', async () => { + let capturedTransaction: any; + let capturedEvent: any; + + const runner = createRunner() + .expect({ + transaction: (transaction: any) => { + capturedTransaction = transaction; + expect(transaction.transaction).toBe('GET /api/test'); + }, + }) + .expect({ + event: (event: any) => { + capturedEvent = event; + + expect(event).toMatchObject({ + exception: { + values: expect.arrayContaining([ + expect.objectContaining({ + type: 'AI_ToolExecutionError', + value: 'Error executing tool calculateTool: Not implemented', + }), + ]), + }, + }); + }, + }) + .start(); + + await runner.completed(); + + const transactionTraceId = capturedTransaction?.contexts?.trace?.trace_id; + const errorTraceId = capturedEvent?.contexts?.trace?.trace_id; + + expect(transactionTraceId).toBeDefined(); + expect(errorTraceId).toBeDefined(); + expect(transactionTraceId).toMatch(/^[a-f0-9]{32}$/); + expect(errorTraceId).toMatch(/^[a-f0-9]{32}$/); + + expect(errorTraceId).toBe(transactionTraceId); + }); + }); + + createEsmAndCjsTests(__dirname, 'scenario-express-error.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { + test('Vercel AI errors should inherit parent trace context from server HTTP request', async () => { + let capturedTransaction: any; + let capturedEvent: any; + + const runner = createRunner() + .withMockSentryServer() + .expect({ + transaction: (transaction: any) => { + capturedTransaction = transaction; + // Express creates a transaction like "GET /api/chat" + expect(transaction.transaction).toBe('GET /api/chat'); + }, + }) + .expect({ + event: (event: any) => { + capturedEvent = event; + + expect(event).toMatchObject({ + exception: { + values: expect.arrayContaining([ + expect.objectContaining({ + type: 'AI_ToolExecutionError', + value: 'Error executing tool calculateTool: Calculation service unavailable', + }), + ]), + }, + }); + }, + }) + .start(); + + await runner.completed(); + + const transactionTraceId = capturedTransaction?.contexts?.trace?.trace_id; + const errorTraceId = capturedEvent?.contexts?.trace?.trace_id; + + expect(transactionTraceId).toBeDefined(); + expect(errorTraceId).toBeDefined(); + expect(transactionTraceId).toMatch(/^[a-f0-9]{32}$/); + expect(errorTraceId).toMatch(/^[a-f0-9]{32}$/); + + expect(errorTraceId).toBe(transactionTraceId); + }); + }); }); From f70062a71791b4bec0a2b9e9ce896764ccc4027f Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Wed, 23 Jul 2025 14:16:41 +0200 Subject: [PATCH 2/4] rename to outer span --- .../suites/tracing/vercelai/scenario-error.mjs | 2 +- .../node-integration-tests/suites/tracing/vercelai/test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error.mjs index e2658c17f801..b09421cb09a6 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error.mjs @@ -5,7 +5,7 @@ import { z } from 'zod'; async function run() { // Create a manual outer span (simulating the root span) - await Sentry.startSpan({ op: 'http.server', name: 'GET /api/test', description: 'HTTP server request' }, async () => { + await Sentry.startSpan({ op: 'outer', name: 'outer span', description: 'outer span' }, async () => { // It is expected that the error will bubble up naturally to the outer span await generateText({ model: new MockLanguageModelV1({ diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts index 17a01d2e61c7..f58cf63cdd58 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts @@ -426,7 +426,7 @@ describe('Vercel AI integration', () => { .expect({ transaction: (transaction: any) => { capturedTransaction = transaction; - expect(transaction.transaction).toBe('GET /api/test'); + expect(transaction.transaction).toBe('outer span'); }, }) .expect({ From 16073049bd6fa83ead1294041bcad78d1e8b17a9 Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Wed, 23 Jul 2025 15:16:36 +0200 Subject: [PATCH 3/4] try timeout sol --- .../vercelai/scenario-express-error.mjs | 22 +++++++++++-------- .../suites/tracing/vercelai/test.ts | 4 ++-- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-express-error.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-express-error.mjs index 896ceceba6f4..b30eb20108d3 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-express-error.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-express-error.mjs @@ -62,19 +62,23 @@ async function run() { const server = createServer(app); // Start server and make request - server.listen(0, () => { + server.listen(0, async () => { const port = server.address()?.port; // eslint-disable-next-line no-console console.log(JSON.stringify({ port })); - // Make the request that will trigger the error - fetch(`http://localhost:${port}/api/chat`) - .then(() => { - server.close(); - }) - .catch(() => { - server.close(); - }); + try { + // Make the request that will trigger the error + const response = await fetch(`http://localhost:${port}/api/chat`); + await response.json(); // Consume the response + } catch (error) { + // Expected to fail due to the tool error, but we still want to capture it + } + + // Give time for Sentry to process and send the error + setTimeout(() => { + server.close(); + }, 1000); }); } diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts index f58cf63cdd58..2d881412dac5 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts @@ -458,7 +458,7 @@ describe('Vercel AI integration', () => { expect(errorTraceId).toMatch(/^[a-f0-9]{32}$/); expect(errorTraceId).toBe(transactionTraceId); - }); + }, 30000); }); createEsmAndCjsTests(__dirname, 'scenario-express-error.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { @@ -504,6 +504,6 @@ describe('Vercel AI integration', () => { expect(errorTraceId).toMatch(/^[a-f0-9]{32}$/); expect(errorTraceId).toBe(transactionTraceId); - }); + }, 30000); }); }); From e4c7c0dcd7302223cc1240e4ccf908b5f63f3098 Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Wed, 23 Jul 2025 15:41:10 +0200 Subject: [PATCH 4/4] Revert "try timeout sol" This reverts commit 16073049bd6fa83ead1294041bcad78d1e8b17a9. --- .../vercelai/scenario-express-error.mjs | 22 ++++++++----------- .../suites/tracing/vercelai/test.ts | 4 ++-- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-express-error.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-express-error.mjs index b30eb20108d3..896ceceba6f4 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-express-error.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-express-error.mjs @@ -62,23 +62,19 @@ async function run() { const server = createServer(app); // Start server and make request - server.listen(0, async () => { + server.listen(0, () => { const port = server.address()?.port; // eslint-disable-next-line no-console console.log(JSON.stringify({ port })); - try { - // Make the request that will trigger the error - const response = await fetch(`http://localhost:${port}/api/chat`); - await response.json(); // Consume the response - } catch (error) { - // Expected to fail due to the tool error, but we still want to capture it - } - - // Give time for Sentry to process and send the error - setTimeout(() => { - server.close(); - }, 1000); + // Make the request that will trigger the error + fetch(`http://localhost:${port}/api/chat`) + .then(() => { + server.close(); + }) + .catch(() => { + server.close(); + }); }); } diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts index 2d881412dac5..f58cf63cdd58 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts @@ -458,7 +458,7 @@ describe('Vercel AI integration', () => { expect(errorTraceId).toMatch(/^[a-f0-9]{32}$/); expect(errorTraceId).toBe(transactionTraceId); - }, 30000); + }); }); createEsmAndCjsTests(__dirname, 'scenario-express-error.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { @@ -504,6 +504,6 @@ describe('Vercel AI integration', () => { expect(errorTraceId).toMatch(/^[a-f0-9]{32}$/); expect(errorTraceId).toBe(transactionTraceId); - }, 30000); + }); }); });