Skip to content

Commit fd2fb4c

Browse files
RulaKhaledRulaKhaled
committed
add tests
1 parent d1dab91 commit fd2fb4c

File tree

7 files changed

+762
-95
lines changed

7 files changed

+762
-95
lines changed

‎dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts

Lines changed: 0 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -197,73 +197,6 @@ describe('Vercel AI integration', () => {
197197
]),
198198
};
199199

200-
// Todo: Add missing attribute spans for v5
201-
// Right now only second span is recorded as it's manually opted in via explicit telemetry option
202-
const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_V5 = {
203-
transaction: 'main',
204-
spans: expect.arrayContaining([
205-
expect.objectContaining({
206-
data: {
207-
'vercel.ai.model.id': 'mock-model-id',
208-
'vercel.ai.model.provider': 'mock-provider',
209-
'vercel.ai.operationId': 'ai.generateText',
210-
'vercel.ai.pipeline.name': 'generateText',
211-
'vercel.ai.prompt': '{"prompt":"Where is the second span?"}',
212-
'vercel.ai.response.finishReason': 'stop',
213-
'gen_ai.response.text': expect.any(String),
214-
'vercel.ai.settings.maxRetries': 2,
215-
// 'vercel.ai.settings.maxSteps': 1,
216-
'vercel.ai.streaming': false,
217-
'gen_ai.prompt': '{"prompt":"Where is the second span?"}',
218-
'gen_ai.response.model': 'mock-model-id',
219-
'gen_ai.usage.input_tokens': 10,
220-
'gen_ai.usage.output_tokens': 20,
221-
'gen_ai.usage.total_tokens': 30,
222-
'operation.name': 'ai.generateText',
223-
'sentry.op': 'gen_ai.invoke_agent',
224-
'sentry.origin': 'auto.vercelai.otel',
225-
},
226-
description: 'generateText',
227-
op: 'gen_ai.invoke_agent',
228-
origin: 'auto.vercelai.otel',
229-
status: 'ok',
230-
}),
231-
// doGenerate
232-
expect.objectContaining({
233-
data: {
234-
'sentry.origin': 'auto.vercelai.otel',
235-
'sentry.op': 'gen_ai.generate_text',
236-
'operation.name': 'ai.generateText.doGenerate',
237-
'vercel.ai.operationId': 'ai.generateText.doGenerate',
238-
'vercel.ai.model.provider': 'mock-provider',
239-
'vercel.ai.model.id': 'mock-model-id',
240-
'vercel.ai.settings.maxRetries': 2,
241-
'gen_ai.system': 'mock-provider',
242-
'gen_ai.request.model': 'mock-model-id',
243-
'vercel.ai.pipeline.name': 'generateText.doGenerate',
244-
'vercel.ai.streaming': false,
245-
'vercel.ai.response.finishReason': 'stop',
246-
'vercel.ai.response.model': 'mock-model-id',
247-
'vercel.ai.response.id': expect.any(String),
248-
'gen_ai.response.text': 'Second span here!',
249-
'vercel.ai.response.timestamp': expect.any(String),
250-
// 'vercel.ai.prompt.format': expect.any(String),
251-
'gen_ai.request.messages': expect.any(String),
252-
'gen_ai.response.finish_reasons': ['stop'],
253-
'gen_ai.usage.input_tokens': 10,
254-
'gen_ai.usage.output_tokens': 20,
255-
'gen_ai.response.id': expect.any(String),
256-
'gen_ai.response.model': 'mock-model-id',
257-
'gen_ai.usage.total_tokens': 30,
258-
},
259-
description: 'generate_text mock-model-id',
260-
op: 'gen_ai.generate_text',
261-
origin: 'auto.vercelai.otel',
262-
status: 'ok',
263-
}),
264-
]),
265-
};
266-
267200
const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = {
268201
transaction: 'main',
269202
spans: expect.arrayContaining([
@@ -605,23 +538,6 @@ describe('Vercel AI integration', () => {
605538
});
606539
});
607540

608-
// Test with specific Vercel AI v5 version
609-
createEsmAndCjsTests(
610-
__dirname,
611-
'scenario-v5.mjs',
612-
'instrument.mjs',
613-
(createRunner, test) => {
614-
test('creates ai related spans with v5', async () => {
615-
await createRunner().expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_V5 }).start().completed();
616-
});
617-
},
618-
{
619-
additionalDependencies: {
620-
ai: '^5.0.0',
621-
},
622-
},
623-
);
624-
625541
createEsmAndCjsTests(__dirname, 'scenario-error-in-tool-express.mjs', 'instrument.mjs', (createRunner, test) => {
626542
test('captures error in tool in express server', async () => {
627543
const expectedTransaction = {

‎dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument-with-pii.mjs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import * as Sentry from '@sentry/node';
2+
import { loggingTransport } from '@sentry-internal/node-integration-tests';
3+
4+
Sentry.init({
5+
dsn: 'https://[email protected]/1337',
6+
release: '1.0',
7+
tracesSampleRate: 1.0,
8+
sendDefaultPii: true,
9+
transport: loggingTransport,
10+
integrations: [Sentry.vercelAIIntegration()],
11+
});

‎dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument.mjs

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
import * as Sentry from '@sentry/node';
2+
import { loggingTransport } from '@sentry-internal/node-integration-tests';
3+
4+
Sentry.init({
5+
dsn: 'https://[email protected]/1337',
6+
release: '1.0',
7+
tracesSampleRate: 1.0,
8+
transport: loggingTransport,
9+
integrations: [Sentry.vercelAIIntegration()],
10+
});

‎dev-packages/node-integration-tests/suites/tracing/vercelai/v5/scenario-error-in-tool-express.mjs

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
import * as Sentry from '@sentry/node';
2+
import { startExpressServerAndSendPortToRunner } from '@sentry-internal/node-integration-tests';
3+
import { generateText, tool } from 'ai';
4+
import { MockLanguageModelV2 } from 'ai/test';
5+
import express from 'express';
6+
import { z } from 'zod';
7+
8+
const app = express();
9+
10+
app.get('/test/error-in-tool', async (_req, res, next) => {
11+
Sentry.setTag('test-tag', 'test-value');
12+
13+
try {
14+
await generateText({
15+
model: new MockLanguageModelV2({
16+
doGenerate: async () => ({
17+
finishReason: 'tool-calls',
18+
usage: { inputTokens: 15, outputTokens: 25 },
19+
content: [
20+
{
21+
type: 'tool-call',
22+
toolCallId: 'call-1',
23+
toolName: 'getWeather',
24+
input: JSON.stringify({ location: 'San Francisco' }),
25+
},
26+
],
27+
}),
28+
}),
29+
tools: {
30+
getWeather: tool({
31+
inputSchema: z.object({ location: z.string() }),
32+
execute: async () => {
33+
throw new Error('Error in tool');
34+
},
35+
}),
36+
},
37+
prompt: 'What is the weather in San Francisco?',
38+
});
39+
} catch (error) {
40+
next(error);
41+
return;
42+
}
43+
44+
res.send({ message: 'OK' });
45+
});
46+
Sentry.setupExpressErrorHandler(app);
47+
48+
startExpressServerAndSendPortToRunner(app);

‎dev-packages/node-integration-tests/suites/tracing/vercelai/v5/scenario-error-in-tool.mjs

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
import * as Sentry from '@sentry/node';
2+
import { generateText, tool } from 'ai';
3+
import { MockLanguageModelV2 } from 'ai/test';
4+
import { z } from 'zod';
5+
6+
async function run() {
7+
Sentry.setTag('test-tag', 'test-value');
8+
9+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
10+
await generateText({
11+
model: new MockLanguageModelV2({
12+
doGenerate: async () => ({
13+
finishReason: 'tool-calls',
14+
usage: { inputTokens: 15, outputTokens: 25, totalTokens: 40 },
15+
content: [
16+
{
17+
type: 'tool-call',
18+
toolCallId: 'call-1',
19+
toolName: 'getWeather',
20+
input: JSON.stringify({ location: 'San Francisco' }),
21+
},
22+
],
23+
}),
24+
}),
25+
tools: {
26+
getWeather: tool({
27+
inputSchema: z.object({ location: z.string() }),
28+
execute: async () => {
29+
throw new Error('Error in tool');
30+
},
31+
}),
32+
},
33+
prompt: 'What is the weather in San Francisco?',
34+
});
35+
});
36+
}
37+
38+
run();

‎dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-v5.mjs renamed to ‎dev-packages/node-integration-tests/suites/tracing/vercelai/v5/scenario.mjs

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import * as Sentry from '@sentry/node';
2-
import { generateText } from 'ai';
2+
import { generateText, tool } from 'ai';
33
import { MockLanguageModelV2 } from 'ai/test';
44
import { z } from 'zod';
55

@@ -35,24 +35,21 @@ async function run() {
3535
doGenerate: async () => ({
3636
finishReason: 'tool-calls',
3737
usage: { inputTokens: 15, outputTokens: 25, totalTokens: 40 },
38-
content: [{ type: 'text', text: 'Tool call completed!' }],
39-
toolCalls: [
38+
content: [
4039
{
41-
toolCallType: 'function',
40+
type: 'tool-call',
4241
toolCallId: 'call-1',
4342
toolName: 'getWeather',
44-
args: '{ "location": "San Francisco" }',
43+
input: JSON.stringify({ location: 'San Francisco' }),
4544
},
4645
],
4746
}),
4847
}),
4948
tools: {
50-
getWeather: {
51-
parameters: z.object({ location: z.string() }),
52-
execute: async args => {
53-
return `Weather in ${args.location}: Sunny, 72°F`;
54-
},
55-
},
49+
getWeather: tool({
50+
inputSchema: z.object({ location: z.string() }),
51+
execute: async ({ location }) => `Weather in ${location}: Sunny, 72°F`,
52+
}),
5653
},
5754
prompt: 'What is the weather in San Francisco?',
5855
});

0 commit comments

Comments
 (0)