Skip to content

Commit 9f6ae21

Browse files
committed
Merge branch 'develop' into cg-ignore-static-assets
2 parents bfe2bb3 + f307a22 commit 9f6ae21

File tree

12 files changed

+730
-502
lines changed

12 files changed

+730
-502
lines changed

dev-packages/node-integration-tests/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
"build:dev": "yarn build",
1515
"build:transpile": "rollup -c rollup.npm.config.mjs",
1616
"build:types": "tsc -p tsconfig.types.json",
17-
"clean": "rimraf -g **/node_modules && run-p clean:script",
17+
"clean": "rimraf -g suites/**/node_modules suites/**/tmp_* && run-p clean:script",
1818
"clean:script": "node scripts/clean.js",
1919
"lint": "eslint . --format stylish",
2020
"fix": "eslint . --format stylish --fix",
Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
import * as Sentry from '@sentry/node';
2+
import { generateText } from 'ai';
3+
import { MockLanguageModelV2 } from 'ai/test';
4+
import { z } from 'zod';
5+
6+
async function run() {
7+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
8+
await generateText({
9+
model: new MockLanguageModelV2({
10+
doGenerate: async () => ({
11+
finishReason: 'stop',
12+
usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
13+
content: [{ type: 'text', text: 'First span here!' }],
14+
}),
15+
}),
16+
prompt: 'Where is the first span?',
17+
});
18+
19+
// This span should have input and output prompts attached because telemetry is explicitly enabled.
20+
await generateText({
21+
experimental_telemetry: { isEnabled: true },
22+
model: new MockLanguageModelV2({
23+
doGenerate: async () => ({
24+
finishReason: 'stop',
25+
usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
26+
content: [{ type: 'text', text: 'Second span here!' }],
27+
}),
28+
}),
29+
prompt: 'Where is the second span?',
30+
});
31+
32+
// This span should include tool calls and tool results
33+
await generateText({
34+
model: new MockLanguageModelV2({
35+
doGenerate: async () => ({
36+
finishReason: 'tool-calls',
37+
usage: { inputTokens: 15, outputTokens: 25, totalTokens: 40 },
38+
content: [{ type: 'text', text: 'Tool call completed!' }],
39+
toolCalls: [
40+
{
41+
toolCallType: 'function',
42+
toolCallId: 'call-1',
43+
toolName: 'getWeather',
44+
args: '{ "location": "San Francisco" }',
45+
},
46+
],
47+
}),
48+
}),
49+
tools: {
50+
getWeather: {
51+
parameters: z.object({ location: z.string() }),
52+
execute: async args => {
53+
return `Weather in ${args.location}: Sunny, 72°F`;
54+
},
55+
},
56+
},
57+
prompt: 'What is the weather in San Francisco?',
58+
});
59+
60+
// This span should not be captured because we've disabled telemetry
61+
await generateText({
62+
experimental_telemetry: { isEnabled: false },
63+
model: new MockLanguageModelV2({
64+
doGenerate: async () => ({
65+
finishReason: 'stop',
66+
usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
67+
content: [{ type: 'text', text: 'Third span here!' }],
68+
}),
69+
}),
70+
prompt: 'Where is the third span?',
71+
});
72+
});
73+
}
74+
75+
run();

dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,73 @@ describe('Vercel AI integration', () => {
197197
]),
198198
};
199199

200+
// Todo: Add missing attribute spans for v5
201+
// Right now only second span is recorded as it's manually opted in via explicit telemetry option
202+
const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_V5 = {
203+
transaction: 'main',
204+
spans: expect.arrayContaining([
205+
expect.objectContaining({
206+
data: {
207+
'vercel.ai.model.id': 'mock-model-id',
208+
'vercel.ai.model.provider': 'mock-provider',
209+
'vercel.ai.operationId': 'ai.generateText',
210+
'vercel.ai.pipeline.name': 'generateText',
211+
'vercel.ai.prompt': '{"prompt":"Where is the second span?"}',
212+
'vercel.ai.response.finishReason': 'stop',
213+
'gen_ai.response.text': expect.any(String),
214+
'vercel.ai.settings.maxRetries': 2,
215+
// 'vercel.ai.settings.maxSteps': 1,
216+
'vercel.ai.streaming': false,
217+
'gen_ai.prompt': '{"prompt":"Where is the second span?"}',
218+
'gen_ai.response.model': 'mock-model-id',
219+
'gen_ai.usage.input_tokens': 10,
220+
'gen_ai.usage.output_tokens': 20,
221+
'gen_ai.usage.total_tokens': 30,
222+
'operation.name': 'ai.generateText',
223+
'sentry.op': 'gen_ai.invoke_agent',
224+
'sentry.origin': 'auto.vercelai.otel',
225+
},
226+
description: 'generateText',
227+
op: 'gen_ai.invoke_agent',
228+
origin: 'auto.vercelai.otel',
229+
status: 'ok',
230+
}),
231+
// doGenerate
232+
expect.objectContaining({
233+
data: {
234+
'sentry.origin': 'auto.vercelai.otel',
235+
'sentry.op': 'gen_ai.generate_text',
236+
'operation.name': 'ai.generateText.doGenerate',
237+
'vercel.ai.operationId': 'ai.generateText.doGenerate',
238+
'vercel.ai.model.provider': 'mock-provider',
239+
'vercel.ai.model.id': 'mock-model-id',
240+
'vercel.ai.settings.maxRetries': 2,
241+
'gen_ai.system': 'mock-provider',
242+
'gen_ai.request.model': 'mock-model-id',
243+
'vercel.ai.pipeline.name': 'generateText.doGenerate',
244+
'vercel.ai.streaming': false,
245+
'vercel.ai.response.finishReason': 'stop',
246+
'vercel.ai.response.model': 'mock-model-id',
247+
'vercel.ai.response.id': expect.any(String),
248+
'gen_ai.response.text': 'Second span here!',
249+
'vercel.ai.response.timestamp': expect.any(String),
250+
// 'vercel.ai.prompt.format': expect.any(String),
251+
'gen_ai.request.messages': expect.any(String),
252+
'gen_ai.response.finish_reasons': ['stop'],
253+
'gen_ai.usage.input_tokens': 10,
254+
'gen_ai.usage.output_tokens': 20,
255+
'gen_ai.response.id': expect.any(String),
256+
'gen_ai.response.model': 'mock-model-id',
257+
'gen_ai.usage.total_tokens': 30,
258+
},
259+
description: 'generate_text mock-model-id',
260+
op: 'gen_ai.generate_text',
261+
origin: 'auto.vercelai.otel',
262+
status: 'ok',
263+
}),
264+
]),
265+
};
266+
200267
const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = {
201268
transaction: 'main',
202269
spans: expect.arrayContaining([
@@ -538,6 +605,23 @@ describe('Vercel AI integration', () => {
538605
});
539606
});
540607

608+
// Test with specific Vercel AI v5 version
609+
createEsmAndCjsTests(
610+
__dirname,
611+
'scenario-v5.mjs',
612+
'instrument.mjs',
613+
(createRunner, test) => {
614+
test('creates ai related spans with v5', async () => {
615+
await createRunner().expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_V5 }).start().completed();
616+
});
617+
},
618+
{
619+
additionalDependencies: {
620+
ai: '^5.0.0',
621+
},
622+
},
623+
);
624+
541625
createEsmAndCjsTests(__dirname, 'scenario-error-in-tool-express.mjs', 'instrument.mjs', (createRunner, test) => {
542626
test('captures error in tool in express server', async () => {
543627
const expectedTransaction = {

dev-packages/node-integration-tests/utils/runner.ts

Lines changed: 99 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,10 @@ import type {
1414
import { normalize } from '@sentry/core';
1515
import { createBasicSentryServer } from '@sentry-internal/test-utils';
1616
import { execSync, spawn, spawnSync } from 'child_process';
17-
import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'fs';
18-
import { join } from 'path';
17+
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'fs';
18+
import { basename, join } from 'path';
1919
import { inspect } from 'util';
20-
import { afterAll, beforeAll, describe, test } from 'vitest';
20+
import { afterAll, describe, test } from 'vitest';
2121
import {
2222
assertEnvelopeHeader,
2323
assertSentryCheckIn,
@@ -174,7 +174,10 @@ export function createEsmAndCjsTests(
174174
testFn: typeof test | typeof test.fails,
175175
mode: 'esm' | 'cjs',
176176
) => void,
177-
options?: { failsOnCjs?: boolean; failsOnEsm?: boolean },
177+
// `additionalDependencies` to install in a tmp dir for the esm and cjs tests
178+
// This could be used to override packages that live in the parent package.json for the specific run of the test
179+
// e.g. `{ ai: '^5.0.0' }` to test Vercel AI v5
180+
options?: { failsOnCjs?: boolean; failsOnEsm?: boolean; additionalDependencies?: Record<string, string> },
178181
): void {
179182
const mjsScenarioPath = join(cwd, scenarioPath);
180183
const mjsInstrumentPath = join(cwd, instrumentPath);
@@ -187,36 +190,107 @@ export function createEsmAndCjsTests(
187190
throw new Error(`Instrument file not found: ${mjsInstrumentPath}`);
188191
}
189192

190-
const cjsScenarioPath = join(cwd, `tmp_${scenarioPath.replace('.mjs', '.cjs')}`);
191-
const cjsInstrumentPath = join(cwd, `tmp_${instrumentPath.replace('.mjs', '.cjs')}`);
193+
// Create a dedicated tmp directory that includes copied ESM & CJS scenario/instrument files.
194+
// If additionalDependencies are provided, we also create a nested package.json and install them there.
195+
const uniqueId = `${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
196+
const tmpDirPath = join(cwd, `tmp_${uniqueId}`);
197+
mkdirSync(tmpDirPath);
198+
199+
// Copy ESM files as-is into tmp dir
200+
const esmScenarioBasename = basename(scenarioPath);
201+
const esmInstrumentBasename = basename(instrumentPath);
202+
const esmScenarioPathForRun = join(tmpDirPath, esmScenarioBasename);
203+
const esmInstrumentPathForRun = join(tmpDirPath, esmInstrumentBasename);
204+
writeFileSync(esmScenarioPathForRun, readFileSync(mjsScenarioPath, 'utf8'));
205+
writeFileSync(esmInstrumentPathForRun, readFileSync(mjsInstrumentPath, 'utf8'));
206+
207+
// Pre-create CJS converted files inside tmp dir
208+
const cjsScenarioPath = join(tmpDirPath, esmScenarioBasename.replace('.mjs', '.cjs'));
209+
const cjsInstrumentPath = join(tmpDirPath, esmInstrumentBasename.replace('.mjs', '.cjs'));
210+
convertEsmFileToCjs(esmScenarioPathForRun, cjsScenarioPath);
211+
convertEsmFileToCjs(esmInstrumentPathForRun, cjsInstrumentPath);
212+
213+
// Create a minimal package.json with requested dependencies (if any) and install them
214+
const additionalDependencies = options?.additionalDependencies ?? {};
215+
if (Object.keys(additionalDependencies).length > 0) {
216+
const packageJson = {
217+
name: 'tmp-integration-test',
218+
private: true,
219+
version: '0.0.0',
220+
dependencies: additionalDependencies,
221+
} as const;
222+
223+
writeFileSync(join(tmpDirPath, 'package.json'), JSON.stringify(packageJson, null, 2));
224+
225+
try {
226+
const deps = Object.entries(additionalDependencies).map(([name, range]) => {
227+
if (!range || typeof range !== 'string') {
228+
throw new Error(`Invalid version range for "${name}": ${String(range)}`);
229+
}
230+
return `${name}@${range}`;
231+
});
192232

193-
describe('esm', () => {
194-
const testFn = options?.failsOnEsm ? test.fails : test;
195-
callback(() => createRunner(mjsScenarioPath).withFlags('--import', mjsInstrumentPath), testFn, 'esm');
196-
});
233+
if (deps.length > 0) {
234+
// Prefer npm for temp installs to avoid Yarn engine strictness; see https://github.com/vercel/ai/issues/7777
235+
// We rely on the generated package.json dependencies and run a plain install.
236+
const result = spawnSync('npm', ['install', '--silent', '--no-audit', '--no-fund'], {
237+
cwd: tmpDirPath,
238+
encoding: 'utf8',
239+
});
240+
241+
if (process.env.DEBUG) {
242+
// eslint-disable-next-line no-console
243+
console.log('[additionalDependencies via npm]', deps.join(' '));
244+
// eslint-disable-next-line no-console
245+
console.log('[npm stdout]', result.stdout);
246+
// eslint-disable-next-line no-console
247+
console.log('[npm stderr]', result.stderr);
248+
}
197249

198-
describe('cjs', () => {
199-
beforeAll(() => {
200-
// For the CJS runner, we create some temporary files...
201-
convertEsmFileToCjs(mjsScenarioPath, cjsScenarioPath);
202-
convertEsmFileToCjs(mjsInstrumentPath, cjsInstrumentPath);
250+
if (result.error) {
251+
throw new Error(`Failed to install additionalDependencies in tmp dir ${tmpDirPath}: ${result.error.message}`);
252+
}
253+
if (typeof result.status === 'number' && result.status !== 0) {
254+
throw new Error(
255+
`Failed to install additionalDependencies in tmp dir ${tmpDirPath} (exit ${result.status}):\n${
256+
result.stderr || result.stdout || '(no output)'
257+
}`,
258+
);
259+
}
260+
}
261+
} catch (e) {
262+
// eslint-disable-next-line no-console
263+
console.error('Failed to install additionalDependencies:', e);
264+
throw e;
265+
}
266+
}
267+
268+
describe('esm/cjs', () => {
269+
const esmTestFn = options?.failsOnEsm ? test.fails : test;
270+
describe('esm', () => {
271+
callback(
272+
() => createRunner(esmScenarioPathForRun).withFlags('--import', esmInstrumentPathForRun),
273+
esmTestFn,
274+
'esm',
275+
);
276+
});
277+
278+
const cjsTestFn = options?.failsOnCjs ? test.fails : test;
279+
describe('cjs', () => {
280+
callback(() => createRunner(cjsScenarioPath).withFlags('--require', cjsInstrumentPath), cjsTestFn, 'cjs');
203281
});
204282

283+
// Clean up the tmp directory after both esm and cjs suites have run
205284
afterAll(() => {
206285
try {
207-
unlinkSync(cjsInstrumentPath);
208-
} catch {
209-
// Ignore errors here
210-
}
211-
try {
212-
unlinkSync(cjsScenarioPath);
286+
rmSync(tmpDirPath, { recursive: true, force: true });
213287
} catch {
214-
// Ignore errors here
288+
if (process.env.DEBUG) {
289+
// eslint-disable-next-line no-console
290+
console.error(`Failed to remove tmp dir: ${tmpDirPath}`);
291+
}
215292
}
216293
});
217-
218-
const testFn = options?.failsOnCjs ? test.fails : test;
219-
callback(() => createRunner(cjsScenarioPath).withFlags('--require', cjsInstrumentPath), testFn, 'cjs');
220294
});
221295
}
222296

0 commit comments

Comments
 (0)