diff --git a/dev-packages/node-integration-tests/package.json b/dev-packages/node-integration-tests/package.json index 5512b058f3f6..9692f5e180bb 100644 --- a/dev-packages/node-integration-tests/package.json +++ b/dev-packages/node-integration-tests/package.json @@ -14,7 +14,7 @@ "build:dev": "yarn build", "build:transpile": "rollup -c rollup.npm.config.mjs", "build:types": "tsc -p tsconfig.types.json", - "clean": "rimraf -g **/node_modules && run-p clean:script", + "clean": "rimraf -g suites/**/node_modules suites/**/tmp_* && run-p clean:script", "clean:script": "node scripts/clean.js", "lint": "eslint . --format stylish", "fix": "eslint . --format stylish --fix", diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-v5.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-v5.mjs new file mode 100644 index 000000000000..8cfe6d64ad05 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-v5.mjs @@ -0,0 +1,75 @@ +import * as Sentry from '@sentry/node'; +import { generateText } from 'ai'; +import { MockLanguageModelV2 } from 'ai/test'; +import { z } from 'zod'; + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + await generateText({ + model: new MockLanguageModelV2({ + doGenerate: async () => ({ + finishReason: 'stop', + usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 }, + content: [{ type: 'text', text: 'First span here!' }], + }), + }), + prompt: 'Where is the first span?', + }); + + // This span should have input and output prompts attached because telemetry is explicitly enabled. + await generateText({ + experimental_telemetry: { isEnabled: true }, + model: new MockLanguageModelV2({ + doGenerate: async () => ({ + finishReason: 'stop', + usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 }, + content: [{ type: 'text', text: 'Second span here!' }], + }), + }), + prompt: 'Where is the second span?', + }); + + // This span should include tool calls and tool results + await generateText({ + model: new MockLanguageModelV2({ + doGenerate: async () => ({ + finishReason: 'tool-calls', + usage: { inputTokens: 15, outputTokens: 25, totalTokens: 40 }, + content: [{ type: 'text', text: 'Tool call completed!' }], + toolCalls: [ + { + toolCallType: 'function', + toolCallId: 'call-1', + toolName: 'getWeather', + args: '{ "location": "San Francisco" }', + }, + ], + }), + }), + tools: { + getWeather: { + parameters: z.object({ location: z.string() }), + execute: async args => { + return `Weather in ${args.location}: Sunny, 72°F`; + }, + }, + }, + prompt: 'What is the weather in San Francisco?', + }); + + // This span should not be captured because we've disabled telemetry + await generateText({ + experimental_telemetry: { isEnabled: false }, + model: new MockLanguageModelV2({ + doGenerate: async () => ({ + finishReason: 'stop', + usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 }, + content: [{ type: 'text', text: 'Third span here!' }], + }), + }), + prompt: 'Where is the third span?', + }); + }); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts index 94fd0dde8486..720345cc7d86 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts @@ -197,6 +197,73 @@ describe('Vercel AI integration', () => { ]), }; + // Todo: Add missing attribute spans for v5 + // Right now only second span is recorded as it's manually opted in via explicit telemetry option + const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_V5 = { + transaction: 'main', + spans: expect.arrayContaining([ + expect.objectContaining({ + data: { + 'vercel.ai.model.id': 'mock-model-id', + 'vercel.ai.model.provider': 'mock-provider', + 'vercel.ai.operationId': 'ai.generateText', + 'vercel.ai.pipeline.name': 'generateText', + 'vercel.ai.prompt': '{"prompt":"Where is the second span?"}', + 'vercel.ai.response.finishReason': 'stop', + 'gen_ai.response.text': expect.any(String), + 'vercel.ai.settings.maxRetries': 2, + // 'vercel.ai.settings.maxSteps': 1, + 'vercel.ai.streaming': false, + 'gen_ai.prompt': '{"prompt":"Where is the second span?"}', + 'gen_ai.response.model': 'mock-model-id', + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 20, + 'gen_ai.usage.total_tokens': 30, + 'operation.name': 'ai.generateText', + 'sentry.op': 'gen_ai.invoke_agent', + 'sentry.origin': 'auto.vercelai.otel', + }, + description: 'generateText', + op: 'gen_ai.invoke_agent', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + // doGenerate + expect.objectContaining({ + data: { + 'sentry.origin': 'auto.vercelai.otel', + 'sentry.op': 'gen_ai.generate_text', + 'operation.name': 'ai.generateText.doGenerate', + 'vercel.ai.operationId': 'ai.generateText.doGenerate', + 'vercel.ai.model.provider': 'mock-provider', + 'vercel.ai.model.id': 'mock-model-id', + 'vercel.ai.settings.maxRetries': 2, + 'gen_ai.system': 'mock-provider', + 'gen_ai.request.model': 'mock-model-id', + 'vercel.ai.pipeline.name': 'generateText.doGenerate', + 'vercel.ai.streaming': false, + 'vercel.ai.response.finishReason': 'stop', + 'vercel.ai.response.model': 'mock-model-id', + 'vercel.ai.response.id': expect.any(String), + 'gen_ai.response.text': 'Second span here!', + 'vercel.ai.response.timestamp': expect.any(String), + // 'vercel.ai.prompt.format': expect.any(String), + 'gen_ai.request.messages': expect.any(String), + 'gen_ai.response.finish_reasons': ['stop'], + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 20, + 'gen_ai.response.id': expect.any(String), + 'gen_ai.response.model': 'mock-model-id', + 'gen_ai.usage.total_tokens': 30, + }, + description: 'generate_text mock-model-id', + op: 'gen_ai.generate_text', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + ]), + }; + const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = { transaction: 'main', spans: expect.arrayContaining([ @@ -538,6 +605,23 @@ describe('Vercel AI integration', () => { }); }); + // Test with specific Vercel AI v5 version + createEsmAndCjsTests( + __dirname, + 'scenario-v5.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('creates ai related spans with v5', async () => { + await createRunner().expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_V5 }).start().completed(); + }); + }, + { + additionalDependencies: { + ai: '^5.0.0', + }, + }, + ); + createEsmAndCjsTests(__dirname, 'scenario-error-in-tool-express.mjs', 'instrument.mjs', (createRunner, test) => { test('captures error in tool in express server', async () => { const expectedTransaction = { diff --git a/dev-packages/node-integration-tests/utils/runner.ts b/dev-packages/node-integration-tests/utils/runner.ts index 44118747c45c..f4a176688280 100644 --- a/dev-packages/node-integration-tests/utils/runner.ts +++ b/dev-packages/node-integration-tests/utils/runner.ts @@ -14,10 +14,10 @@ import type { import { normalize } from '@sentry/core'; import { createBasicSentryServer } from '@sentry-internal/test-utils'; import { execSync, spawn, spawnSync } from 'child_process'; -import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'fs'; -import { join } from 'path'; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'fs'; +import { basename, join } from 'path'; import { inspect } from 'util'; -import { afterAll, beforeAll, describe, test } from 'vitest'; +import { afterAll, describe, test } from 'vitest'; import { assertEnvelopeHeader, assertSentryCheckIn, @@ -174,7 +174,10 @@ export function createEsmAndCjsTests( testFn: typeof test | typeof test.fails, mode: 'esm' | 'cjs', ) => void, - options?: { failsOnCjs?: boolean; failsOnEsm?: boolean }, + // `additionalDependencies` to install in a tmp dir for the esm and cjs tests + // This could be used to override packages that live in the parent package.json for the specific run of the test + // e.g. `{ ai: '^5.0.0' }` to test Vercel AI v5 + options?: { failsOnCjs?: boolean; failsOnEsm?: boolean; additionalDependencies?: Record }, ): void { const mjsScenarioPath = join(cwd, scenarioPath); const mjsInstrumentPath = join(cwd, instrumentPath); @@ -187,36 +190,107 @@ export function createEsmAndCjsTests( throw new Error(`Instrument file not found: ${mjsInstrumentPath}`); } - const cjsScenarioPath = join(cwd, `tmp_${scenarioPath.replace('.mjs', '.cjs')}`); - const cjsInstrumentPath = join(cwd, `tmp_${instrumentPath.replace('.mjs', '.cjs')}`); + // Create a dedicated tmp directory that includes copied ESM & CJS scenario/instrument files. + // If additionalDependencies are provided, we also create a nested package.json and install them there. + const uniqueId = `${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`; + const tmpDirPath = join(cwd, `tmp_${uniqueId}`); + mkdirSync(tmpDirPath); + + // Copy ESM files as-is into tmp dir + const esmScenarioBasename = basename(scenarioPath); + const esmInstrumentBasename = basename(instrumentPath); + const esmScenarioPathForRun = join(tmpDirPath, esmScenarioBasename); + const esmInstrumentPathForRun = join(tmpDirPath, esmInstrumentBasename); + writeFileSync(esmScenarioPathForRun, readFileSync(mjsScenarioPath, 'utf8')); + writeFileSync(esmInstrumentPathForRun, readFileSync(mjsInstrumentPath, 'utf8')); + + // Pre-create CJS converted files inside tmp dir + const cjsScenarioPath = join(tmpDirPath, esmScenarioBasename.replace('.mjs', '.cjs')); + const cjsInstrumentPath = join(tmpDirPath, esmInstrumentBasename.replace('.mjs', '.cjs')); + convertEsmFileToCjs(esmScenarioPathForRun, cjsScenarioPath); + convertEsmFileToCjs(esmInstrumentPathForRun, cjsInstrumentPath); + + // Create a minimal package.json with requested dependencies (if any) and install them + const additionalDependencies = options?.additionalDependencies ?? {}; + if (Object.keys(additionalDependencies).length > 0) { + const packageJson = { + name: 'tmp-integration-test', + private: true, + version: '0.0.0', + dependencies: additionalDependencies, + } as const; + + writeFileSync(join(tmpDirPath, 'package.json'), JSON.stringify(packageJson, null, 2)); + + try { + const deps = Object.entries(additionalDependencies).map(([name, range]) => { + if (!range || typeof range !== 'string') { + throw new Error(`Invalid version range for "${name}": ${String(range)}`); + } + return `${name}@${range}`; + }); - describe('esm', () => { - const testFn = options?.failsOnEsm ? test.fails : test; - callback(() => createRunner(mjsScenarioPath).withFlags('--import', mjsInstrumentPath), testFn, 'esm'); - }); + if (deps.length > 0) { + // Prefer npm for temp installs to avoid Yarn engine strictness; see https://github.com/vercel/ai/issues/7777 + // We rely on the generated package.json dependencies and run a plain install. + const result = spawnSync('npm', ['install', '--silent', '--no-audit', '--no-fund'], { + cwd: tmpDirPath, + encoding: 'utf8', + }); + + if (process.env.DEBUG) { + // eslint-disable-next-line no-console + console.log('[additionalDependencies via npm]', deps.join(' ')); + // eslint-disable-next-line no-console + console.log('[npm stdout]', result.stdout); + // eslint-disable-next-line no-console + console.log('[npm stderr]', result.stderr); + } - describe('cjs', () => { - beforeAll(() => { - // For the CJS runner, we create some temporary files... - convertEsmFileToCjs(mjsScenarioPath, cjsScenarioPath); - convertEsmFileToCjs(mjsInstrumentPath, cjsInstrumentPath); + if (result.error) { + throw new Error(`Failed to install additionalDependencies in tmp dir ${tmpDirPath}: ${result.error.message}`); + } + if (typeof result.status === 'number' && result.status !== 0) { + throw new Error( + `Failed to install additionalDependencies in tmp dir ${tmpDirPath} (exit ${result.status}):\n${ + result.stderr || result.stdout || '(no output)' + }`, + ); + } + } + } catch (e) { + // eslint-disable-next-line no-console + console.error('Failed to install additionalDependencies:', e); + throw e; + } + } + + describe('esm/cjs', () => { + const esmTestFn = options?.failsOnEsm ? test.fails : test; + describe('esm', () => { + callback( + () => createRunner(esmScenarioPathForRun).withFlags('--import', esmInstrumentPathForRun), + esmTestFn, + 'esm', + ); + }); + + const cjsTestFn = options?.failsOnCjs ? test.fails : test; + describe('cjs', () => { + callback(() => createRunner(cjsScenarioPath).withFlags('--require', cjsInstrumentPath), cjsTestFn, 'cjs'); }); + // Clean up the tmp directory after both esm and cjs suites have run afterAll(() => { try { - unlinkSync(cjsInstrumentPath); - } catch { - // Ignore errors here - } - try { - unlinkSync(cjsScenarioPath); + rmSync(tmpDirPath, { recursive: true, force: true }); } catch { - // Ignore errors here + if (process.env.DEBUG) { + // eslint-disable-next-line no-console + console.error(`Failed to remove tmp dir: ${tmpDirPath}`); + } } }); - - const testFn = options?.failsOnCjs ? test.fails : test; - callback(() => createRunner(cjsScenarioPath).withFlags('--require', cjsInstrumentPath), testFn, 'cjs'); }); }