test(node): Enable additionalDependencies in integration runner

dev-packages/node-integration-tests/package.json

@@ -14,7 +14,7 @@
     "build:dev": "yarn build",
     "build:transpile": "rollup -c rollup.npm.config.mjs",
     "build:types": "tsc -p tsconfig.types.json",
-    "clean": "rimraf -g **/node_modules && run-p clean:script",
+    "clean": "rimraf -g suites/**/node_modules suites/**/tmp_* && run-p clean:script",
     "clean:script": "node scripts/clean.js",
     "lint": "eslint . --format stylish",
     "fix": "eslint . --format stylish --fix",

dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-v5.mjs

@@ -0,0 +1,75 @@
+import * as Sentry from '@sentry/node';
+import { generateText } from 'ai';
+import { MockLanguageModelV2 } from 'ai/test';
+import { z } from 'zod';
+
+async function run() {
+  await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
+    await generateText({
+      model: new MockLanguageModelV2({
+        doGenerate: async () => ({
+          finishReason: 'stop',
+          usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
+          content: [{ type: 'text', text: 'First span here!' }],
+        }),
+      }),
+      prompt: 'Where is the first span?',
+    });
+
+    // This span should have input and output prompts attached because telemetry is explicitly enabled.
+    await generateText({
+      experimental_telemetry: { isEnabled: true },
+      model: new MockLanguageModelV2({
+        doGenerate: async () => ({
+          finishReason: 'stop',
+          usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
+          content: [{ type: 'text', text: 'Second span here!' }],
+        }),
+      }),
+      prompt: 'Where is the second span?',
+    });
+
+    // This span should include tool calls and tool results
+    await generateText({
+      model: new MockLanguageModelV2({
+        doGenerate: async () => ({
+          finishReason: 'tool-calls',
+          usage: { inputTokens: 15, outputTokens: 25, totalTokens: 40 },
+          content: [{ type: 'text', text: 'Tool call completed!' }],
+          toolCalls: [
+            {
+              toolCallType: 'function',
+              toolCallId: 'call-1',
+              toolName: 'getWeather',
+              args: '{ "location": "San Francisco" }',
+            },
+          ],
+        }),
+      }),
+      tools: {
+        getWeather: {
+          parameters: z.object({ location: z.string() }),
+          execute: async args => {
+            return `Weather in ${args.location}: Sunny, 72°F`;
+          },
+        },
+      },
+      prompt: 'What is the weather in San Francisco?',
+    });
+
+    // This span should not be captured because we've disabled telemetry
+    await generateText({
+      experimental_telemetry: { isEnabled: false },
+      model: new MockLanguageModelV2({
+        doGenerate: async () => ({
+          finishReason: 'stop',
+          usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
+          content: [{ type: 'text', text: 'Third span here!' }],
+        }),
+      }),
+      prompt: 'Where is the third span?',
+    });
+  });
+}
+
+run();

dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts

@@ -197,6 +197,73 @@ describe('Vercel AI integration', () => {
     ]),
   };
 
+  // Todo: Add missing attribute spans for v5
+  // Right now only second span is recorded as it's manually opted in via explicit telemetry option
+  const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_V5 = {
+    transaction: 'main',
+    spans: expect.arrayContaining([
+      expect.objectContaining({
+        data: {
+          'vercel.ai.model.id': 'mock-model-id',
+          'vercel.ai.model.provider': 'mock-provider',
+          'vercel.ai.operationId': 'ai.generateText',
+          'vercel.ai.pipeline.name': 'generateText',
+          'vercel.ai.prompt': '{"prompt":"Where is the second span?"}',
+          'vercel.ai.response.finishReason': 'stop',
+          'gen_ai.response.text': expect.any(String),
+          'vercel.ai.settings.maxRetries': 2,
+          // 'vercel.ai.settings.maxSteps': 1,
+          'vercel.ai.streaming': false,
+          'gen_ai.prompt': '{"prompt":"Where is the second span?"}',
+          'gen_ai.response.model': 'mock-model-id',
+          'gen_ai.usage.input_tokens': 10,
+          'gen_ai.usage.output_tokens': 20,
+          'gen_ai.usage.total_tokens': 30,
+          'operation.name': 'ai.generateText',
+          'sentry.op': 'gen_ai.invoke_agent',
+          'sentry.origin': 'auto.vercelai.otel',
+        },
+        description: 'generateText',
+        op: 'gen_ai.invoke_agent',
+        origin: 'auto.vercelai.otel',
+        status: 'ok',
+      }),
+      // doGenerate
+      expect.objectContaining({
+        data: {
+          'sentry.origin': 'auto.vercelai.otel',
+          'sentry.op': 'gen_ai.generate_text',
+          'operation.name': 'ai.generateText.doGenerate',
+          'vercel.ai.operationId': 'ai.generateText.doGenerate',
+          'vercel.ai.model.provider': 'mock-provider',
+          'vercel.ai.model.id': 'mock-model-id',
+          'vercel.ai.settings.maxRetries': 2,
+          'gen_ai.system': 'mock-provider',
+          'gen_ai.request.model': 'mock-model-id',
+          'vercel.ai.pipeline.name': 'generateText.doGenerate',
+          'vercel.ai.streaming': false,
+          'vercel.ai.response.finishReason': 'stop',
+          'vercel.ai.response.model': 'mock-model-id',
+          'vercel.ai.response.id': expect.any(String),
+          'gen_ai.response.text': 'Second span here!',
+          'vercel.ai.response.timestamp': expect.any(String),
+          // 'vercel.ai.prompt.format': expect.any(String),
+          'gen_ai.request.messages': expect.any(String),
+          'gen_ai.response.finish_reasons': ['stop'],
+          'gen_ai.usage.input_tokens': 10,
+          'gen_ai.usage.output_tokens': 20,
+          'gen_ai.response.id': expect.any(String),
+          'gen_ai.response.model': 'mock-model-id',
+          'gen_ai.usage.total_tokens': 30,
+        },
+        description: 'generate_text mock-model-id',
+        op: 'gen_ai.generate_text',
+        origin: 'auto.vercelai.otel',
+        status: 'ok',
+      }),
+    ]),
+  };
+
   const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = {
     transaction: 'main',
     spans: expect.arrayContaining([
@@ -538,6 +605,23 @@ describe('Vercel AI integration', () => {
     });
   });
 
+  // Test with specific Vercel AI v5 version
+  createEsmAndCjsTests(
+    __dirname,
+    'scenario-v5.mjs',
+    'instrument.mjs',
+    (createRunner, test) => {
+      test('creates ai related spans with v5', async () => {
+        await createRunner().expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_V5 }).start().completed();
+      });
+    },
+    {
+      additionalDependencies: {
+        ai: '^5.0.0',
+      },
+    },
+  );
+
   createEsmAndCjsTests(__dirname, 'scenario-error-in-tool-express.mjs', 'instrument.mjs', (createRunner, test) => {
     test('captures error in tool in express server', async () => {
       const expectedTransaction = {

dev-packages/node-integration-tests/utils/runner.ts

@@ -14,10 +14,10 @@
 import { normalize } from '@sentry/core';
 import { createBasicSentryServer } from '@sentry-internal/test-utils';
 import { execSync, spawn, spawnSync } from 'child_process';
-import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'fs';
-import { join } from 'path';
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'fs';
+import { basename, join } from 'path';
 import { inspect } from 'util';
-import { afterAll, beforeAll, describe, test } from 'vitest';
+import { afterAll, describe, test } from 'vitest';
 import {
   assertEnvelopeHeader,
   assertSentryCheckIn,
@@ -174,7 +174,10 @@
     testFn: typeof test | typeof test.fails,
     mode: 'esm' | 'cjs',
   ) => void,
-  options?: { failsOnCjs?: boolean; failsOnEsm?: boolean },
+  // `additionalDependencies` to install in a tmp dir for the esm and cjs tests
+  // This could be used to override packages that live in the parent package.json for the specific run of the test
+  // e.g. `{ ai: '^5.0.0' }` to test Vercel AI v5
+  options?: { failsOnCjs?: boolean; failsOnEsm?: boolean; additionalDependencies?: Record<string, string> },
 ): void {
   const mjsScenarioPath = join(cwd, scenarioPath);
   const mjsInstrumentPath = join(cwd, instrumentPath);
@@ -187,36 +190,119 @@
     throw new Error(`Instrument file not found: ${mjsInstrumentPath}`);
   }
 
-  const cjsScenarioPath = join(cwd, `tmp_${scenarioPath.replace('.mjs', '.cjs')}`);
-  const cjsInstrumentPath = join(cwd, `tmp_${instrumentPath.replace('.mjs', '.cjs')}`);
-
-  describe('esm', () => {
-    const testFn = options?.failsOnEsm ? test.fails : test;
-    callback(() => createRunner(mjsScenarioPath).withFlags('--import', mjsInstrumentPath), testFn, 'esm');
+  // Create a dedicated tmp directory that includes copied ESM & CJS scenario/instrument files.
+  // If additionalDependencies are provided, we also create a nested package.json and install them there.
+  const uniqueId = `${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
+  const tmpDirPath = join(cwd, `tmp_${uniqueId}`);
+  mkdirSync(tmpDirPath);
+
+  // Ensure tmp dir is removed on process exit as a fallback
+  CLEANUP_STEPS.add(() => {
+    try {
+      rmSync(tmpDirPath, { recursive: true, force: true });
+    } catch {
+      if (process.env.DEBUG) {
+        // eslint-disable-next-line no-console
+        console.error(`Failed to remove tmp dir: ${tmpDirPath}`);
+      }
+    }
   });
 
-  describe('cjs', () => {
-    beforeAll(() => {
-      // For the CJS runner, we create some temporary files...
-      convertEsmFileToCjs(mjsScenarioPath, cjsScenarioPath);
-      convertEsmFileToCjs(mjsInstrumentPath, cjsInstrumentPath);
+  // Copy ESM files as-is into tmp dir
+  const esmScenarioBasename = basename(scenarioPath);
+  const esmInstrumentBasename = basename(instrumentPath);
+  const esmScenarioPathForRun = join(tmpDirPath, esmScenarioBasename);
+  const esmInstrumentPathForRun = join(tmpDirPath, esmInstrumentBasename);
+  writeFileSync(esmScenarioPathForRun, readFileSync(mjsScenarioPath, 'utf8'));
+  writeFileSync(esmInstrumentPathForRun, readFileSync(mjsInstrumentPath, 'utf8'));
+
+  // Pre-create CJS converted files inside tmp dir
+  const cjsScenarioPath = join(tmpDirPath, esmScenarioBasename.replace('.mjs', '.cjs'));
+  const cjsInstrumentPath = join(tmpDirPath, esmInstrumentBasename.replace('.mjs', '.cjs'));
+  convertEsmFileToCjs(esmScenarioPathForRun, cjsScenarioPath);
+  convertEsmFileToCjs(esmInstrumentPathForRun, cjsInstrumentPath);
+
+  // Create a minimal package.json with requested dependencies (if any) and install them
+  const additionalDependencies = options?.additionalDependencies ?? {};
+  if (Object.keys(additionalDependencies).length > 0) {
+    const packageJson = {
+      name: 'tmp-integration-test',
+      private: true,
+      version: '0.0.0',
+      dependencies: additionalDependencies,
+    } as const;
+
+    writeFileSync(join(tmpDirPath, 'package.json'), JSON.stringify(packageJson, null, 2));
+
+    try {
+      const deps = Object.entries(additionalDependencies).map(([name, range]) => {
+        if (!range || typeof range !== 'string') {
+          throw new Error(`Invalid version range for "${name}": ${String(range)}`);
+        }
+        return `${name}@${range}`;
+      });
+
+      if (deps.length > 0) {
+        // Prefer npm for temp installs to avoid Yarn engine strictness; see https://github.com/vercel/ai/issues/7777
+        // We rely on the generated package.json dependencies and run a plain install.
+        const result = spawnSync('npm', ['install', '--silent', '--no-audit', '--no-fund'], {
+          cwd: tmpDirPath,
+          encoding: 'utf8',
+        });
+
+        if (process.env.DEBUG) {
+          // eslint-disable-next-line no-console
+          console.log('[additionalDependencies via npm]', deps.join(' '));
+          // eslint-disable-next-line no-console
+          console.log('[npm stdout]', result.stdout);
+          // eslint-disable-next-line no-console
+          console.log('[npm stderr]', result.stderr);
+        }
+
+        if (result.error) {
+          throw new Error(`Failed to install additionalDependencies in tmp dir ${tmpDirPath}: ${result.error.message}`);
+        }
+        if (typeof result.status === 'number' && result.status !== 0) {
+          throw new Error(
+            `Failed to install additionalDependencies in tmp dir ${tmpDirPath} (exit ${result.status}):\n${
+              result.stderr || result.stdout || '(no output)'
+            }`,
+          );
+        }
+      }
+    } catch (e) {
+      // eslint-disable-next-line no-console
+      console.error('Failed to install additionalDependencies:', e);
+      throw e;
+    }
+  }
+
+  describe('esm/cjs', () => {
+    const esmTestFn = options?.failsOnEsm ? test.fails : test;
+    describe('esm', () => {
+      callback(
+        () => createRunner(esmScenarioPathForRun).withFlags('--import', esmInstrumentPathForRun),
+        esmTestFn,
+        'esm',
+      );
+    });
+
+    const cjsTestFn = options?.failsOnCjs ? test.fails : test;
+    describe('cjs', () => {
+      callback(() => createRunner(cjsScenarioPath).withFlags('--require', cjsInstrumentPath), cjsTestFn, 'cjs');
     });
 
+    // Clean up the tmp directory after both esm and cjs suites have run
     afterAll(() => {
       try {
-        unlinkSync(cjsInstrumentPath);
+        rmSync(tmpDirPath, { recursive: true, force: true });
       } catch {
-        // Ignore errors here
-      }
-      try {
-        unlinkSync(cjsScenarioPath);
-      } catch {
-        // Ignore errors here
+        if (process.env.DEBUG) {
+          // eslint-disable-next-line no-console
+          console.error(`Failed to remove tmp dir: ${tmpDirPath}`);
+        }
       }
     });
-
-    const testFn = options?.failsOnCjs ? test.fails : test;
-    callback(() => createRunner(cjsScenarioPath).withFlags('--require', cjsInstrumentPath), testFn, 'cjs');
   });
 }
 
@@ -232,7 +318,7 @@
   const testPath = join(...paths);
 
   if (!existsSync(testPath)) {
     throw new Error(`Test scenario not found: ${testPath}`);
   }
 
   const expectedEnvelopes: Expected[] = [];