test(node): enable additionalDependencies in integration runner

RulaKhaled · RulaKhaled · commit 86a1543b4920 · 2025-08-08T14:29:05.000+02:00
diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-v5.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-v5.mjs
@@ -0,0 +1,75 @@
+import * as Sentry from '@sentry/node';
+import { generateText } from 'ai';
+import { MockLanguageModelV2 } from 'ai/test';
+import { z } from 'zod';
+
+async function run() {
+  await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
+    await generateText({
+      model: new MockLanguageModelV2({
+        doGenerate: async () => ({
+          finishReason: 'stop',
+          usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
+          content: [{ type: 'text', text: 'First span here!' }],
+        }),
+      }),
+      prompt: 'Where is the first span?',
+    });
+
+    // This span should have input and output prompts attached because telemetry is explicitly enabled.
+    await generateText({
+      experimental_telemetry: { isEnabled: true },
+      model: new MockLanguageModelV2({
+        doGenerate: async () => ({
+          finishReason: 'stop',
+          usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
+          content: [{ type: 'text', text: 'Second span here!' }],
+        }),
+      }),
+      prompt: 'Where is the second span?',
+    });
+
+    // This span should include tool calls and tool results
+    await generateText({
+      model: new MockLanguageModelV2({
+        doGenerate: async () => ({
+          finishReason: 'tool-calls',
+          usage: { inputTokens: 15, outputTokens: 25, totalTokens: 40 },
+          content: [{ type: 'text', text: 'Tool call completed!' }],
+          toolCalls: [
+            {
+              toolCallType: 'function',
+              toolCallId: 'call-1',
+              toolName: 'getWeather',
+              args: '{ "location": "San Francisco" }',
+            },
+          ],
+        }),
+      }),
+      tools: {
+        getWeather: {
+          parameters: z.object({ location: z.string() }),
+          execute: async args => {
+            return `Weather in ${args.location}: Sunny, 72°F`;
+          },
+        },
+      },
+      prompt: 'What is the weather in San Francisco?',
+    });
+
+    // This span should not be captured because we've disabled telemetry
+    await generateText({
+      experimental_telemetry: { isEnabled: false },
+      model: new MockLanguageModelV2({
+        doGenerate: async () => ({
+          finishReason: 'stop',
+          usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
+          content: [{ type: 'text', text: 'Third span here!' }],
+        }),
+      }),
+      prompt: 'Where is the third span?',
+    });
+  });
+}
+
+run();
diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts
@@ -197,6 +197,73 @@ describe('Vercel AI integration', () => {
     ]),
   };
 
+  // Todo: Add missing attribute spans for v5
+  // Right now only second span is recorded as it's manually opted in via explicit telemetry option
+  const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_V5 = {
+    transaction: 'main',
+    spans: expect.arrayContaining([
+      expect.objectContaining({
+        data: {
+          'vercel.ai.model.id': 'mock-model-id',
+          'vercel.ai.model.provider': 'mock-provider',
+          'vercel.ai.operationId': 'ai.generateText',
+          'vercel.ai.pipeline.name': 'generateText',
+          'vercel.ai.prompt': '{"prompt":"Where is the second span?"}',
+          'vercel.ai.response.finishReason': 'stop',
+          'gen_ai.response.text': expect.any(String),
+          'vercel.ai.settings.maxRetries': 2,
+          // 'vercel.ai.settings.maxSteps': 1,
+          'vercel.ai.streaming': false,
+          'gen_ai.prompt': '{"prompt":"Where is the second span?"}',
+          'gen_ai.response.model': 'mock-model-id',
+          'gen_ai.usage.input_tokens': 10,
+          'gen_ai.usage.output_tokens': 20,
+          'gen_ai.usage.total_tokens': 30,
+          'operation.name': 'ai.generateText',
+          'sentry.op': 'gen_ai.invoke_agent',
+          'sentry.origin': 'auto.vercelai.otel',
+        },
+        description: 'generateText',
+        op: 'gen_ai.invoke_agent',
+        origin: 'auto.vercelai.otel',
+        status: 'ok',
+      }),
+      // doGenerate
+      expect.objectContaining({
+        data: {
+          'sentry.origin': 'auto.vercelai.otel',
+          'sentry.op': 'gen_ai.generate_text',
+          'operation.name': 'ai.generateText.doGenerate',
+          'vercel.ai.operationId': 'ai.generateText.doGenerate',
+          'vercel.ai.model.provider': 'mock-provider',
+          'vercel.ai.model.id': 'mock-model-id',
+          'vercel.ai.settings.maxRetries': 2,
+          'gen_ai.system': 'mock-provider',
+          'gen_ai.request.model': 'mock-model-id',
+          'vercel.ai.pipeline.name': 'generateText.doGenerate',
+          'vercel.ai.streaming': false,
+          'vercel.ai.response.finishReason': 'stop',
+          'vercel.ai.response.model': 'mock-model-id',
+          'vercel.ai.response.id': expect.any(String),
+          'gen_ai.response.text': 'Second span here!',
+          'vercel.ai.response.timestamp': expect.any(String),
+          // 'vercel.ai.prompt.format': expect.any(String),
+          'gen_ai.request.messages': expect.any(String),
+          'gen_ai.response.finish_reasons': ['stop'],
+          'gen_ai.usage.input_tokens': 10,
+          'gen_ai.usage.output_tokens': 20,
+          'gen_ai.response.id': expect.any(String),
+          'gen_ai.response.model': 'mock-model-id',
+          'gen_ai.usage.total_tokens': 30,
+        },
+        description: 'generate_text mock-model-id',
+        op: 'gen_ai.generate_text',
+        origin: 'auto.vercelai.otel',
+        status: 'ok',
+      }),
+    ]),
+  };
+
   const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = {
     transaction: 'main',
     spans: expect.arrayContaining([
@@ -538,6 +605,23 @@ describe('Vercel AI integration', () => {
     });
   });
 
+  // Test with specific Vercel AI v5 version
+  createEsmAndCjsTests(
+    __dirname,
+    'scenario-v5.mjs',
+    'instrument.mjs',
+    (createRunner, test) => {
+      test('creates ai related spans with v5', async () => {
+        await createRunner().expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_V5 }).start().completed();
+      });
+    },
+    {
+      additionalDependencies: {
+        ai: '^5.0.0',
+      },
+    },
+  );
+
   createEsmAndCjsTests(__dirname, 'scenario-error-in-tool-express.mjs', 'instrument.mjs', (createRunner, test) => {
     test('captures error in tool in express server', async () => {
       const expectedTransaction = {
diff --git a/dev-packages/node-integration-tests/utils/runner.ts b/dev-packages/node-integration-tests/utils/runner.ts
@@ -14,8 +14,8 @@ import type {
 import { normalize } from '@sentry/core';
 import { createBasicSentryServer } from '@sentry-internal/test-utils';
 import { execSync, spawn, spawnSync } from 'child_process';
-import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'fs';
-import { join } from 'path';
+import { existsSync, mkdirSync, readFileSync, rmSync, unlinkSync, writeFileSync } from 'fs';
+import { basename, join } from 'path';
 import { inspect } from 'util';
 import { afterAll, beforeAll, describe, test } from 'vitest';
 import {
@@ -174,7 +174,7 @@ export function createEsmAndCjsTests(
     testFn: typeof test | typeof test.fails,
     mode: 'esm' | 'cjs',
   ) => void,
-  options?: { failsOnCjs?: boolean; failsOnEsm?: boolean },
+  options?: { failsOnCjs?: boolean; failsOnEsm?: boolean; additionalDependencies?: Record<string, string> },
 ): void {
   const mjsScenarioPath = join(cwd, scenarioPath);
   const mjsInstrumentPath = join(cwd, instrumentPath);
@@ -187,33 +187,110 @@ export function createEsmAndCjsTests(
     throw new Error(`Instrument file not found: ${mjsInstrumentPath}`);
   }
 
-  const cjsScenarioPath = join(cwd, `tmp_${scenarioPath.replace('.mjs', '.cjs')}`);
-  const cjsInstrumentPath = join(cwd, `tmp_${instrumentPath.replace('.mjs', '.cjs')}`);
+  // If additionalDependencies are provided, we create a dedicated tmp directory that includes
+  // copied ESM & CJS scenario/instrument files and a nested package.json with those dependencies installed.
+  const useTmpDir = Boolean(options?.additionalDependencies && Object.keys(options.additionalDependencies).length > 0);
+
+  let tmpDirPath: string | undefined;
+  let esmScenarioPathForRun = mjsScenarioPath;
+  let esmInstrumentPathForRun = mjsInstrumentPath;
+  let cjsScenarioPath = join(cwd, `tmp_${scenarioPath.replace('.mjs', '.cjs')}`);
+  let cjsInstrumentPath = join(cwd, `tmp_${instrumentPath.replace('.mjs', '.cjs')}`);
+
+  if (useTmpDir) {
+    // Create unique tmp directory within the suite folder
+    const uniqueId = `${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
+    tmpDirPath = join(cwd, `tmp_${uniqueId}`);
+    mkdirSync(tmpDirPath);
+
+    // Copy ESM files as-is into tmp dir
+    const esmScenarioBasename = basename(scenarioPath);
+    const esmInstrumentBasename = basename(instrumentPath);
+    esmScenarioPathForRun = join(tmpDirPath, esmScenarioBasename);
+    esmInstrumentPathForRun = join(tmpDirPath, esmInstrumentBasename);
+    writeFileSync(esmScenarioPathForRun, readFileSync(mjsScenarioPath, 'utf8'));
+    writeFileSync(esmInstrumentPathForRun, readFileSync(mjsInstrumentPath, 'utf8'));
+
+    // Pre-create CJS converted files inside tmp dir
+    cjsScenarioPath = join(tmpDirPath, esmScenarioBasename.replace('.mjs', '.cjs'));
+    cjsInstrumentPath = join(tmpDirPath, esmInstrumentBasename.replace('.mjs', '.cjs'));
+    convertEsmFileToCjs(esmScenarioPathForRun, cjsScenarioPath);
+    convertEsmFileToCjs(esmInstrumentPathForRun, cjsInstrumentPath);
+
+    // Create a minimal package.json with requested dependencies
+    const additionalDependencies = options?.additionalDependencies ?? {};
+    const packageJson = {
+      name: 'tmp-integration-test',
+      private: true,
+      version: '0.0.0',
+      dependencies: additionalDependencies,
+    } as const;
+
+    writeFileSync(join(tmpDirPath, 'package.json'), JSON.stringify(packageJson, null, 2));
+
+    // Install the requested dependencies using yarn
+    try {
+      const deps = Object.entries(additionalDependencies).map(([name, range]) => {
+        if (!range || typeof range !== 'string') {
+          throw new Error(`Invalid version range for "${name}": ${String(range)}`);
+        }
+        return `${name}@${range}`;
+      });
+
+      if (deps.length > 0) {
+        // Run yarn add non-interactively, keep output visible when DEBUG is set
+        spawnSync('yarn', ['add', '--non-interactive', ...deps], {
+          cwd: tmpDirPath,
+          stdio: process.env.DEBUG ? 'inherit' : 'ignore',
+        });
+      }
+    } catch (e) {
+      // eslint-disable-next-line no-console
+      console.error('Failed to install additionalDependencies:', e);
+      throw e;
+    }
+  }
 
   describe('esm', () => {
     const testFn = options?.failsOnEsm ? test.fails : test;
-    callback(() => createRunner(mjsScenarioPath).withFlags('--import', mjsInstrumentPath), testFn, 'esm');
+    callback(() => createRunner(esmScenarioPathForRun).withFlags('--import', esmInstrumentPathForRun), testFn, 'esm');
   });
 
   describe('cjs', () => {
-    beforeAll(() => {
-      // For the CJS runner, we create some temporary files...
-      convertEsmFileToCjs(mjsScenarioPath, cjsScenarioPath);
-      convertEsmFileToCjs(mjsInstrumentPath, cjsInstrumentPath);
-    });
-
-    afterAll(() => {
-      try {
-        unlinkSync(cjsInstrumentPath);
-      } catch {
-        // Ignore errors here
-      }
-      try {
-        unlinkSync(cjsScenarioPath);
-      } catch {
-        // Ignore errors here
-      }
-    });
+    if (!useTmpDir) {
+      beforeAll(() => {
+        // For the CJS runner, we create some temporary files...
+        convertEsmFileToCjs(mjsScenarioPath, cjsScenarioPath);
+        convertEsmFileToCjs(mjsInstrumentPath, cjsInstrumentPath);
+      });
+
+      afterAll(() => {
+        try {
+          unlinkSync(cjsInstrumentPath);
+        } catch {
+          // Ignore errors here
+        }
+        try {
+          unlinkSync(cjsScenarioPath);
+        } catch {
+          // Ignore errors here
+        }
+      });
+    } else {
+      // When using a tmp dir, clean up the entire directory once CJS tests are finished
+      afterAll(() => {
+        if (tmpDirPath) {
+          try {
+            rmSync(tmpDirPath, { recursive: true, force: true });
+          } catch {
+            if (process.env.DEBUG) {
+              // eslint-disable-next-line no-console
+              console.error(`Failed to remove tmp dir: ${tmpDirPath}`);
+            }
+          }
+        }
+      });
+    }
 
     const testFn = options?.failsOnCjs ? test.fails : test;
     callback(() => createRunner(cjsScenarioPath).withFlags('--require', cjsInstrumentPath), testFn, 'cjs');
