diff --git a/.size-limit.js b/.size-limit.js index a46d905fe89f..d53eaae56712 100644 --- a/.size-limit.js +++ b/.size-limit.js @@ -233,7 +233,7 @@ module.exports = [ import: createImport('init'), ignore: [...builtinModules, ...nodePrefixedBuiltinModules], gzip: true, - limit: '146 KB', + limit: '147 KB', }, { name: '@sentry/node - without tracing', diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/instrument-with-pii.mjs new file mode 100644 index 000000000000..a53a13af7738 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/instrument-with-pii.mjs @@ -0,0 +1,11 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + sendDefaultPii: true, + transport: loggingTransport, + integrations: [Sentry.openAIIntegration()], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/instrument.mjs new file mode 100644 index 000000000000..f3fbac9d1274 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/instrument.mjs @@ -0,0 +1,11 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + sendDefaultPii: false, + transport: loggingTransport, + integrations: [Sentry.openAIIntegration()], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/scenario.mjs new file mode 100644 index 000000000000..6a323c3adaee --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/scenario.mjs @@ -0,0 +1,297 @@ +import { instrumentOpenAiClient } from '@sentry/core'; +import * as Sentry from '@sentry/node'; + +class MockOpenAIToolCalls { + constructor(config) { + this.apiKey = config.apiKey; + + this.chat = { + completions: { + create: async params => { + // Simulate processing time + await new Promise(resolve => setTimeout(resolve, 10)); + + // If stream is requested, return an async generator + if (params.stream) { + return this._createChatCompletionToolCallsStream(params); + } + + // Non-streaming tool calls response + return { + id: 'chatcmpl-tools-123', + object: 'chat.completion', + created: 1677652300, + model: params.model, + system_fingerprint: 'fp_tools_123', + choices: [ + { + index: 0, + message: { + role: 'assistant', + content: null, + tool_calls: [ + { + id: 'call_12345xyz', + type: 'function', + function: { + name: 'get_weather', + arguments: '{"latitude":48.8566,"longitude":2.3522}', + }, + }, + ], + }, + finish_reason: 'tool_calls', + }, + ], + usage: { + prompt_tokens: 15, + completion_tokens: 25, + total_tokens: 40, + }, + }; + }, + }, + }; + + this.responses = { + create: async params => { + await new Promise(resolve => setTimeout(resolve, 10)); + + // If stream is requested, return an async generator + if (params.stream) { + return this._createResponsesApiToolCallsStream(params); + } + + // Non-streaming tool calls response + return { + id: 'resp_tools_789', + object: 'response', + created_at: 1677652320, + model: params.model, + input_text: Array.isArray(params.input) ? JSON.stringify(params.input) : params.input, + status: 'completed', + output: [ + { + type: 'function_call', + id: 'fc_12345xyz', + call_id: 'call_12345xyz', + name: 'get_weather', + arguments: '{"latitude":48.8566,"longitude":2.3522}', + }, + ], + usage: { + input_tokens: 8, + output_tokens: 12, + total_tokens: 20, + }, + }; + }, + }; + } + + // Create a mock streaming response for chat completions with tool calls + async *_createChatCompletionToolCallsStream(params) { + // First chunk with tool call initialization + yield { + id: 'chatcmpl-stream-tools-123', + object: 'chat.completion.chunk', + created: 1677652305, + model: params.model, + choices: [ + { + index: 0, + delta: { + role: 'assistant', + tool_calls: [ + { + index: 0, + id: 'call_12345xyz', + type: 'function', + function: { name: 'get_weather', arguments: '' }, + }, + ], + }, + finish_reason: null, + }, + ], + }; + + // Second chunk with arguments delta + yield { + id: 'chatcmpl-stream-tools-123', + object: 'chat.completion.chunk', + created: 1677652305, + model: params.model, + choices: [ + { + index: 0, + delta: { + tool_calls: [ + { + index: 0, + function: { arguments: '{"latitude":48.8566,"longitude":2.3522}' }, + }, + ], + }, + finish_reason: 'tool_calls', + }, + ], + usage: { prompt_tokens: 15, completion_tokens: 25, total_tokens: 40 }, + }; + } + + // Create a mock streaming response for responses API with tool calls + async *_createResponsesApiToolCallsStream(params) { + const responseId = 'resp_stream_tools_789'; + + // Response created event + yield { + type: 'response.created', + response: { + id: responseId, + object: 'response', + created_at: 1677652310, + model: params.model, + status: 'in_progress', + output: [], + usage: { input_tokens: 0, output_tokens: 0, total_tokens: 0 }, + }, + sequence_number: 1, + }; + + // Function call output item added + yield { + type: 'response.output_item.added', + response_id: responseId, + output_index: 0, + item: { + type: 'function_call', + id: 'fc_12345xyz', + call_id: 'call_12345xyz', + name: 'get_weather', + arguments: '', + }, + sequence_number: 2, + }; + + // Function call arguments delta events + yield { + type: 'response.function_call_arguments.delta', + response_id: responseId, + item_id: 'fc_12345xyz', + output_index: 0, + delta: '{"latitude":48.8566,"longitude":2.3522}', + sequence_number: 3, + }; + + // Function call arguments done + yield { + type: 'response.function_call_arguments.done', + response_id: responseId, + item_id: 'fc_12345xyz', + output_index: 0, + arguments: '{"latitude":48.8566,"longitude":2.3522}', + sequence_number: 4, + }; + + // Output item done + yield { + type: 'response.output_item.done', + response_id: responseId, + output_index: 0, + item: { + type: 'function_call', + id: 'fc_12345xyz', + call_id: 'call_12345xyz', + name: 'get_weather', + arguments: '{"latitude":48.8566,"longitude":2.3522}', + }, + sequence_number: 5, + }; + + // Response completed event + yield { + type: 'response.completed', + response: { + id: responseId, + object: 'response', + created_at: 1677652310, + model: params.model, + status: 'completed', + output: [ + { + type: 'function_call', + id: 'fc_12345xyz', + call_id: 'call_12345xyz', + name: 'get_weather', + arguments: '{"latitude":48.8566,"longitude":2.3522}', + }, + ], + usage: { input_tokens: 8, output_tokens: 12, total_tokens: 20 }, + }, + sequence_number: 6, + }; + } +} + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const mockClient = new MockOpenAIToolCalls({ + apiKey: 'mock-api-key', + }); + + const client = instrumentOpenAiClient(mockClient); + + const weatherTool = { + type: 'function', + function: { + name: 'get_weather', + description: 'Get the current weather in a given location', + parameters: { + type: 'object', + properties: { + latitude: { type: 'number', description: 'The latitude of the location' }, + longitude: { type: 'number', description: 'The longitude of the location' }, + }, + required: ['latitude', 'longitude'], + }, + }, + }; + + const message = { role: 'user', content: 'What is the weather like in Paris today?' }; + + // Test 1: Chat completion with tools (non-streaming) + await client.chat.completions.create({ + model: 'gpt-4', + messages: [message], + tools: [weatherTool], + }); + + // Test 2: Chat completion with tools (streaming) + const stream1 = await client.chat.completions.create({ + model: 'gpt-4', + messages: [message], + tools: [weatherTool], + stream: true, + }); + for await (const chunk of stream1) void chunk; + + // Test 3: Responses API with tools (non-streaming) + await client.responses.create({ + model: 'gpt-4', + input: [message], + tools: [weatherTool], + }); + + // Test 4: Responses API with tools (streaming) + const stream2 = await client.responses.create({ + model: 'gpt-4', + input: [message], + tools: [weatherTool], + stream: true, + }); + for await (const chunk of stream2) void chunk; + }); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts new file mode 100644 index 000000000000..c5fd4fc97a72 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts @@ -0,0 +1,316 @@ +import { afterAll, describe, expect } from 'vitest'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; + +describe('OpenAI Tool Calls integration', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + const WEATHER_TOOL_DEFINITION = JSON.stringify([ + { + type: 'function', + function: { + name: 'get_weather', + description: 'Get the current weather in a given location', + parameters: { + type: 'object', + properties: { + latitude: { type: 'number', description: 'The latitude of the location' }, + longitude: { type: 'number', description: 'The longitude of the location' }, + }, + required: ['latitude', 'longitude'], + }, + }, + }, + ]); + + const CHAT_TOOL_CALLS = JSON.stringify([ + { + id: 'call_12345xyz', + type: 'function', + function: { + name: 'get_weather', + arguments: '{"latitude":48.8566,"longitude":2.3522}', + }, + }, + ]); + + const CHAT_STREAM_TOOL_CALLS = JSON.stringify([ + { + index: 0, + id: 'call_12345xyz', + type: 'function', + function: { + name: 'get_weather', + arguments: '{"latitude":48.8566,"longitude":2.3522}', + }, + }, + ]); + + const RESPONSES_TOOL_CALLS = JSON.stringify([ + { + type: 'function_call', + id: 'fc_12345xyz', + call_id: 'call_12345xyz', + name: 'get_weather', + arguments: '{"latitude":48.8566,"longitude":2.3522}', + }, + ]); + + const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE = { + transaction: 'main', + spans: expect.arrayContaining([ + // First span - chat completion with tools (non-streaming) + expect.objectContaining({ + data: { + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'manual', + 'gen_ai.system': 'openai', + 'gen_ai.request.model': 'gpt-4', + 'gen_ai.request.available_tools': WEATHER_TOOL_DEFINITION, + 'gen_ai.response.model': 'gpt-4', + 'gen_ai.response.id': 'chatcmpl-tools-123', + 'gen_ai.response.finish_reasons': '["tool_calls"]', + 'gen_ai.usage.input_tokens': 15, + 'gen_ai.usage.output_tokens': 25, + 'gen_ai.usage.total_tokens': 40, + 'openai.response.id': 'chatcmpl-tools-123', + 'openai.response.model': 'gpt-4', + 'openai.response.timestamp': '2023-03-01T06:31:40.000Z', + 'openai.usage.completion_tokens': 25, + 'openai.usage.prompt_tokens': 15, + }, + description: 'chat gpt-4', + op: 'gen_ai.chat', + origin: 'manual', + status: 'ok', + }), + // Second span - chat completion with tools and streaming + expect.objectContaining({ + data: { + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'manual', + 'gen_ai.system': 'openai', + 'gen_ai.request.model': 'gpt-4', + 'gen_ai.request.stream': true, + 'gen_ai.request.available_tools': WEATHER_TOOL_DEFINITION, + 'gen_ai.response.model': 'gpt-4', + 'gen_ai.response.id': 'chatcmpl-stream-tools-123', + 'gen_ai.response.finish_reasons': '["tool_calls"]', + 'gen_ai.response.streaming': true, + 'gen_ai.usage.input_tokens': 15, + 'gen_ai.usage.output_tokens': 25, + 'gen_ai.usage.total_tokens': 40, + 'openai.response.id': 'chatcmpl-stream-tools-123', + 'openai.response.model': 'gpt-4', + 'openai.response.timestamp': '2023-03-01T06:31:45.000Z', + 'openai.usage.completion_tokens': 25, + 'openai.usage.prompt_tokens': 15, + }, + description: 'chat gpt-4 stream-response', + op: 'gen_ai.chat', + origin: 'manual', + status: 'ok', + }), + // Third span - responses API with tools (non-streaming) + expect.objectContaining({ + data: { + 'gen_ai.operation.name': 'responses', + 'sentry.op': 'gen_ai.responses', + 'sentry.origin': 'manual', + 'gen_ai.system': 'openai', + 'gen_ai.request.model': 'gpt-4', + 'gen_ai.request.available_tools': WEATHER_TOOL_DEFINITION, + 'gen_ai.response.model': 'gpt-4', + 'gen_ai.response.id': 'resp_tools_789', + 'gen_ai.response.finish_reasons': '["completed"]', + 'gen_ai.usage.input_tokens': 8, + 'gen_ai.usage.output_tokens': 12, + 'gen_ai.usage.total_tokens': 20, + 'openai.response.id': 'resp_tools_789', + 'openai.response.model': 'gpt-4', + 'openai.response.timestamp': '2023-03-01T06:32:00.000Z', + 'openai.usage.completion_tokens': 12, + 'openai.usage.prompt_tokens': 8, + }, + description: 'responses gpt-4', + op: 'gen_ai.responses', + origin: 'manual', + status: 'ok', + }), + // Fourth span - responses API with tools and streaming + expect.objectContaining({ + data: { + 'gen_ai.operation.name': 'responses', + 'sentry.op': 'gen_ai.responses', + 'sentry.origin': 'manual', + 'gen_ai.system': 'openai', + 'gen_ai.request.model': 'gpt-4', + 'gen_ai.request.stream': true, + 'gen_ai.request.available_tools': WEATHER_TOOL_DEFINITION, + 'gen_ai.response.model': 'gpt-4', + 'gen_ai.response.id': 'resp_stream_tools_789', + 'gen_ai.response.finish_reasons': '["in_progress","completed"]', + 'gen_ai.response.streaming': true, + 'gen_ai.usage.input_tokens': 8, + 'gen_ai.usage.output_tokens': 12, + 'gen_ai.usage.total_tokens': 20, + 'openai.response.id': 'resp_stream_tools_789', + 'openai.response.model': 'gpt-4', + 'openai.response.timestamp': '2023-03-01T06:31:50.000Z', + 'openai.usage.completion_tokens': 12, + 'openai.usage.prompt_tokens': 8, + }, + description: 'responses gpt-4 stream-response', + op: 'gen_ai.responses', + origin: 'manual', + status: 'ok', + }), + ]), + }; + + const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = { + transaction: 'main', + spans: expect.arrayContaining([ + // First span - chat completion with tools (non-streaming) with PII + expect.objectContaining({ + data: { + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'manual', + 'gen_ai.system': 'openai', + 'gen_ai.request.model': 'gpt-4', + 'gen_ai.request.messages': '[{"role":"user","content":"What is the weather like in Paris today?"}]', + 'gen_ai.request.available_tools': WEATHER_TOOL_DEFINITION, + 'gen_ai.response.model': 'gpt-4', + 'gen_ai.response.id': 'chatcmpl-tools-123', + 'gen_ai.response.finish_reasons': '["tool_calls"]', + 'gen_ai.response.text': '[""]', + 'gen_ai.response.tool_calls': CHAT_TOOL_CALLS, + 'gen_ai.usage.input_tokens': 15, + 'gen_ai.usage.output_tokens': 25, + 'gen_ai.usage.total_tokens': 40, + 'openai.response.id': 'chatcmpl-tools-123', + 'openai.response.model': 'gpt-4', + 'openai.response.timestamp': '2023-03-01T06:31:40.000Z', + 'openai.usage.completion_tokens': 25, + 'openai.usage.prompt_tokens': 15, + }, + description: 'chat gpt-4', + op: 'gen_ai.chat', + origin: 'manual', + status: 'ok', + }), + // Second span - chat completion with tools and streaming with PII + expect.objectContaining({ + data: { + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'manual', + 'gen_ai.system': 'openai', + 'gen_ai.request.model': 'gpt-4', + 'gen_ai.request.stream': true, + 'gen_ai.request.messages': '[{"role":"user","content":"What is the weather like in Paris today?"}]', + 'gen_ai.request.available_tools': WEATHER_TOOL_DEFINITION, + 'gen_ai.response.model': 'gpt-4', + 'gen_ai.response.id': 'chatcmpl-stream-tools-123', + 'gen_ai.response.finish_reasons': '["tool_calls"]', + 'gen_ai.response.streaming': true, + 'gen_ai.response.tool_calls': CHAT_STREAM_TOOL_CALLS, + 'gen_ai.usage.input_tokens': 15, + 'gen_ai.usage.output_tokens': 25, + 'gen_ai.usage.total_tokens': 40, + 'openai.response.id': 'chatcmpl-stream-tools-123', + 'openai.response.model': 'gpt-4', + 'openai.response.timestamp': '2023-03-01T06:31:45.000Z', + 'openai.usage.completion_tokens': 25, + 'openai.usage.prompt_tokens': 15, + }, + description: 'chat gpt-4 stream-response', + op: 'gen_ai.chat', + origin: 'manual', + status: 'ok', + }), + // Third span - responses API with tools (non-streaming) with PII + expect.objectContaining({ + data: { + 'gen_ai.operation.name': 'responses', + 'sentry.op': 'gen_ai.responses', + 'sentry.origin': 'manual', + 'gen_ai.system': 'openai', + 'gen_ai.request.model': 'gpt-4', + 'gen_ai.request.messages': '[{"role":"user","content":"What is the weather like in Paris today?"}]', + 'gen_ai.request.available_tools': WEATHER_TOOL_DEFINITION, + 'gen_ai.response.model': 'gpt-4', + 'gen_ai.response.id': 'resp_tools_789', + 'gen_ai.response.finish_reasons': '["completed"]', + 'gen_ai.response.tool_calls': RESPONSES_TOOL_CALLS, + 'gen_ai.usage.input_tokens': 8, + 'gen_ai.usage.output_tokens': 12, + 'gen_ai.usage.total_tokens': 20, + 'openai.response.id': 'resp_tools_789', + 'openai.response.model': 'gpt-4', + 'openai.response.timestamp': '2023-03-01T06:32:00.000Z', + 'openai.usage.completion_tokens': 12, + 'openai.usage.prompt_tokens': 8, + }, + description: 'responses gpt-4', + op: 'gen_ai.responses', + origin: 'manual', + status: 'ok', + }), + // Fourth span - responses API with tools and streaming with PII + expect.objectContaining({ + data: { + 'gen_ai.operation.name': 'responses', + 'sentry.op': 'gen_ai.responses', + 'sentry.origin': 'manual', + 'gen_ai.system': 'openai', + 'gen_ai.request.model': 'gpt-4', + 'gen_ai.request.stream': true, + 'gen_ai.request.messages': '[{"role":"user","content":"What is the weather like in Paris today?"}]', + 'gen_ai.request.available_tools': WEATHER_TOOL_DEFINITION, + 'gen_ai.response.model': 'gpt-4', + 'gen_ai.response.id': 'resp_stream_tools_789', + 'gen_ai.response.finish_reasons': '["in_progress","completed"]', + 'gen_ai.response.streaming': true, + 'gen_ai.response.tool_calls': RESPONSES_TOOL_CALLS, + 'gen_ai.usage.input_tokens': 8, + 'gen_ai.usage.output_tokens': 12, + 'gen_ai.usage.total_tokens': 20, + 'openai.response.id': 'resp_stream_tools_789', + 'openai.response.model': 'gpt-4', + 'openai.response.timestamp': '2023-03-01T06:31:50.000Z', + 'openai.usage.completion_tokens': 12, + 'openai.usage.prompt_tokens': 8, + }, + description: 'responses gpt-4 stream-response', + op: 'gen_ai.responses', + origin: 'manual', + status: 'ok', + }), + ]), + }; + + createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { + test('creates openai tool calls related spans with sendDefaultPii: false', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE }) + .start() + .completed(); + }); + }); + + createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { + test('creates openai tool calls related spans with sendDefaultPii: true', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE }) + .start() + .completed(); + }); + }); +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/scenario.mjs index faf554ede924..fde651c3c1ff 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/scenario.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/scenario.mjs @@ -160,7 +160,6 @@ class MockOpenAI { }, }, tool_choice: 'auto', - tools: [], top_p: 1.0, truncation: 'disabled', user: null, @@ -216,7 +215,6 @@ class MockOpenAI { }, }, tool_choice: 'auto', - tools: [], top_p: 1.0, truncation: 'disabled', user: null, diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts index f83a592ef262..e72d0144d36c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts @@ -92,7 +92,7 @@ describe('OpenAI integration', () => { 'gen_ai.usage.total_tokens': 30, 'openai.response.id': 'chatcmpl-stream-123', 'openai.response.model': 'gpt-4', - 'openai.response.stream': true, + 'gen_ai.response.streaming': true, 'openai.response.timestamp': '2023-03-01T06:31:40.000Z', 'openai.usage.completion_tokens': 18, 'openai.usage.prompt_tokens': 12, @@ -119,7 +119,7 @@ describe('OpenAI integration', () => { 'gen_ai.usage.total_tokens': 16, 'openai.response.id': 'resp_stream_456', 'openai.response.model': 'gpt-4', - 'openai.response.stream': true, + 'gen_ai.response.streaming': true, 'openai.response.timestamp': '2023-03-01T06:31:50.000Z', 'openai.usage.completion_tokens': 10, 'openai.usage.prompt_tokens': 6, @@ -242,7 +242,7 @@ describe('OpenAI integration', () => { 'gen_ai.usage.total_tokens': 30, 'openai.response.id': 'chatcmpl-stream-123', 'openai.response.model': 'gpt-4', - 'openai.response.stream': true, + 'gen_ai.response.streaming': true, 'openai.response.timestamp': '2023-03-01T06:31:40.000Z', 'openai.usage.completion_tokens': 18, 'openai.usage.prompt_tokens': 12, @@ -271,7 +271,7 @@ describe('OpenAI integration', () => { 'gen_ai.usage.total_tokens': 16, 'openai.response.id': 'resp_stream_456', 'openai.response.model': 'gpt-4', - 'openai.response.stream': true, + 'gen_ai.response.streaming': true, 'openai.response.timestamp': '2023-03-01T06:31:50.000Z', 'openai.usage.completion_tokens': 10, 'openai.usage.prompt_tokens': 6, diff --git a/packages/core/src/utils/gen-ai-attributes.ts b/packages/core/src/utils/gen-ai-attributes.ts index c2f398a52a76..d1b45532e8a5 100644 --- a/packages/core/src/utils/gen-ai-attributes.ts +++ b/packages/core/src/utils/gen-ai-attributes.ts @@ -91,22 +91,39 @@ export const GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE = 'gen_ai.usage.output_tokens' export const GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE = 'gen_ai.usage.total_tokens'; /** - * The operation name for OpenAI API calls + * The operation name */ export const GEN_AI_OPERATION_NAME_ATTRIBUTE = 'gen_ai.operation.name'; /** - * The prompt messages sent to OpenAI (stringified JSON) + * The prompt messages * Only recorded when recordInputs is enabled */ export const GEN_AI_REQUEST_MESSAGES_ATTRIBUTE = 'gen_ai.request.messages'; /** - * The response text from OpenAI (stringified JSON array) + * The response text * Only recorded when recordOutputs is enabled */ export const GEN_AI_RESPONSE_TEXT_ATTRIBUTE = 'gen_ai.response.text'; +/** + * The available tools from incoming request + * Only recorded when recordInputs is enabled + */ +export const GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE = 'gen_ai.request.available_tools'; + +/** + * Whether the response is a streaming response + */ +export const GEN_AI_RESPONSE_STREAMING_ATTRIBUTE = 'gen_ai.response.streaming'; + +/** + * The tool calls from the response + * Only recorded when recordOutputs is enabled + */ +export const GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE = 'gen_ai.response.tool_calls'; + // ============================================================================= // OPENAI-SPECIFIC ATTRIBUTES // ============================================================================= @@ -136,11 +153,6 @@ export const OPENAI_USAGE_COMPLETION_TOKENS_ATTRIBUTE = 'openai.usage.completion */ export const OPENAI_USAGE_PROMPT_TOKENS_ATTRIBUTE = 'openai.usage.prompt_tokens'; -/** - * Whether the response is a stream response - */ -export const OPENAI_RESPONSE_STREAM_ATTRIBUTE = 'openai.response.stream'; - // ============================================================================= // OPENAI OPERATIONS // ============================================================================= diff --git a/packages/core/src/utils/openai/constants.ts b/packages/core/src/utils/openai/constants.ts index 462f007b0585..c4952b123b0f 100644 --- a/packages/core/src/utils/openai/constants.ts +++ b/packages/core/src/utils/openai/constants.ts @@ -3,6 +3,12 @@ export const OPENAI_INTEGRATION_NAME = 'OpenAI'; // https://platform.openai.com/docs/quickstart?api-mode=responses // https://platform.openai.com/docs/quickstart?api-mode=chat export const INSTRUMENTED_METHODS = ['responses.create', 'chat.completions.create'] as const; +export const RESPONSES_TOOL_CALL_EVENT_TYPES = [ + 'response.output_item.added', + 'response.function_call_arguments.delta', + 'response.function_call_arguments.done', + 'response.output_item.done', +] as const; export const RESPONSE_EVENT_TYPES = [ 'response.created', 'response.in_progress', @@ -11,4 +17,5 @@ export const RESPONSE_EVENT_TYPES = [ 'response.incomplete', 'response.queued', 'response.output_text.delta', + ...RESPONSES_TOOL_CALL_EVENT_TYPES, ] as const; diff --git a/packages/core/src/utils/openai/index.ts b/packages/core/src/utils/openai/index.ts index bb8e4f983ee7..8bd1c3625782 100644 --- a/packages/core/src/utils/openai/index.ts +++ b/packages/core/src/utils/openai/index.ts @@ -5,6 +5,7 @@ import { startSpan, startSpanManual } from '../../tracing/trace'; import type { Span, SpanAttributeValue } from '../../types-hoist/span'; import { GEN_AI_OPERATION_NAME_ATTRIBUTE, + GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE, GEN_AI_REQUEST_FREQUENCY_PENALTY_ATTRIBUTE, GEN_AI_REQUEST_MESSAGES_ATTRIBUTE, GEN_AI_REQUEST_MODEL_ATTRIBUTE, @@ -14,6 +15,7 @@ import { GEN_AI_REQUEST_TOP_P_ATTRIBUTE, GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE, GEN_AI_RESPONSE_TEXT_ATTRIBUTE, + GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE, GEN_AI_SYSTEM_ATTRIBUTE, } from '../gen-ai-attributes'; import { OPENAI_INTEGRATION_NAME } from './constants'; @@ -50,6 +52,24 @@ function extractRequestAttributes(args: unknown[], methodPath: string): Record 0 && typeof args[0] === 'object' && args[0] !== null) { + const params = args[0] as Record; + + const tools = Array.isArray(params.tools) ? params.tools : []; + const hasWebSearchOptions = params.web_search_options && typeof params.web_search_options === 'object'; + const webSearchOptions = hasWebSearchOptions + ? [{ type: 'web_search_options', ...(params.web_search_options as Record) }] + : []; + + const availableTools = [...tools, ...webSearchOptions]; + + if (availableTools.length > 0) { + attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE] = JSON.stringify(availableTools); + } + } + if (args.length > 0 && typeof args[0] === 'object' && args[0] !== null) { const params = args[0] as Record; @@ -70,7 +90,7 @@ function extractRequestAttributes(args: unknown[], methodPath: string): Record choice.message?.tool_calls) + .filter(calls => Array.isArray(calls) && calls.length > 0) + .flat(); + + if (toolCalls.length > 0) { + span.setAttributes({ + [GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]: JSON.stringify(toolCalls), + }); + } + } } } /** * Add attributes for Responses API responses */ -function addResponsesApiAttributes(span: Span, response: OpenAIResponseObject): void { +function addResponsesApiAttributes(span: Span, response: OpenAIResponseObject, recordOutputs?: boolean): void { setCommonResponseAttributes(span, response.id, response.model, response.created_at); if (response.status) { span.setAttributes({ @@ -110,6 +144,24 @@ function addResponsesApiAttributes(span: Span, response: OpenAIResponseObject): response.usage.total_tokens, ); } + + // Extract function calls from output (only if recordOutputs is true) + if (recordOutputs) { + const responseWithOutput = response as OpenAIResponseObject & { output?: unknown[] }; + if (Array.isArray(responseWithOutput.output) && responseWithOutput.output.length > 0) { + // Filter for function_call type objects in the output array + const functionCalls = responseWithOutput.output.filter( + (item): unknown => + typeof item === 'object' && item !== null && (item as Record).type === 'function_call', + ); + + if (functionCalls.length > 0) { + span.setAttributes({ + [GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]: JSON.stringify(functionCalls), + }); + } + } + } } /** @@ -122,13 +174,13 @@ function addResponseAttributes(span: Span, result: unknown, recordOutputs?: bool const response = result as OpenAiResponse; if (isChatCompletionResponse(response)) { - addChatCompletionAttributes(span, response); + addChatCompletionAttributes(span, response, recordOutputs); if (recordOutputs && response.choices?.length) { const responseTexts = response.choices.map(choice => choice.message?.content || ''); span.setAttributes({ [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: JSON.stringify(responseTexts) }); } } else if (isResponsesApiResponse(response)) { - addResponsesApiAttributes(span, response); + addResponsesApiAttributes(span, response, recordOutputs); if (recordOutputs && response.output_text) { span.setAttributes({ [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: response.output_text }); } diff --git a/packages/core/src/utils/openai/streaming.ts b/packages/core/src/utils/openai/streaming.ts index 88d4c6adf893..2791e715920e 100644 --- a/packages/core/src/utils/openai/streaming.ts +++ b/packages/core/src/utils/openai/streaming.ts @@ -3,12 +3,18 @@ import { SPAN_STATUS_ERROR } from '../../tracing'; import type { Span } from '../../types-hoist/span'; import { GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE, + GEN_AI_RESPONSE_STREAMING_ATTRIBUTE, GEN_AI_RESPONSE_TEXT_ATTRIBUTE, - OPENAI_RESPONSE_STREAM_ATTRIBUTE, + GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE, } from '../gen-ai-attributes'; import { RESPONSE_EVENT_TYPES } from './constants'; import type { OpenAIResponseObject } from './types'; -import { type ChatCompletionChunk, type ResponseStreamingEvent } from './types'; +import { + type ChatCompletionChunk, + type ChatCompletionToolCall, + type ResponseFunctionCall, + type ResponseStreamingEvent, +} from './types'; import { isChatCompletionChunk, isResponsesApiStreamEvent, @@ -38,6 +44,49 @@ interface StreamingState { completionTokens: number | undefined; /** Total number of tokens used (prompt + completion). */ totalTokens: number | undefined; + /** + * Accumulated tool calls from Chat Completion streaming, indexed by tool call index. + * @see https://platform.openai.com/docs/guides/function-calling?api-mode=chat#streaming + */ + chatCompletionToolCalls: Record; + /** + * Accumulated function calls from Responses API streaming. + * @see https://platform.openai.com/docs/guides/function-calling?api-mode=responses#streaming + */ + responsesApiToolCalls: Array; +} + +/** + * Processes tool calls from a chat completion chunk delta. + * Follows the pattern: accumulate by index, then convert to array at the end. + * + * @param toolCalls - Array of tool calls from the delta. + * @param state - The current streaming state to update. + * + * @see https://platform.openai.com/docs/guides/function-calling#streaming + */ +function processChatCompletionToolCalls(toolCalls: ChatCompletionToolCall[], state: StreamingState): void { + for (const toolCall of toolCalls) { + const index = toolCall.index; + if (index === undefined || !toolCall.function) continue; + + // Initialize tool call if this is the first chunk for this index + if (!(index in state.chatCompletionToolCalls)) { + state.chatCompletionToolCalls[index] = { + ...toolCall, + function: { + name: toolCall.function.name, + arguments: toolCall.function.arguments || '', + }, + }; + } else { + // Accumulate function arguments from subsequent chunks + const existingToolCall = state.chatCompletionToolCalls[index]; + if (toolCall.function.arguments && existingToolCall?.function) { + existingToolCall.function.arguments += toolCall.function.arguments; + } + } + } } /** @@ -64,8 +113,15 @@ function processChatCompletionChunk(chunk: ChatCompletionChunk, state: Streaming } for (const choice of chunk.choices ?? []) { - if (recordOutputs && choice.delta?.content) { - state.responseTexts.push(choice.delta.content); + if (recordOutputs) { + if (choice.delta?.content) { + state.responseTexts.push(choice.delta.content); + } + + // Handle tool calls from delta + if (choice.delta?.tool_calls) { + processChatCompletionToolCalls(choice.delta.tool_calls, state); + } } if (choice.finish_reason) { state.finishReasons.push(choice.finish_reason); @@ -109,9 +165,17 @@ function processResponsesApiEvent( return; } - if (recordOutputs && event.type === 'response.output_text.delta' && 'delta' in event && event.delta) { - state.responseTexts.push(event.delta); - return; + // Handle output text delta + if (recordOutputs) { + // Handle tool call events for Responses API + if (event.type === 'response.output_item.done' && 'item' in event) { + state.responsesApiToolCalls.push(event.item); + } + + if (event.type === 'response.output_text.delta' && 'delta' in event && event.delta) { + state.responseTexts.push(event.delta); + return; + } } if ('response' in event) { @@ -166,6 +230,8 @@ export async function* instrumentStream( promptTokens: undefined, completionTokens: undefined, totalTokens: undefined, + chatCompletionToolCalls: {}, + responsesApiToolCalls: [], }; try { @@ -182,7 +248,7 @@ export async function* instrumentStream( setTokenUsageAttributes(span, state.promptTokens, state.completionTokens, state.totalTokens); span.setAttributes({ - [OPENAI_RESPONSE_STREAM_ATTRIBUTE]: true, + [GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true, }); if (state.finishReasons.length) { @@ -197,6 +263,16 @@ export async function* instrumentStream( }); } + // Set tool calls attribute if any were accumulated + const chatCompletionToolCallsArray = Object.values(state.chatCompletionToolCalls); + const allToolCalls = [...chatCompletionToolCallsArray, ...state.responsesApiToolCalls]; + + if (allToolCalls.length > 0) { + span.setAttributes({ + [GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]: JSON.stringify(allToolCalls), + }); + } + span.end(); } } diff --git a/packages/core/src/utils/openai/types.ts b/packages/core/src/utils/openai/types.ts index a5854868fbe2..7ac8fb8d7b91 100644 --- a/packages/core/src/utils/openai/types.ts +++ b/packages/core/src/utils/openai/types.ts @@ -50,6 +50,7 @@ export interface OpenAiChatCompletionObject { content: string | null; refusal?: string | null; annotations?: Array; // Depends on whether annotations are enabled + tool_calls?: Array; }; logprobs?: unknown | null; finish_reason: string | null; @@ -144,7 +145,11 @@ export type ResponseStreamingEvent = | ResponseCompletedEvent | ResponseIncompleteEvent | ResponseQueuedEvent - | ResponseOutputTextDeltaEvent; + | ResponseOutputTextDeltaEvent + | ResponseOutputItemAddedEvent + | ResponseFunctionCallArgumentsDeltaEvent + | ResponseFunctionCallArgumentsDoneEvent + | ResponseOutputItemDoneEvent; interface ResponseCreatedEvent { type: 'response.created'; @@ -191,6 +196,78 @@ interface ResponseQueuedEvent { sequence_number: number; } +/** + * @see https://platform.openai.com/docs/api-reference/realtime-server-events/response/output_item/added + */ +interface ResponseOutputItemAddedEvent { + type: 'response.output_item.added'; + output_index: number; + item: unknown; + event_id: string; + response_id: string; +} + +/** + * @see https://platform.openai.com/docs/api-reference/realtime-server-events/response/function_call_arguments/delta + */ +interface ResponseFunctionCallArgumentsDeltaEvent { + type: 'response.function_call_arguments.delta'; + item_id: string; + output_index: number; + delta: string; + call_id: string; + event_id: string; + response_id: string; +} + +/** + * @see https://platform.openai.com/docs/api-reference/realtime-server-events/response/function_call_arguments/done + */ +interface ResponseFunctionCallArgumentsDoneEvent { + type: 'response.function_call_arguments.done'; + response_id: string; + item_id: string; + output_index: number; + arguments: string; + call_id: string; + event_id: string; +} + +/** + * @see https://platform.openai.com/docs/api-reference/realtime-server-events/response/output_item/done + */ +interface ResponseOutputItemDoneEvent { + type: 'response.output_item.done'; + response_id: string; + output_index: number; + item: unknown; + event_id: string; +} + +/** + * Tool call object for Chat Completion streaming + */ +export interface ChatCompletionToolCall { + index?: number; // Present for streaming responses + id: string; + type?: string; // Could be missing for streaming responses + function?: { + name: string; + arguments?: string; + }; +} + +/** + * Function call object for Responses API + */ +export interface ResponseFunctionCall { + type: string; + id: string; + call_id: string; + name: string; + arguments: string; +} + /** * Chat Completion streaming chunk type * @see https://platform.openai.com/docs/api-reference/chat-streaming/streaming @@ -209,7 +286,7 @@ export interface ChatCompletionChunk { role: string; function_call?: object; refusal?: string | null; - tool_calls?: Array; + tool_calls?: Array; }; logprobs?: unknown | null; finish_reason?: string | null;