From 7368f0bda2a462fecbbb224319bcf8592045e53f Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 3 Sep 2025 14:05:44 +0200 Subject: [PATCH 1/2] fix(node): Add `origin` for OpenAI spans & test auto instrumentation We used to mock the OpenAI client and manually instrument this. This adds a test for actual auto-instrumentation of OpenAI, and adjusts the origin to be fixed. --- .../node-integration-tests/package.json | 1 + .../tracing/openai/openai-tool-calls/test.ts | 32 +++---- .../tracing/openai/scenario-root-span.mjs | 61 ++++++++++++ .../suites/tracing/openai/test.ts | 95 ++++++++++++++----- packages/core/src/utils/openai/index.ts | 2 + yarn.lock | 5 + 6 files changed, 156 insertions(+), 40 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/openai/scenario-root-span.mjs diff --git a/dev-packages/node-integration-tests/package.json b/dev-packages/node-integration-tests/package.json index e779ec11671f..0ee77860f21d 100644 --- a/dev-packages/node-integration-tests/package.json +++ b/dev-packages/node-integration-tests/package.json @@ -58,6 +58,7 @@ "nock": "^13.5.5", "node-cron": "^3.0.3", "node-schedule": "^2.1.1", + "openai": "5.18.1", "pg": "8.16.0", "postgres": "^3.4.7", "proxy": "^2.1.1", diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts index c5fd4fc97a72..a03181d5625b 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts @@ -65,7 +65,7 @@ describe('OpenAI Tool Calls integration', () => { data: { 'gen_ai.operation.name': 'chat', 'sentry.op': 'gen_ai.chat', - 'sentry.origin': 'manual', + 'sentry.origin': 'auto.function.openai', 'gen_ai.system': 'openai', 'gen_ai.request.model': 'gpt-4', 'gen_ai.request.available_tools': WEATHER_TOOL_DEFINITION, @@ -83,7 +83,7 @@ describe('OpenAI Tool Calls integration', () => { }, description: 'chat gpt-4', op: 'gen_ai.chat', - origin: 'manual', + origin: 'auto.function.openai', status: 'ok', }), // Second span - chat completion with tools and streaming @@ -91,7 +91,7 @@ describe('OpenAI Tool Calls integration', () => { data: { 'gen_ai.operation.name': 'chat', 'sentry.op': 'gen_ai.chat', - 'sentry.origin': 'manual', + 'sentry.origin': 'auto.function.openai', 'gen_ai.system': 'openai', 'gen_ai.request.model': 'gpt-4', 'gen_ai.request.stream': true, @@ -111,7 +111,7 @@ describe('OpenAI Tool Calls integration', () => { }, description: 'chat gpt-4 stream-response', op: 'gen_ai.chat', - origin: 'manual', + origin: 'auto.function.openai', status: 'ok', }), // Third span - responses API with tools (non-streaming) @@ -119,7 +119,7 @@ describe('OpenAI Tool Calls integration', () => { data: { 'gen_ai.operation.name': 'responses', 'sentry.op': 'gen_ai.responses', - 'sentry.origin': 'manual', + 'sentry.origin': 'auto.function.openai', 'gen_ai.system': 'openai', 'gen_ai.request.model': 'gpt-4', 'gen_ai.request.available_tools': WEATHER_TOOL_DEFINITION, @@ -137,7 +137,7 @@ describe('OpenAI Tool Calls integration', () => { }, description: 'responses gpt-4', op: 'gen_ai.responses', - origin: 'manual', + origin: 'auto.function.openai', status: 'ok', }), // Fourth span - responses API with tools and streaming @@ -145,7 +145,7 @@ describe('OpenAI Tool Calls integration', () => { data: { 'gen_ai.operation.name': 'responses', 'sentry.op': 'gen_ai.responses', - 'sentry.origin': 'manual', + 'sentry.origin': 'auto.function.openai', 'gen_ai.system': 'openai', 'gen_ai.request.model': 'gpt-4', 'gen_ai.request.stream': true, @@ -165,7 +165,7 @@ describe('OpenAI Tool Calls integration', () => { }, description: 'responses gpt-4 stream-response', op: 'gen_ai.responses', - origin: 'manual', + origin: 'auto.function.openai', status: 'ok', }), ]), @@ -179,7 +179,7 @@ describe('OpenAI Tool Calls integration', () => { data: { 'gen_ai.operation.name': 'chat', 'sentry.op': 'gen_ai.chat', - 'sentry.origin': 'manual', + 'sentry.origin': 'auto.function.openai', 'gen_ai.system': 'openai', 'gen_ai.request.model': 'gpt-4', 'gen_ai.request.messages': '[{"role":"user","content":"What is the weather like in Paris today?"}]', @@ -200,7 +200,7 @@ describe('OpenAI Tool Calls integration', () => { }, description: 'chat gpt-4', op: 'gen_ai.chat', - origin: 'manual', + origin: 'auto.function.openai', status: 'ok', }), // Second span - chat completion with tools and streaming with PII @@ -208,7 +208,7 @@ describe('OpenAI Tool Calls integration', () => { data: { 'gen_ai.operation.name': 'chat', 'sentry.op': 'gen_ai.chat', - 'sentry.origin': 'manual', + 'sentry.origin': 'auto.function.openai', 'gen_ai.system': 'openai', 'gen_ai.request.model': 'gpt-4', 'gen_ai.request.stream': true, @@ -230,7 +230,7 @@ describe('OpenAI Tool Calls integration', () => { }, description: 'chat gpt-4 stream-response', op: 'gen_ai.chat', - origin: 'manual', + origin: 'auto.function.openai', status: 'ok', }), // Third span - responses API with tools (non-streaming) with PII @@ -238,7 +238,7 @@ describe('OpenAI Tool Calls integration', () => { data: { 'gen_ai.operation.name': 'responses', 'sentry.op': 'gen_ai.responses', - 'sentry.origin': 'manual', + 'sentry.origin': 'auto.function.openai', 'gen_ai.system': 'openai', 'gen_ai.request.model': 'gpt-4', 'gen_ai.request.messages': '[{"role":"user","content":"What is the weather like in Paris today?"}]', @@ -258,7 +258,7 @@ describe('OpenAI Tool Calls integration', () => { }, description: 'responses gpt-4', op: 'gen_ai.responses', - origin: 'manual', + origin: 'auto.function.openai', status: 'ok', }), // Fourth span - responses API with tools and streaming with PII @@ -266,7 +266,7 @@ describe('OpenAI Tool Calls integration', () => { data: { 'gen_ai.operation.name': 'responses', 'sentry.op': 'gen_ai.responses', - 'sentry.origin': 'manual', + 'sentry.origin': 'auto.function.openai', 'gen_ai.system': 'openai', 'gen_ai.request.model': 'gpt-4', 'gen_ai.request.stream': true, @@ -288,7 +288,7 @@ describe('OpenAI Tool Calls integration', () => { }, description: 'responses gpt-4 stream-response', op: 'gen_ai.responses', - origin: 'manual', + origin: 'auto.function.openai', status: 'ok', }), ]), diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/scenario-root-span.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/scenario-root-span.mjs new file mode 100644 index 000000000000..d1a06e5ccbb2 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/scenario-root-span.mjs @@ -0,0 +1,61 @@ +import express from 'express'; +import OpenAI from 'openai'; + +const PORT = 3333; + +function startMockOpenAiServer() { + const app = express(); + app.use(express.json()); + + app.post('/openai/chat/completions', (req, res) => { + res.send({ + id: 'chatcmpl-mock123', + object: 'chat.completion', + created: 1677652288, + model: req.body.model, + system_fingerprint: 'fp_44709d6fcb', + choices: [ + { + index: 0, + message: { + role: 'assistant', + content: 'Hello from OpenAI mock!', + }, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: 10, + completion_tokens: 15, + total_tokens: 25, + }, + }); + }); + return app.listen(PORT); +} + +async function run() { + const server = startMockOpenAiServer(); + + const client = new OpenAI({ + baseURL: `http://localhost:${PORT}/openai`, + apiKey: 'mock-api-key', + }); + + const response = await client.chat.completions.create({ + model: 'gpt-3.5-turbo', + messages: [ + { role: 'system', content: 'You are a helpful assistant.' }, + { role: 'user', content: 'What is the capital of France?' }, + ], + temperature: 0.7, + max_tokens: 100, + }); + + // eslint-disable-next-line no-console + console.log(JSON.stringify(response)); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts index e72d0144d36c..967cf55bb130 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts @@ -14,7 +14,7 @@ describe('OpenAI integration', () => { data: { 'gen_ai.operation.name': 'chat', 'sentry.op': 'gen_ai.chat', - 'sentry.origin': 'manual', + 'sentry.origin': 'auto.function.openai', 'gen_ai.system': 'openai', 'gen_ai.request.model': 'gpt-3.5-turbo', 'gen_ai.request.temperature': 0.7, @@ -32,7 +32,7 @@ describe('OpenAI integration', () => { }, description: 'chat gpt-3.5-turbo', op: 'gen_ai.chat', - origin: 'manual', + origin: 'auto.function.openai', status: 'ok', }), // Second span - responses API @@ -40,7 +40,7 @@ describe('OpenAI integration', () => { data: { 'gen_ai.operation.name': 'responses', 'sentry.op': 'gen_ai.responses', - 'sentry.origin': 'manual', + 'sentry.origin': 'auto.function.openai', 'gen_ai.system': 'openai', 'gen_ai.request.model': 'gpt-3.5-turbo', 'gen_ai.response.model': 'gpt-3.5-turbo', @@ -57,7 +57,7 @@ describe('OpenAI integration', () => { }, description: 'responses gpt-3.5-turbo', op: 'gen_ai.responses', - origin: 'manual', + origin: 'auto.function.openai', status: 'ok', }), // Third span - error handling @@ -65,13 +65,13 @@ describe('OpenAI integration', () => { data: { 'gen_ai.operation.name': 'chat', 'sentry.op': 'gen_ai.chat', - 'sentry.origin': 'manual', + 'sentry.origin': 'auto.function.openai', 'gen_ai.system': 'openai', 'gen_ai.request.model': 'error-model', }, description: 'chat error-model', op: 'gen_ai.chat', - origin: 'manual', + origin: 'auto.function.openai', status: 'unknown_error', }), // Fourth span - chat completions streaming @@ -79,7 +79,7 @@ describe('OpenAI integration', () => { data: { 'gen_ai.operation.name': 'chat', 'sentry.op': 'gen_ai.chat', - 'sentry.origin': 'manual', + 'sentry.origin': 'auto.function.openai', 'gen_ai.system': 'openai', 'gen_ai.request.model': 'gpt-4', 'gen_ai.request.temperature': 0.8, @@ -99,7 +99,7 @@ describe('OpenAI integration', () => { }, description: 'chat gpt-4 stream-response', op: 'gen_ai.chat', - origin: 'manual', + origin: 'auto.function.openai', status: 'ok', }), // Fifth span - responses API streaming @@ -107,7 +107,7 @@ describe('OpenAI integration', () => { data: { 'gen_ai.operation.name': 'responses', 'sentry.op': 'gen_ai.responses', - 'sentry.origin': 'manual', + 'sentry.origin': 'auto.function.openai', 'gen_ai.system': 'openai', 'gen_ai.request.model': 'gpt-4', 'gen_ai.request.stream': true, @@ -126,7 +126,7 @@ describe('OpenAI integration', () => { }, description: 'responses gpt-4 stream-response', op: 'gen_ai.responses', - origin: 'manual', + origin: 'auto.function.openai', status: 'ok', }), // Sixth span - error handling in streaming context @@ -137,11 +137,11 @@ describe('OpenAI integration', () => { 'gen_ai.request.stream': true, 'gen_ai.system': 'openai', 'sentry.op': 'gen_ai.chat', - 'sentry.origin': 'manual', + 'sentry.origin': 'auto.function.openai', }, description: 'chat error-model stream-response', op: 'gen_ai.chat', - origin: 'manual', + origin: 'auto.function.openai', status: 'internal_error', }), ]), @@ -155,7 +155,7 @@ describe('OpenAI integration', () => { data: { 'gen_ai.operation.name': 'chat', 'sentry.op': 'gen_ai.chat', - 'sentry.origin': 'manual', + 'sentry.origin': 'auto.function.openai', 'gen_ai.system': 'openai', 'gen_ai.request.model': 'gpt-3.5-turbo', 'gen_ai.request.temperature': 0.7, @@ -176,7 +176,7 @@ describe('OpenAI integration', () => { }, description: 'chat gpt-3.5-turbo', op: 'gen_ai.chat', - origin: 'manual', + origin: 'auto.function.openai', status: 'ok', }), // Second span - responses API with PII @@ -184,7 +184,7 @@ describe('OpenAI integration', () => { data: { 'gen_ai.operation.name': 'responses', 'sentry.op': 'gen_ai.responses', - 'sentry.origin': 'manual', + 'sentry.origin': 'auto.function.openai', 'gen_ai.system': 'openai', 'gen_ai.request.model': 'gpt-3.5-turbo', 'gen_ai.request.messages': '"Translate this to French: Hello"', @@ -203,7 +203,7 @@ describe('OpenAI integration', () => { }, description: 'responses gpt-3.5-turbo', op: 'gen_ai.responses', - origin: 'manual', + origin: 'auto.function.openai', status: 'ok', }), // Third span - error handling with PII @@ -211,14 +211,14 @@ describe('OpenAI integration', () => { data: { 'gen_ai.operation.name': 'chat', 'sentry.op': 'gen_ai.chat', - 'sentry.origin': 'manual', + 'sentry.origin': 'auto.function.openai', 'gen_ai.system': 'openai', 'gen_ai.request.model': 'error-model', 'gen_ai.request.messages': '[{"role":"user","content":"This will fail"}]', }, description: 'chat error-model', op: 'gen_ai.chat', - origin: 'manual', + origin: 'auto.function.openai', status: 'unknown_error', }), // Fourth span - chat completions streaming with PII @@ -226,7 +226,7 @@ describe('OpenAI integration', () => { data: expect.objectContaining({ 'gen_ai.operation.name': 'chat', 'sentry.op': 'gen_ai.chat', - 'sentry.origin': 'manual', + 'sentry.origin': 'auto.function.openai', 'gen_ai.system': 'openai', 'gen_ai.request.model': 'gpt-4', 'gen_ai.request.temperature': 0.8, @@ -249,7 +249,7 @@ describe('OpenAI integration', () => { }), description: 'chat gpt-4 stream-response', op: 'gen_ai.chat', - origin: 'manual', + origin: 'auto.function.openai', status: 'ok', }), // Fifth span - responses API streaming with PII @@ -257,7 +257,7 @@ describe('OpenAI integration', () => { data: expect.objectContaining({ 'gen_ai.operation.name': 'responses', 'sentry.op': 'gen_ai.responses', - 'sentry.origin': 'manual', + 'sentry.origin': 'auto.function.openai', 'gen_ai.system': 'openai', 'gen_ai.request.model': 'gpt-4', 'gen_ai.request.stream': true, @@ -278,7 +278,7 @@ describe('OpenAI integration', () => { }), description: 'responses gpt-4 stream-response', op: 'gen_ai.responses', - origin: 'manual', + origin: 'auto.function.openai', status: 'ok', }), // Sixth span - error handling in streaming context with PII @@ -290,11 +290,11 @@ describe('OpenAI integration', () => { 'gen_ai.request.messages': '[{"role":"user","content":"This will fail"}]', 'gen_ai.system': 'openai', 'sentry.op': 'gen_ai.chat', - 'sentry.origin': 'manual', + 'sentry.origin': 'auto.function.openai', }, description: 'chat error-model stream-response', op: 'gen_ai.chat', - origin: 'manual', + origin: 'auto.function.openai', status: 'internal_error', }), ]), @@ -350,4 +350,51 @@ describe('OpenAI integration', () => { .completed(); }); }); + + createEsmAndCjsTests(__dirname, 'scenario-root-span.mjs', 'instrument.mjs', (createRunner, test) => { + test('it works without a wrapping span', async () => { + await createRunner() + // First the span that our mock express server is emitting, unrelated to this test + .expect({ + transaction: { + transaction: 'POST /openai/chat/completions', + }, + }) + .expect({ + transaction: { + transaction: 'chat gpt-3.5-turbo', + contexts: { + trace: { + span_id: expect.any(String), + trace_id: expect.any(String), + data: { + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.function.openai', + 'gen_ai.system': 'openai', + 'gen_ai.request.model': 'gpt-3.5-turbo', + 'gen_ai.request.temperature': 0.7, + 'gen_ai.response.model': 'gpt-3.5-turbo', + 'gen_ai.response.id': 'chatcmpl-mock123', + 'gen_ai.response.finish_reasons': '["stop"]', + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 15, + 'gen_ai.usage.total_tokens': 25, + 'openai.response.id': 'chatcmpl-mock123', + 'openai.response.model': 'gpt-3.5-turbo', + 'openai.response.timestamp': '2023-03-01T06:31:28.000Z', + 'openai.usage.completion_tokens': 15, + 'openai.usage.prompt_tokens': 10, + }, + op: 'gen_ai.chat', + origin: 'auto.function.openai', + status: 'ok', + }, + }, + }, + }) + .start() + .completed(); + }); + }); }); diff --git a/packages/core/src/utils/openai/index.ts b/packages/core/src/utils/openai/index.ts index 3fb8b1bf8b98..060117d52964 100644 --- a/packages/core/src/utils/openai/index.ts +++ b/packages/core/src/utils/openai/index.ts @@ -1,5 +1,6 @@ import { getCurrentScope } from '../../currentScopes'; import { captureException } from '../../exports'; +import { SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '../../semanticAttributes'; import { SPAN_STATUS_ERROR } from '../../tracing'; import { startSpan, startSpanManual } from '../../tracing/trace'; import type { Span, SpanAttributeValue } from '../../types-hoist/span'; @@ -49,6 +50,7 @@ function extractRequestAttributes(args: unknown[], methodPath: string): Record = { [GEN_AI_SYSTEM_ATTRIBUTE]: 'openai', [GEN_AI_OPERATION_NAME_ATTRIBUTE]: getOperationName(methodPath), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.function.openai', }; // Chat completion API accepts web_search_options and tools as parameters diff --git a/yarn.lock b/yarn.lock index d4a6aa4ec1be..225ff96259e7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -23845,6 +23845,11 @@ open@^9.1.0: is-inside-container "^1.0.0" is-wsl "^2.2.0" +openai@5.18.1: + version "5.18.1" + resolved "https://registry.yarnpkg.com/openai/-/openai-5.18.1.tgz#1c4884aefcada7ec684771e03c860c381f1902c1" + integrity sha512-iXSOfLlOL+jgnFr5CGrB2SEZw5C92o1nrFW2SasoAXj4QxGhfeJPgg8zkX+vaCfX80cT6CWjgaGnq7z9XzbyRw== + opener@^1.5.2: version "1.5.2" resolved "https://registry.yarnpkg.com/opener/-/opener-1.5.2.tgz#5d37e1f35077b9dcac4301372271afdeb2a13598" From 1bfdb5ae1aa944f66f50dab0223da4037b2193cc Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 3 Sep 2025 14:19:51 +0200 Subject: [PATCH 2/2] fix cloudflare test --- .../cloudflare-integration-tests/suites/tracing/openai/test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/openai/test.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/openai/test.ts index 1dc4ca077665..fc38fc6339b2 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/openai/test.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/openai/test.ts @@ -31,7 +31,7 @@ it('traces a basic chat completion request', async () => { }), description: 'chat gpt-3.5-turbo', op: 'gen_ai.chat', - origin: 'manual', + origin: 'auto.function.openai', }), ]), );