diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/CHANGELOG.md b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/CHANGELOG.md index 91065edbc1..a695620657 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/CHANGELOG.md +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +- Record prompt and completion events regardless of span sampling decision. + ([#3226](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3226)) + ## Version 2.1b0 (2025-01-18) - Coerce openai response_format to semconv format diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py index 307b312fca..072365abb7 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py @@ -56,11 +56,8 @@ def traced_method(wrapped, instance, args, kwargs): attributes=span_attributes, end_on_exit=False, ) as span: - if span.is_recording(): - for message in kwargs.get("messages", []): - event_logger.emit( - message_to_event(message, capture_content) - ) + for message in kwargs.get("messages", []): + event_logger.emit(message_to_event(message, capture_content)) start = default_timer() result = None @@ -76,6 +73,9 @@ def traced_method(wrapped, instance, args, kwargs): _set_response_attributes( span, result, event_logger, capture_content ) + for choice in getattr(result, "choices", []): + event_logger.emit(choice_to_event(choice, capture_content)) + span.end() return result @@ -114,11 +114,8 @@ async def traced_method(wrapped, instance, args, kwargs): attributes=span_attributes, end_on_exit=False, ) as span: - if span.is_recording(): - for message in kwargs.get("messages", []): - event_logger.emit( - message_to_event(message, capture_content) - ) + for message in kwargs.get("messages", []): + event_logger.emit(message_to_event(message, capture_content)) start = default_timer() result = None @@ -134,6 +131,9 @@ async def traced_method(wrapped, instance, args, kwargs): _set_response_attributes( span, result, event_logger, capture_content ) + for choice in getattr(result, "choices", []): + event_logger.emit(choice_to_event(choice, capture_content)) + span.end() return result @@ -228,12 +228,8 @@ def _set_response_attributes( ) if getattr(result, "choices", None): - choices = result.choices - for choice in choices: - event_logger.emit(choice_to_event(choice, capture_content)) - finish_reasons = [] - for choice in choices: + for choice in result.choices: finish_reasons.append(choice.finish_reason or "error") set_span_attribute( @@ -333,42 +329,43 @@ def setup(self): def cleanup(self): if self._span_started: - if self.response_model: + if self.span.is_recording(): + if self.response_model: + set_span_attribute( + self.span, + GenAIAttributes.GEN_AI_RESPONSE_MODEL, + self.response_model, + ) + + if self.response_id: + set_span_attribute( + self.span, + GenAIAttributes.GEN_AI_RESPONSE_ID, + self.response_id, + ) + set_span_attribute( self.span, - GenAIAttributes.GEN_AI_RESPONSE_MODEL, - self.response_model, + GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS, + self.prompt_tokens, ) - - if self.response_id: set_span_attribute( self.span, - GenAIAttributes.GEN_AI_RESPONSE_ID, - self.response_id, + GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, + self.completion_tokens, ) - set_span_attribute( - self.span, - GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS, - self.prompt_tokens, - ) - set_span_attribute( - self.span, - GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, - self.completion_tokens, - ) - - set_span_attribute( - self.span, - GenAIAttributes.GEN_AI_OPENAI_RESPONSE_SERVICE_TIER, - self.service_tier, - ) + set_span_attribute( + self.span, + GenAIAttributes.GEN_AI_OPENAI_RESPONSE_SERVICE_TIER, + self.service_tier, + ) - set_span_attribute( - self.span, - GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS, - self.finish_reasons, - ) + set_span_attribute( + self.span, + GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS, + self.finish_reasons, + ) for idx, choice in enumerate(self.choice_buffers): message = {"role": "assistant"} diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_async_chat_completion_streaming_unsampled.yaml b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_async_chat_completion_streaming_unsampled.yaml new file mode 100644 index 0000000000..efffcd7423 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_async_chat_completion_streaming_unsampled.yaml @@ -0,0 +1,117 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "role": "user", + "content": "Say this is a test" + } + ], + "model": "gpt-4", + "stream": true, + "stream_options": { + "include_usage": true + } + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '142' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.26.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.26.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |+ + data: {"id":"chatcmpl-ASv9ejXDUtAhGOJJxWuw026zdinc4","object":"chat.completion.chunk","created":1731456250,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} + + data: {"id":"chatcmpl-ASv9ejXDUtAhGOJJxWuw026zdinc4","object":"chat.completion.chunk","created":1731456250,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"This"},"logprobs":null,"finish_reason":null}],"usage":null} + + data: {"id":"chatcmpl-ASv9ejXDUtAhGOJJxWuw026zdinc4","object":"chat.completion.chunk","created":1731456250,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" is"},"logprobs":null,"finish_reason":null}],"usage":null} + + data: {"id":"chatcmpl-ASv9ejXDUtAhGOJJxWuw026zdinc4","object":"chat.completion.chunk","created":1731456250,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" a"},"logprobs":null,"finish_reason":null}],"usage":null} + + data: {"id":"chatcmpl-ASv9ejXDUtAhGOJJxWuw026zdinc4","object":"chat.completion.chunk","created":1731456250,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" test"},"logprobs":null,"finish_reason":null}],"usage":null} + + data: {"id":"chatcmpl-ASv9ejXDUtAhGOJJxWuw026zdinc4","object":"chat.completion.chunk","created":1731456250,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} + + data: {"id":"chatcmpl-ASv9ejXDUtAhGOJJxWuw026zdinc4","object":"chat.completion.chunk","created":1731456250,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} + + data: {"id":"chatcmpl-ASv9ejXDUtAhGOJJxWuw026zdinc4","object":"chat.completion.chunk","created":1731456250,"model":"gpt-4-0613","system_fingerprint":null,"choices":[],"usage":{"prompt_tokens":12,"completion_tokens":5,"total_tokens":17,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}} + + data: [DONE] + + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8e1a80bd2f31e1e5-MRS + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Wed, 13 Nov 2024 00:04:11 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: test_openai_org_id + openai-processing-ms: + - '196' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '1000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '999977' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 1ms + x-request-id: + - req_cc9204ae23338b130df11c8c5b5f31af + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_chat_completion_with_content_span_unsampled.yaml b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_chat_completion_with_content_span_unsampled.yaml new file mode 100644 index 0000000000..2abb443fe3 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_chat_completion_with_content_span_unsampled.yaml @@ -0,0 +1,134 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "role": "user", + "content": "Say this is a test" + } + ], + "model": "gpt-4o-mini", + "stream": false + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '106' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.54.3 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.54.3 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.6 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "id": "chatcmpl-ASYMQRl3A3DXL9FWCK9tnGRcKIO7q", + "object": "chat.completion", + "created": 1731368630, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "This is a test.", + "refusal": null + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 12, + "completion_tokens": 5, + "total_tokens": 17, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "system_fingerprint": "fp_0ba0d124f1" + } + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8e122593ff368bc8-SIN + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Mon, 11 Nov 2024 23:43:50 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + content-length: + - '765' + openai-organization: test_openai_org_id + openai-processing-ms: + - '287' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '199977' + x-ratelimit-reset-requests: + - 8.64s + x-ratelimit-reset-tokens: + - 6ms + x-request-id: + - req_58cff97afd0e7c0bba910ccf0b044a6f + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/conftest.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/conftest.py index 51521dbadd..5b80b49d6b 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/conftest.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/conftest.py @@ -33,6 +33,7 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( InMemorySpanExporter, ) +from opentelemetry.sdk.trace.sampling import ALWAYS_OFF @pytest.fixture(scope="function", name="span_exporter") @@ -194,6 +195,29 @@ def instrument_with_content( instrumentor.uninstrument() +@pytest.fixture(scope="function") +def instrument_with_content_unsampled( + span_exporter, event_logger_provider, meter_provider +): + os.environ.update( + {OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "True"} + ) + + tracer_provider = TracerProvider(sampler=ALWAYS_OFF) + tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + + instrumentor = OpenAIInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + event_logger_provider=event_logger_provider, + meter_provider=meter_provider, + ) + + yield instrumentor + os.environ.pop(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, None) + instrumentor.uninstrument() + + class LiteralBlockScalar(str): """Formats the string as a literal block scalar, preserving whitespace and without interpreting escape characters""" diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_async_chat_completions.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_async_chat_completions.py index 65c596796d..468caa232c 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_async_chat_completions.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_async_chat_completions.py @@ -633,6 +633,55 @@ async def test_async_chat_completion_multiple_tools_streaming_no_content( ) +@pytest.mark.vcr() +@pytest.mark.asyncio() +async def test_async_chat_completion_streaming_unsampled( + span_exporter, + log_exporter, + async_openai_client, + instrument_with_content_unsampled, +): + llm_model_value = "gpt-4" + messages_value = [{"role": "user", "content": "Say this is a test"}] + + kwargs = { + "model": llm_model_value, + "messages": messages_value, + "stream": True, + "stream_options": {"include_usage": True}, + } + + response_stream_result = "" + response = await async_openai_client.chat.completions.create(**kwargs) + async for chunk in response: + if chunk.choices: + response_stream_result += chunk.choices[0].delta.content or "" + + spans = span_exporter.get_finished_spans() + assert len(spans) == 0 + + logs = log_exporter.get_finished_logs() + assert len(logs) == 2 + + user_message = {"content": "Say this is a test"} + assert_message_in_logs(logs[0], "gen_ai.user.message", user_message, None) + + choice_event = { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": response_stream_result}, + } + assert_message_in_logs(logs[1], "gen_ai.choice", choice_event, None) + + assert logs[0].log_record.trace_id is not None + assert logs[0].log_record.span_id is not None + assert logs[0].log_record.trace_flags == 0 + + assert logs[0].log_record.trace_id == logs[1].log_record.trace_id + assert logs[0].log_record.span_id == logs[1].log_record.span_id + assert logs[0].log_record.trace_flags == logs[1].log_record.trace_flags + + async def async_chat_completion_multiple_tools_streaming( span_exporter, log_exporter, async_openai_client, expect_content ): @@ -856,9 +905,12 @@ def assert_all_attributes( def assert_log_parent(log, span): - assert log.log_record.trace_id == span.get_span_context().trace_id - assert log.log_record.span_id == span.get_span_context().span_id - assert log.log_record.trace_flags == span.get_span_context().trace_flags + if span: + assert log.log_record.trace_id == span.get_span_context().trace_id + assert log.log_record.span_id == span.get_span_context().span_id + assert ( + log.log_record.trace_flags == span.get_span_context().trace_flags + ) def get_current_weather_tool_definition(): diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_chat_completions.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_chat_completions.py index 9685903603..914d5b5b98 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_chat_completions.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_chat_completions.py @@ -659,6 +659,48 @@ def test_chat_completion_multiple_tools_streaming_no_content( ) +@pytest.mark.vcr() +def test_chat_completion_with_content_span_unsampled( + span_exporter, + log_exporter, + openai_client, + instrument_with_content_unsampled, +): + llm_model_value = "gpt-4o-mini" + messages_value = [{"role": "user", "content": "Say this is a test"}] + + response = openai_client.chat.completions.create( + messages=messages_value, model=llm_model_value, stream=False + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 0 + + logs = log_exporter.get_finished_logs() + assert len(logs) == 2 + + user_message = {"content": messages_value[0]["content"]} + assert_message_in_logs(logs[0], "gen_ai.user.message", user_message, None) + + choice_event = { + "index": 0, + "finish_reason": "stop", + "message": { + "role": "assistant", + "content": response.choices[0].message.content, + }, + } + assert_message_in_logs(logs[1], "gen_ai.choice", choice_event, None) + + assert logs[0].log_record.trace_id is not None + assert logs[0].log_record.span_id is not None + assert logs[0].log_record.trace_flags == 0 + + assert logs[0].log_record.trace_id == logs[1].log_record.trace_id + assert logs[0].log_record.span_id == logs[1].log_record.span_id + assert logs[0].log_record.trace_flags == logs[1].log_record.trace_flags + + def chat_completion_multiple_tools_streaming( span_exporter, log_exporter, openai_client, expect_content ): @@ -878,9 +920,12 @@ def assert_all_attributes( def assert_log_parent(log, span): - assert log.log_record.trace_id == span.get_span_context().trace_id - assert log.log_record.span_id == span.get_span_context().span_id - assert log.log_record.trace_flags == span.get_span_context().trace_flags + if span: + assert log.log_record.trace_id == span.get_span_context().trace_id + assert log.log_record.span_id == span.get_span_context().span_id + assert ( + log.log_record.trace_flags == span.get_span_context().trace_flags + ) def get_current_weather_tool_definition():