From 7e139ac07ac5a7910646fc8019ba6c7e5c45c114 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Mon, 29 Sep 2025 13:51:40 +0000 Subject: [PATCH 01/14] More changes --- .../instrumentation/vertexai/patch.py | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py index c35a9a317a..6a49ee71c6 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py @@ -41,6 +41,9 @@ request_to_events, response_to_events, ) +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) from opentelemetry.trace import SpanKind, Tracer from opentelemetry.util.genai.types import ContentCapturingMode @@ -147,17 +150,11 @@ def _with_new_instrumentation( ): params = _extract_params(*args, **kwargs) api_endpoint: str = instance.api_endpoint # type: ignore[reportUnknownMemberType] - span_attributes = { - **get_genai_request_attributes(False, params), - **get_server_attributes(api_endpoint), - } - - span_name = get_span_name(span_attributes) - + request_attributes = get_genai_request_attributes(True, params) + server_attributes = get_server_attributes(api_endpoint) with self.tracer.start_as_current_span( - name=span_name, + name=f"{GenAI.GenAiOperationNameValues.CHAT.value} {request_attributes.get(GenAI.GEN_AI_REQUEST_MODEL, '')}", kind=SpanKind.CLIENT, - attributes=span_attributes, ) as span: def handle_response( @@ -165,14 +162,23 @@ def handle_response( | prediction_service_v1beta1.GenerateContentResponse | None, ) -> None: + response_attributes = ( + {} + if not response + else get_genai_response_attributes(response) + ) if span.is_recording() and response: # When streaming, this is called multiple times so attributes would be # overwritten. In practice, it looks the API only returns the interesting # attributes on the last streamed response. However, I couldn't find # documentation for this and setting attributes shouldn't be too expensive. span.set_attributes( - get_genai_response_attributes(response) + **response_attributes, + **server_attributes, + **request_attributes, ) + # event = Event(name="gen_ai.client.inference.operation.details") + self.event_logger.emit( create_operation_details_event( api_endpoint=api_endpoint, From 5dd2a2143771a46c21817fca357603cef9faf5c2 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Tue, 30 Sep 2025 18:57:14 +0000 Subject: [PATCH 02/14] Update instrumentation to emit spans in the new format, call the upload hook --- cassettes/test_tool_events_no_content | 83 ++++++++++ cassettes/test_tool_events_no_content.yaml | 83 ++++++++++ .../instrumentation/vertexai/__init__.py | 6 + .../instrumentation/vertexai/patch.py | 106 ++++++++---- .../instrumentation/vertexai/utils.py | 55 +------ ...test_tool_events_with_completion_hook.yaml | 151 ++++++++++++++++++ .../tests/conftest.py | 29 ++++ .../tests/requirements.latest.txt | 1 + .../tests/requirements.oldest.txt | 2 +- .../test_chat_completions_experimental.py | 14 +- .../test_function_calling_experimental.py | 99 +++++++++++- .../test_tool_events_no_content.yaml | 83 ++++++++++ 12 files changed, 625 insertions(+), 87 deletions(-) create mode 100644 cassettes/test_tool_events_no_content create mode 100644 cassettes/test_tool_events_no_content.yaml create mode 100644 instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_tool_events_with_completion_hook.yaml create mode 100644 tests/cassettes/test_tool_events_no_content.yaml diff --git a/cassettes/test_tool_events_no_content b/cassettes/test_tool_events_no_content new file mode 100644 index 0000000000..a44eef4320 --- /dev/null +++ b/cassettes/test_tool_events_no_content @@ -0,0 +1,83 @@ +interactions: +- request: + body: "{\n \"contents\": [\n {\n \"role\": \"user\",\n \"parts\": + [\n {\n \"text\": \"Get weather details in New Delhi and San + Francisco?\"\n }\n ]\n },\n {\n \"role\": \"model\",\n + \ \"parts\": [\n {\n \"functionCall\": {\n \"name\": + \"get_current_weather\",\n \"args\": {\n \"location\": + \"New Delhi\"\n }\n }\n },\n {\n \"functionCall\": + {\n \"name\": \"get_current_weather\",\n \"args\": {\n + \ \"location\": \"San Francisco\"\n }\n }\n + \ }\n ]\n },\n {\n \"role\": \"user\",\n \"parts\": + [\n {\n \"functionResponse\": {\n \"name\": \"get_current_weather\",\n + \ \"response\": {\n \"content\": \"{\\\"temperature\\\": + 35, \\\"unit\\\": \\\"C\\\"}\"\n }\n }\n },\n {\n + \ \"functionResponse\": {\n \"name\": \"get_current_weather\",\n + \ \"response\": {\n \"content\": \"{\\\"temperature\\\": + 25, \\\"unit\\\": \\\"C\\\"}\"\n }\n }\n }\n ]\n + \ }\n ],\n \"tools\": [\n {\n \"functionDeclarations\": [\n {\n + \ \"name\": \"get_current_weather\",\n \"description\": \"Get + the current weather in a given location\",\n \"parameters\": {\n \"type\": + 6,\n \"properties\": {\n \"location\": {\n \"type\": + 1,\n \"description\": \"The location for which to get the weather. + It can be a city name, a city name and state, or a zip code. Examples: 'San + Francisco', 'San Francisco, CA', '95616', etc.\"\n }\n },\n + \ \"propertyOrdering\": [\n \"location\"\n ]\n + \ }\n }\n ]\n }\n ]\n}" + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '1731' + Content-Type: + - application/json + User-Agent: + - python-requests/2.32.3 + x-goog-api-client: + - model-builder/1.79.0+top_google_constructor_method+vertexai.generative_models.GenerativeModel.generate_content + gl-python/3.11.9 grpc/1.68.1 gax/2.23.0 gapic/1.79.0+top_google_constructor_method+vertexai.generative_models.GenerativeModel.generate_content + x-goog-request-params: + - model=projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro + method: POST + uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint + response: + body: + string: !!binary | + H4sIAAAAAAAC/91TwU7rMBC89ytWPRcHECdOVG3EqwQFkXJCCBlnSRccb7A3BYT679ghQODAgeM7 + OTueHc+OndcRwBi9Zz8+hNdYxNJwibE62N2bvAM1hqCrhI0v8LHFIEABagqBXAU+QuSxBN3KGp2Q + 0ULswEQsldoqyJ8bNBI5Z9NIgn3QxkRREH5ANwHLFcUG5gdCYA8chTxstKVfVQtEWIs04TDLStyg + 5QZ9UBVzZVEZrjPquPKSBarcDrnsCW8T1bALbHGn8XwfjalxP2kQLW1Ig14up5erf/lytZhNV/n8 + g1CiaLKJcdUB0IfWbR7JS9OllNbehm4odFZ6V74xao63bbVwd9yrDpT/54z7WbeTvyeXp5f6MzmP + OrBL3bOLfJ6ubHpS3JwuimKxPP4WMdeaOuL3E4acOl5DqUV//g5f+JrLr15lLLel0tRYLXfsa7XZ + U+cxODIpwwL9hgyqY3ToteCMncSYBiel1/ZOSqIDnR/ePhu2o+F6PUpf29EbabJOVcEDAAA= + headers: + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=UTF-8 + Date: + - Tue, 30 Sep 2025 15:26:53 GMT + Server: + - scaffolding on HTTPServer2 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + WWW-Authenticate: + - Bearer realm="https://accounts.google.com/" + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-XSS-Protection: + - '0' + status: + code: 401 + message: Unauthorized +version: 1 diff --git a/cassettes/test_tool_events_no_content.yaml b/cassettes/test_tool_events_no_content.yaml new file mode 100644 index 0000000000..cb70bab7dd --- /dev/null +++ b/cassettes/test_tool_events_no_content.yaml @@ -0,0 +1,83 @@ +interactions: +- request: + body: "{\n \"contents\": [\n {\n \"role\": \"user\",\n \"parts\": + [\n {\n \"text\": \"Get weather details in New Delhi and San + Francisco?\"\n }\n ]\n },\n {\n \"role\": \"model\",\n + \ \"parts\": [\n {\n \"functionCall\": {\n \"name\": + \"get_current_weather\",\n \"args\": {\n \"location\": + \"New Delhi\"\n }\n }\n },\n {\n \"functionCall\": + {\n \"name\": \"get_current_weather\",\n \"args\": {\n + \ \"location\": \"San Francisco\"\n }\n }\n + \ }\n ]\n },\n {\n \"role\": \"user\",\n \"parts\": + [\n {\n \"functionResponse\": {\n \"name\": \"get_current_weather\",\n + \ \"response\": {\n \"content\": \"{\\\"temperature\\\": + 35, \\\"unit\\\": \\\"C\\\"}\"\n }\n }\n },\n {\n + \ \"functionResponse\": {\n \"name\": \"get_current_weather\",\n + \ \"response\": {\n \"content\": \"{\\\"temperature\\\": + 25, \\\"unit\\\": \\\"C\\\"}\"\n }\n }\n }\n ]\n + \ }\n ],\n \"tools\": [\n {\n \"functionDeclarations\": [\n {\n + \ \"name\": \"get_current_weather\",\n \"description\": \"Get + the current weather in a given location\",\n \"parameters\": {\n \"type\": + 6,\n \"properties\": {\n \"location\": {\n \"type\": + 1,\n \"description\": \"The location for which to get the weather. + It can be a city name, a city name and state, or a zip code. Examples: 'San + Francisco', 'San Francisco, CA', '95616', etc.\"\n }\n },\n + \ \"propertyOrdering\": [\n \"location\"\n ]\n + \ }\n }\n ]\n }\n ]\n}" + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '1731' + Content-Type: + - application/json + User-Agent: + - python-requests/2.32.3 + x-goog-api-client: + - model-builder/1.79.0+top_google_constructor_method+vertexai.generative_models.GenerativeModel.generate_content + gl-python/3.11.9 grpc/1.68.1 gax/2.23.0 gapic/1.79.0+top_google_constructor_method+vertexai.generative_models.GenerativeModel.generate_content + x-goog-request-params: + - model=projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro + method: POST + uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint + response: + body: + string: !!binary | + H4sIAAAAAAAC/91TwU7rMBC89ytWPRcHECdOVG3EqwQFkXJCCBlnSRccb7A3BYT679ghQODAgeM7 + OTueHc+OndcRwBi9Zz8+hNdYxNJwibE62N2bvAM1hqCrhI0v8LHFIEABagqBXAU+QuSxBN3KGp2Q + 0ULswEQsldoqyJ8bNBI5Z9NIgn3QxkRREH5ANwHLFcUG5gdCYA8chTxstKVfVQtEWIs04TDLStyg + 5QZ9UBVzZVEZrjPquPKSBarcDrnsCW8T1bALbHGn8XwfjalxP2kQLW1Ig14up5erf/lytZhNV/n8 + g1CiaLKJcdUB0IfWbR7JS9OllNbehm4odFZ6V74xao63bbVwd9yrDpT/54z7WbeTvyeXp5f6MzmP + OrBL3bOLfJ6ubHpS3JwuimKxPP4WMdeaOuL3E4acOl5DqUV//g5f+JrLr15lLLel0tRYLXfsa7XZ + U+cxODIpwwL9hgyqY3ToteCMncSYBiel1/ZOSqIDnR/ePhu2o+F6PUpf29EbabJOVcEDAAA= + headers: + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=UTF-8 + Date: + - Tue, 30 Sep 2025 15:27:34 GMT + Server: + - scaffolding on HTTPServer2 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + WWW-Authenticate: + - Bearer realm="https://accounts.google.com/" + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-XSS-Protection: + - '0' + status: + code: 401 + message: Unauthorized +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/__init__.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/__init__.py index fd241d1316..98e2ed57ed 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/__init__.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/__init__.py @@ -60,6 +60,7 @@ from opentelemetry.instrumentation.vertexai.utils import is_content_enabled from opentelemetry.semconv.schemas import Schemas from opentelemetry.trace import get_tracer +from opentelemetry.util.genai.completion_hook import load_completion_hook def _methods_to_wrap( @@ -109,6 +110,9 @@ def instrumentation_dependencies(self) -> Collection[str]: def _instrument(self, **kwargs: Any): """Enable VertexAI instrumentation.""" + completion_hook = ( + kwargs.get("completion_hook") or load_completion_hook() + ) sem_conv_opt_in_mode = _OpenTelemetrySemanticConventionStability._get_opentelemetry_stability_opt_in_mode( _OpenTelemetryStabilitySignalType.GEN_AI, ) @@ -141,6 +145,7 @@ def _instrument(self, **kwargs: Any): event_logger, is_content_enabled(sem_conv_opt_in_mode), sem_conv_opt_in_mode, + completion_hook, ) elif sem_conv_opt_in_mode == _StabilityMode.GEN_AI_LATEST_EXPERIMENTAL: # Type checker now knows it's the other literal @@ -149,6 +154,7 @@ def _instrument(self, **kwargs: Any): event_logger, is_content_enabled(sem_conv_opt_in_mode), sem_conv_opt_in_mode, + completion_hook, ) else: raise RuntimeError(f"{sem_conv_opt_in_mode} mode not supported") diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py index 6a49ee71c6..639e127920 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py @@ -14,7 +14,9 @@ from __future__ import annotations +import json from contextlib import contextmanager +from dataclasses import asdict from typing import ( TYPE_CHECKING, Any, @@ -27,13 +29,14 @@ overload, ) -from opentelemetry._events import EventLogger +from opentelemetry._events import Event, EventLogger from opentelemetry.instrumentation._semconv import ( _StabilityMode, ) from opentelemetry.instrumentation.vertexai.utils import ( GenerateContentParams, - create_operation_details_event, + convert_content_to_message, + convert_response_to_output_messages, get_genai_request_attributes, get_genai_response_attributes, get_server_attributes, @@ -45,7 +48,11 @@ gen_ai_attributes as GenAI, ) from opentelemetry.trace import SpanKind, Tracer -from opentelemetry.util.genai.types import ContentCapturingMode +from opentelemetry.util.genai.completion_hook import CompletionHook +from opentelemetry.util.genai.types import ( + ContentCapturingMode, + Text, +) if TYPE_CHECKING: from google.cloud.aiplatform_v1.services.prediction_service import client @@ -113,6 +120,7 @@ def __init__( sem_conv_opt_in_mode: Literal[ _StabilityMode.GEN_AI_LATEST_EXPERIMENTAL ], + completion_hook: CompletionHook, ) -> None: ... @overload @@ -122,6 +130,7 @@ def __init__( event_logger: EventLogger, capture_content: bool, sem_conv_opt_in_mode: Literal[_StabilityMode.DEFAULT], + completion_hook: CompletionHook, ) -> None: ... def __init__( @@ -133,11 +142,13 @@ def __init__( Literal[_StabilityMode.DEFAULT], Literal[_StabilityMode.GEN_AI_LATEST_EXPERIMENTAL], ], + completion_hook: CompletionHook, ) -> None: self.tracer = tracer self.event_logger = event_logger self.capture_content = capture_content self.sem_conv_opt_in_mode = sem_conv_opt_in_mode + self.completion_hook = completion_hook @contextmanager def _with_new_instrumentation( @@ -149,11 +160,9 @@ def _with_new_instrumentation( kwargs: Any, ): params = _extract_params(*args, **kwargs) - api_endpoint: str = instance.api_endpoint # type: ignore[reportUnknownMemberType] request_attributes = get_genai_request_attributes(True, params) - server_attributes = get_server_attributes(api_endpoint) with self.tracer.start_as_current_span( - name=f"{GenAI.GenAiOperationNameValues.CHAT.value} {request_attributes.get(GenAI.GEN_AI_REQUEST_MODEL, '')}", + name=f"{GenAI.GenAiOperationNameValues.CHAT.value} {request_attributes.get(GenAI.GEN_AI_REQUEST_MODEL, '')}".strip(), kind=SpanKind.CLIENT, ) as span: @@ -162,30 +171,69 @@ def handle_response( | prediction_service_v1beta1.GenerateContentResponse | None, ) -> None: - response_attributes = ( - {} - if not response - else get_genai_response_attributes(response) + attributes = ( + get_server_attributes(instance.api_endpoint) # type: ignore[reportUnknownMemberType] + | request_attributes + | get_genai_response_attributes(response) ) - if span.is_recording() and response: - # When streaming, this is called multiple times so attributes would be - # overwritten. In practice, it looks the API only returns the interesting - # attributes on the last streamed response. However, I couldn't find - # documentation for this and setting attributes shouldn't be too expensive. - span.set_attributes( - **response_attributes, - **server_attributes, - **request_attributes, - ) - # event = Event(name="gen_ai.client.inference.operation.details") - - self.event_logger.emit( - create_operation_details_event( - api_endpoint=api_endpoint, - params=params, - capture_content=capture_content, - response=response, - ) + system_instructions, inputs, outputs = [], [], [] + if params.system_instruction: + system_instructions = [ + Text( + content="\n".join( + part.text + for part in params.system_instruction.parts + ) + ) + ] + if params.contents: + inputs = [ + convert_content_to_message(content) + for content in params.contents + ] + if response: + outputs = convert_response_to_output_messages(response) + content = { + k: [asdict(x) for x in v] + for k, v in [ + ( + GenAI.GEN_AI_SYSTEM_INSTRUCTIONS, + system_instructions, + ), + (GenAI.GEN_AI_INPUT_MESSAGES, inputs), + (GenAI.GEN_AI_OUTPUT_MESSAGES, outputs), + ] + if v + } + if span.is_recording(): + span.set_attributes(attributes) + if capture_content in frozenset( + [ + ContentCapturingMode.SPAN_AND_EVENT, + ContentCapturingMode.SPAN_ONLY, + ] + ): + span.set_attributes( + {k: json.dumps(v) for k, v in content.items()} + ) + event = Event( + name="gen_ai.client.inference.operation.details", + ) + event.attributes = attributes + if capture_content in frozenset( + [ + ContentCapturingMode.SPAN_AND_EVENT, + ContentCapturingMode.EVENT_ONLY, + ] + ): + event.attributes |= content + self.event_logger.emit(event) + self.completion_hook.on_completion( + inputs=inputs, + outputs=outputs, + system_instruction=system_instructions, + span=span, + log_record=event, ) yield handle_response diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py index c9c370794e..e179bdd28e 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py @@ -17,7 +17,7 @@ from __future__ import annotations import re -from dataclasses import asdict, dataclass +from dataclasses import dataclass from os import environ from typing import ( TYPE_CHECKING, @@ -192,8 +192,11 @@ def get_genai_request_attributes( # pylint: disable=too-many-branches def get_genai_response_attributes( response: prediction_service.GenerateContentResponse - | prediction_service_v1beta1.GenerateContentResponse, + | prediction_service_v1beta1.GenerateContentResponse + | None, ) -> dict[str, AttributeValue]: + if not response: + return {} finish_reasons: list[str] = [ _map_finish_reason(candidate.finish_reason) for candidate in response.candidates @@ -307,55 +310,13 @@ def request_to_events( yield user_event(role=content.role, content=request_content) -def create_operation_details_event( - *, - api_endpoint: str, - response: prediction_service.GenerateContentResponse - | prediction_service_v1beta1.GenerateContentResponse - | None, - params: GenerateContentParams, - capture_content: ContentCapturingMode, -) -> Event: - event = Event(name="gen_ai.client.inference.operation.details") - attributes: dict[str, AnyValue] = { - **get_genai_request_attributes(True, params), - **get_server_attributes(api_endpoint), - **(get_genai_response_attributes(response) if response else {}), - } - event.attributes = attributes - if capture_content in { - ContentCapturingMode.NO_CONTENT, - ContentCapturingMode.SPAN_ONLY, - }: - return event - if params.system_instruction: - attributes[GenAIAttributes.GEN_AI_SYSTEM_INSTRUCTIONS] = [ - { - "type": "text", - "content": "\n".join( - part.text for part in params.system_instruction.parts - ), - } - ] - if params.contents: - attributes[GenAIAttributes.GEN_AI_INPUT_MESSAGES] = [ - asdict(_convert_content_to_message(content)) - for content in params.contents - ] - if response and response.candidates: - attributes[GenAIAttributes.GEN_AI_OUTPUT_MESSAGES] = [ - asdict(x) for x in _convert_response_to_output_messages(response) - ] - return event - - -def _convert_response_to_output_messages( +def convert_response_to_output_messages( response: prediction_service.GenerateContentResponse | prediction_service_v1beta1.GenerateContentResponse, ) -> list[OutputMessage]: output_messages: list[OutputMessage] = [] for candidate in response.candidates: - message = _convert_content_to_message(candidate.content) + message = convert_content_to_message(candidate.content) output_messages.append( OutputMessage( finish_reason=_map_finish_reason(candidate.finish_reason), @@ -366,7 +327,7 @@ def _convert_response_to_output_messages( return output_messages -def _convert_content_to_message( +def convert_content_to_message( content: content.Content | content_v1beta1.Content, ) -> InputMessage: parts: MessagePart = [] diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_tool_events_with_completion_hook.yaml b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_tool_events_with_completion_hook.yaml new file mode 100644 index 0000000000..20a8db0761 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_tool_events_with_completion_hook.yaml @@ -0,0 +1,151 @@ +interactions: +- request: + body: |- + { + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Get weather details in New Delhi and San Francisco?" + } + ] + }, + { + "role": "model", + "parts": [ + { + "functionCall": { + "name": "get_current_weather", + "args": { + "location": "New Delhi" + } + } + }, + { + "functionCall": { + "name": "get_current_weather", + "args": { + "location": "San Francisco" + } + } + } + ] + }, + { + "role": "user", + "parts": [ + { + "functionResponse": { + "name": "get_current_weather", + "response": { + "content": "{\"temperature\": 35, \"unit\": \"C\"}" + } + } + }, + { + "functionResponse": { + "name": "get_current_weather", + "response": { + "content": "{\"temperature\": 25, \"unit\": \"C\"}" + } + } + } + ] + } + ], + "tools": [ + { + "functionDeclarations": [ + { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": 6, + "properties": { + "location": { + "type": 1, + "description": "The location for which to get the weather. It can be a city name, a city name and state, or a zip code. Examples: 'San Francisco', 'San Francisco, CA', '95616', etc." + } + }, + "propertyOrdering": [ + "location" + ] + } + } + ] + } + ] + } + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '1731' + Content-Type: + - application/json + User-Agent: + - python-requests/2.32.3 + method: POST + uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint + response: + body: + string: |- + { + "candidates": [ + { + "content": { + "role": "model", + "parts": [ + { + "text": "The weather in New Delhi is 35\u00b0C and in San Francisco is 25\u00b0C.", + "thoughtSignature": "CqcOAcu98PDUiUq32HLxu6y5JNxvlcEjIaedPcBi5V86Hbf3vRgAXC4k0aMma0v1gotZVHrinF9edI9bEAdQdFR+2xOsaV1ntNeO4o35ymNNpm1rEv2p047eWxSABiXJ3VANecxNqQuVgxZyhClOn2BNmR/xGb43REabpcMzboVGVT6iKSJ/g3sCIt4bddY1IQ5zTdSV7lvCyZLuu9736VCJjULzAslhxSb9/xBlQu/pvcak6CmFFLZuDamqeJ3RvhalpklF6qz/Eq9nhhpdRPTERmGU7mNe4fiSpql6JxelO56ksKKGzSG+USa9vxcCQRRgAoaLlMHRegwPjfoA4hHgbpLBIUk9BNXrzmW5APOpJO3rqgIvKQ/WsZzIBI3l+slI51WHgVLr4qNvDUe8VKLhbApVj3L+rb5dQ/u6U9V2oSVA5pQFt6Lqubyjg8yCl0q+0dsvIYUqJSaDRONMfS5hwVu+RTU/SOEXLwTNEiDDuj3wO8xQuyitDAHqya797sOH2ThlSoT/c7V3Du00TBCk0eq8XaxyTobPMCaKNbrqjRTfMn5fortZlPwyAJqoTZsaSeaRufCFpgBMy34/QcLiCdVwKN6rLgBpDhtzThnyBzavDP2ltu8sxWEisJXEOel0q8LvBqTqlFY9dWrRScnB/TvitGnwqlV0PVpWU60wH/2Agexw0+aFlMRYxiFRjuHPK9R2UfLbEzVpDC8M+f16QRZOnWQc8L+cxEW/xHT6hCUxcTmSrpldAs2Ss8A21NcPTlr0lNviZxB+4xDq1dxB/VMS/qOnGVbr11nEOOVj+bWJyP/LLTjdUigGs+ISAzmEr2zkUirLdlmz/Mz1YXk9CMIus8WlPVzhQZuXY8dhSKxks0w8bJ5TjJelAwPGFgz6iWSz44IeaH22Fxdvub28/mdmvy3OerI3vfwlePUC9L8n2cg7gZjIE59PXi2x1EUTJfvcBrVTO5qqPuRpJZ9/d3ryEtS9tHsMnzfcEiS2lxNb+EqKaHk74hqvt6vEA+eQaYtMqG5TO79dpGDt0UD1cJEwngfMiL85+5qWHW4g2hSSJYgg55GtdZbxdGs960p8G02FbqUOSVcDGcIBDCkvFSGHJqMATvYQzChiwP2AnDVLN0aWd43UEl3rHESdOGLOqrxetcWmVsDacowhiy6jleS/Xml7d8mqqs2483XCOVMMYPcFh9budRed7PIf7eAYhWzGwt123JLbxqwBOBf1MHUJl2OQBy62HCJLYMlRhr1s2KVRK9Een5cgUa3H07r/s46YS2A0KOP/B0Ub0MflvGBAgo8LABj5cY2Ao1hTKcnBjgJPTNS0awETbtzU3Y74SPMsJ1Ipu2o76TFS3YFq1Fbit1VsceuYxzcqU04NFBtXAU7YojLv9LBOZXk6SukOWI/2cNUOHlzlWsSi34heVFGH4AcDbhc0/QZHhjSTE9FkBY9feTtSnTR16Q1bg3K9+7rxArD51Wj8q1ZNhVtZrzpuWGqoneI0WqJ25U5f1jSzQFW2HyzmIs/B5KiFEk4Do0EiaVcCQ5/1UGReYbgtSKa8PAvUGh2Lqiev1VMl6i4cDs7s3U9swvZdBdK+SauYC7UqEVov1G8UR8Bth2YncqUPSv7W2X7ppA6PwDXSCqCSgp3wnfEW8tQIa6AQW2s4Udp0lbYWr3Vu2DSK3iu8mVa3ybvHs5Go4SR9zJ8xElMn4adprz8jokYpoXK94DVeaUetKqTw6X3FzRb+K+e/W0MHZVEQtAjk2wwcNqAfs8JdXi2Mh/l+0PFp6Hhz0rI1rzZ3B08lVR+QU+0XpN5rerUmq44oE6t+dP6Cv8yl5kVBCjvtHZ9DLUGeSEofOR3JXs34rkWeOLfQ+vRZkYoiIRKkqFhf/RnRoU18vmfsbvK9C3XcO3wmfo58Iw2uYd9L/p6VdB27pPUiBJbTAzXbEA3CWUbedMg2vDZiucS4iBkWlmN91CofwgmNfiImdcauz5ug3TC5qRWUIqX/nNMSscPjpbhp3nmja727fyH1p4ytpXQbRX0do28KpY5XQ9yF9gAWxG2zHxtBGiZVG4iXRlpZlfmFUc4kQLUTrInYUoagh9zF6Cxwrz7smRqgOkWKOYiOTiR9ofTkvpmQpdGpDOsxjcPRd3V+jjkfZwT5yzsM7IOAzzhZKKvPWgfJfzaAI43W+dUWiMWzM5KX7Wliu5J3uEXtldo0lW2WpOiuFK5GIPBh5482IbSAtA14YEplOYVCrCI9UG356TImFYyCLuNKAxbzA3U9TSJCk0tUbssWKq8aIaoEbtvE43P1G8x5teVFR5Wu1nZypgKgYFSS5tRL+niDV82yaCOTIGjdA6yFHq8gRlv9xkmZjyPIB4LfAhAIEz9X0+hNvZBfTofi1UXRh/54GIqr1YIXmJVYKb/srAU5Qne5lQdvFauFIQbnlESVUeb1RA==" + } + ] + }, + "finishReason": 1, + "avgLogprobs": -1.18875130740079 + } + ], + "usageMetadata": { + "promptTokenCount": 128, + "candidatesTokenCount": 22, + "totalTokenCount": 575, + "trafficType": 1, + "promptTokensDetails": [ + { + "modality": 1, + "tokenCount": 128 + } + ], + "candidatesTokensDetails": [ + { + "modality": 1, + "tokenCount": 22 + } + ], + "thoughtsTokenCount": 425 + }, + "modelVersion": "gemini-2.5-pro", + "createTime": "2025-08-19T14:55:39.882212Z", + "responseId": "a5CkaKTsNa3hgLUP59n1oQo" + } + headers: + Content-Type: + - application/json; charset=UTF-8 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + content-length: + - '3277' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/conftest.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/conftest.py index c642eebcf4..1280a88339 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/conftest.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/conftest.py @@ -190,6 +190,35 @@ def instrument_with_experimental_semconvs( instrumentor.uninstrument() +@pytest.fixture(scope="function") +def instrument_with_upload_hook( + tracer_provider, event_logger_provider, meter_provider, request +): + # Reset global state.. + _OpenTelemetrySemanticConventionStability._initialized = False + os.environ.update( + { + OTEL_SEMCONV_STABILITY_OPT_IN: "gen_ai_latest_experimental", + "OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK": "fsspec_upload", + "OTEL_INSTRUMENTATION_GENAI_UPLOAD_BASE_PATH": "memory://", + OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "SPAN_AND_EVENT", + } + ) + instrumentor = VertexAIInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + event_logger_provider=event_logger_provider, + meter_provider=meter_provider, + ) + + yield instrumentor + os.environ.pop(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, None) + os.environ.pop("OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK", None) + os.environ.pop("OTEL_INSTRUMENTATION_GENAI_UPLOAD_BASE_PATH", None) + if instrumentor.is_instrumented_by_opentelemetry: + instrumentor.uninstrument() + + @pytest.fixture(scope="function") def instrument_with_content( tracer_provider, event_logger_provider, meter_provider, request diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/requirements.latest.txt b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/requirements.latest.txt index 1497f0fc3a..6098baf83c 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/requirements.latest.txt +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/requirements.latest.txt @@ -78,6 +78,7 @@ PyYAML==6.0.2 requests==2.32.3 rsa==4.9 shapely==2.0.6 +fsspec==2025.9.0 six==1.17.0 tomli==2.2.1 typing_extensions==4.12.2 diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/requirements.oldest.txt b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/requirements.oldest.txt index b38d74c562..fd9e803fb9 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/requirements.oldest.txt +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/requirements.oldest.txt @@ -69,7 +69,7 @@ opentelemetry-api==1.37 opentelemetry-sdk==1.37 opentelemetry-semantic-conventions==0.58b0 opentelemetry-instrumentation==0.58b0 - +fsspec==2025.9.0 -e instrumentation-genai/opentelemetry-instrumentation-vertexai[instruments] -e util/opentelemetry-util-genai diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py index f1bc28d5f6..84aa9aee8b 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py @@ -52,11 +52,12 @@ def test_generate_content( "gen_ai.request.model": "gemini-2.5-pro", "gen_ai.response.finish_reasons": ("stop",), "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.system": "vertex_ai", "gen_ai.usage.input_tokens": 5, "gen_ai.usage.output_tokens": 5, "server.address": "us-central1-aiplatform.googleapis.com", "server.port": 443, + "gen_ai.input.messages": '[{"role": "user", "parts": [{"content": "Say this is a test", "type": "text"}]}]', + "gen_ai.output.messages": '[{"role": "model", "parts": [{"content": "This is a test.", "type": "text"}], "finish_reason": "stop"}]', } logs = log_exporter.get_finished_logs() @@ -108,10 +109,11 @@ def test_generate_content_without_events( assert spans[0].name == "chat gemini-2.5-pro" assert dict(spans[0].attributes) == { "gen_ai.operation.name": "chat", + "gen_ai.output.messages": '[{"role": "model", "parts": [{"content": "This is a test.", "type": "text"}], "finish_reason": "stop"}]', + "gen_ai.input.messages": '[{"role": "user", "parts": [{"content": "Say this is a test", "type": "text"}]}]', "gen_ai.request.model": "gemini-2.5-pro", "gen_ai.response.finish_reasons": ("stop",), "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.system": "vertex_ai", "gen_ai.usage.input_tokens": 5, "gen_ai.usage.output_tokens": 5, "server.address": "us-central1-aiplatform.googleapis.com", @@ -173,9 +175,9 @@ def test_generate_content_empty_model( assert dict(spans[0].attributes) == { "gen_ai.operation.name": "chat", "gen_ai.request.model": "", - "gen_ai.system": "vertex_ai", "server.address": "us-central1-aiplatform.googleapis.com", "server.port": 443, + "gen_ai.input.messages": '[{"role": "user", "parts": [{"content": "Say this is a test", "type": "text"}]}]', } assert_span_error(spans[0]) @@ -206,9 +208,9 @@ def test_generate_content_missing_model( assert dict(spans[0].attributes) == { "gen_ai.operation.name": "chat", "gen_ai.request.model": "gemini-does-not-exist", - "gen_ai.system": "vertex_ai", "server.address": "us-central1-aiplatform.googleapis.com", "server.port": 443, + "gen_ai.input.messages": '[{"role": "user", "parts": [{"content": "Say this is a test", "type": "text"}]}]', } assert_span_error(spans[0]) @@ -241,9 +243,9 @@ def test_generate_content_invalid_temperature( "gen_ai.operation.name": "chat", "gen_ai.request.model": "gemini-2.5-pro", "gen_ai.request.temperature": 1000.0, - "gen_ai.system": "vertex_ai", "server.address": "us-central1-aiplatform.googleapis.com", "server.port": 443, + "gen_ai.input.messages": '[{"role": "user", "parts": [{"content": "Say this is a test", "type": "text"}]}]', } assert_span_error(spans[0]) @@ -325,7 +327,7 @@ def test_generate_content_extra_params( "gen_ai.request.top_p": 0.949999988079071, "gen_ai.response.finish_reasons": ("length",), "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.system": "vertex_ai", + "gen_ai.request.seed": 12345, "gen_ai.usage.input_tokens": 5, "gen_ai.usage.output_tokens": 0, "server.address": "us-central1-aiplatform.googleapis.com", diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py index a7b61a09c7..597412d3ee 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py @@ -1,3 +1,7 @@ +import json +from typing import Any + +import fsspec import pytest from opentelemetry.instrumentation.vertexai import VertexAIInstrumentor @@ -13,7 +17,6 @@ ) -@pytest.mark.vcr() def test_function_call_choice( span_exporter: InMemorySpanExporter, log_exporter: InMemoryLogExporter, @@ -31,10 +34,11 @@ def test_function_call_choice( "gen_ai.request.model": "gemini-2.5-pro", "gen_ai.response.finish_reasons": ("stop",), "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.system": "vertex_ai", "gen_ai.usage.input_tokens": 74, "gen_ai.usage.output_tokens": 16, "server.address": "us-central1-aiplatform.googleapis.com", + "gen_ai.input.messages": '[{"role": "user", "parts": [{"content": "Get weather details in New Delhi and San Francisco?", "type": "text"}]}]', + "gen_ai.output.messages": '[{"role": "model", "parts": [{"arguments": {"location": "New Delhi"}, "name": "get_current_weather", "id": "get_current_weather_0", "type": "tool_call"}, {"arguments": {"location": "San Francisco"}, "name": "get_current_weather", "id": "get_current_weather_1", "type": "tool_call"}], "finish_reason": "stop"}]', "server.port": 443, } @@ -127,11 +131,12 @@ def test_tool_events( "gen_ai.request.model": "gemini-2.5-pro", "gen_ai.response.finish_reasons": ("stop",), "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.system": "vertex_ai", "gen_ai.usage.input_tokens": 128, "gen_ai.usage.output_tokens": 26, "server.address": "us-central1-aiplatform.googleapis.com", "server.port": 443, + "gen_ai.input.messages": '[{"role": "user", "parts": [{"content": "Get weather details in New Delhi and San Francisco?", "type": "text"}]}, {"role": "model", "parts": [{"arguments": {"location": "New Delhi"}, "name": "get_current_weather", "id": "get_current_weather_0", "type": "tool_call"}, {"arguments": {"location": "San Francisco"}, "name": "get_current_weather", "id": "get_current_weather_1", "type": "tool_call"}]}, {"role": "user", "parts": [{"response": {"content": "{\\"temperature\\": 35, \\"unit\\": \\"C\\"}"}, "id": "get_current_weather_0", "type": "tool_call_response"}, {"response": {"content": "{\\"temperature\\": 25, \\"unit\\": \\"C\\"}"}, "id": "get_current_weather_1", "type": "tool_call_response"}]}]', + "gen_ai.output.messages": '[{"role": "model", "parts": [{"content": "The current temperature in New Delhi is 35\\u00b0C, and in San Francisco, it is 25\\u00b0C.", "type": "text"}], "finish_reason": "stop"}]', } logs = log_exporter.get_finished_logs() assert len(logs) == 1 @@ -225,7 +230,6 @@ def test_tool_events_no_content( "gen_ai.request.model": "gemini-2.5-pro", "gen_ai.response.finish_reasons": ("stop",), "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.system": "vertex_ai", "gen_ai.usage.input_tokens": 128, "gen_ai.usage.output_tokens": 22, "server.address": "us-central1-aiplatform.googleapis.com", @@ -244,3 +248,90 @@ def test_tool_events_no_content( "gen_ai.usage.input_tokens": 128, "gen_ai.usage.output_tokens": 22, } + + +def assert_fsspec_equal(path: str, value: Any) -> None: + __tracebackhide__ = True + with fsspec.open(path, "r") as file: + assert json.load(file) == value + + +@pytest.mark.vcr() +def test_tool_events_with_completion_hook( + span_exporter: InMemorySpanExporter, + log_exporter: InMemoryLogExporter, + instrument_with_upload_hook: VertexAIInstrumentor, + generate_content: callable, +): + ask_about_weather_function_response(generate_content) + + # Emits span + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + logs = log_exporter.get_finished_logs() + assert len(logs) == 1 + assert_fsspec_equal( + spans[0].attributes["gen_ai.output.messages_ref"], + [ + { + "role": "model", + "parts": [ + { + "content": "The weather in New Delhi is 35°C and in San Francisco is 25°C.", + "type": "text", + } + ], + "finish_reason": "stop", + } + ], + ) + assert_fsspec_equal( + spans[0].attributes["gen_ai.input.messages_ref"], + [ + { + "parts": [ + { + "content": "Get weather details in New Delhi and San Francisco?", + "type": "text", + } + ], + "role": "user", + }, + { + "parts": [ + { + "arguments": {"location": "New Delhi"}, + "id": "get_current_weather_0", + "name": "get_current_weather", + "type": "tool_call", + }, + { + "arguments": {"location": "San Francisco"}, + "id": "get_current_weather_1", + "name": "get_current_weather", + "type": "tool_call", + }, + ], + "role": "model", + }, + { + "parts": [ + { + "id": "get_current_weather_0", + "response": { + "content": '{"temperature": 35, "unit": "C"}' + }, + "type": "tool_call_response", + }, + { + "id": "get_current_weather_1", + "response": { + "content": '{"temperature": 25, "unit": "C"}' + }, + "type": "tool_call_response", + }, + ], + "role": "user", + }, + ], + ) diff --git a/tests/cassettes/test_tool_events_no_content.yaml b/tests/cassettes/test_tool_events_no_content.yaml new file mode 100644 index 0000000000..a39a0dbf40 --- /dev/null +++ b/tests/cassettes/test_tool_events_no_content.yaml @@ -0,0 +1,83 @@ +interactions: +- request: + body: "{\n \"contents\": [\n {\n \"role\": \"user\",\n \"parts\": + [\n {\n \"text\": \"Get weather details in New Delhi and San + Francisco?\"\n }\n ]\n },\n {\n \"role\": \"model\",\n + \ \"parts\": [\n {\n \"functionCall\": {\n \"name\": + \"get_current_weather\",\n \"args\": {\n \"location\": + \"New Delhi\"\n }\n }\n },\n {\n \"functionCall\": + {\n \"name\": \"get_current_weather\",\n \"args\": {\n + \ \"location\": \"San Francisco\"\n }\n }\n + \ }\n ]\n },\n {\n \"role\": \"user\",\n \"parts\": + [\n {\n \"functionResponse\": {\n \"name\": \"get_current_weather\",\n + \ \"response\": {\n \"content\": \"{\\\"temperature\\\": + 35, \\\"unit\\\": \\\"C\\\"}\"\n }\n }\n },\n {\n + \ \"functionResponse\": {\n \"name\": \"get_current_weather\",\n + \ \"response\": {\n \"content\": \"{\\\"temperature\\\": + 25, \\\"unit\\\": \\\"C\\\"}\"\n }\n }\n }\n ]\n + \ }\n ],\n \"tools\": [\n {\n \"functionDeclarations\": [\n {\n + \ \"name\": \"get_current_weather\",\n \"description\": \"Get + the current weather in a given location\",\n \"parameters\": {\n \"type\": + 6,\n \"properties\": {\n \"location\": {\n \"type\": + 1,\n \"description\": \"The location for which to get the weather. + It can be a city name, a city name and state, or a zip code. Examples: 'San + Francisco', 'San Francisco, CA', '95616', etc.\"\n }\n },\n + \ \"propertyOrdering\": [\n \"location\"\n ]\n + \ }\n }\n ]\n }\n ]\n}" + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '1731' + Content-Type: + - application/json + User-Agent: + - python-requests/2.32.3 + x-goog-api-client: + - model-builder/1.79.0+top_google_constructor_method+vertexai.generative_models.GenerativeModel.generate_content + gl-python/3.11.9 grpc/1.68.1 gax/2.23.0 gapic/1.79.0+top_google_constructor_method+vertexai.generative_models.GenerativeModel.generate_content + x-goog-request-params: + - model=projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro + method: POST + uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint + response: + body: + string: !!binary | + H4sIAAAAAAAC/91TwU7rMBC89ytWPRcHECdOVG3EqwQFkXJCCBlnSRccb7A3BYT679ghQODAgeM7 + OTueHc+OndcRwBi9Zz8+hNdYxNJwibE62N2bvAM1hqCrhI0v8LHFIEABagqBXAU+QuSxBN3KGp2Q + 0ULswEQsldoqyJ8bNBI5Z9NIgn3QxkRREH5ANwHLFcUG5gdCYA8chTxstKVfVQtEWIs04TDLStyg + 5QZ9UBVzZVEZrjPquPKSBarcDrnsCW8T1bALbHGn8XwfjalxP2kQLW1Ig14up5erf/lytZhNV/n8 + g1CiaLKJcdUB0IfWbR7JS9OllNbehm4odFZ6V74xao63bbVwd9yrDpT/54z7WbeTvyeXp5f6MzmP + OrBL3bOLfJ6ubHpS3JwuimKxPP4WMdeaOuL3E4acOl5DqUV//g5f+JrLr15lLLel0tRYLXfsa7XZ + U+cxODIpwwL9hgyqY3ToteCMncSYBiel1/ZOSqIDnR/ePhu2o+F6PUpf29EbabJOVcEDAAA= + headers: + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=UTF-8 + Date: + - Tue, 30 Sep 2025 15:31:01 GMT + Server: + - scaffolding on HTTPServer2 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + WWW-Authenticate: + - Bearer realm="https://accounts.google.com/" + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-XSS-Protection: + - '0' + status: + code: 401 + message: Unauthorized +version: 1 From a65ce738af0a4a6d33fce693a8519c3e050e8109 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Tue, 30 Sep 2025 19:31:54 +0000 Subject: [PATCH 03/14] Update example and changelog --- .../CHANGELOG.md | 6 +++--- .../examples/manual/.env | 14 ++++++++++++++ .../examples/zero-code/.env | 14 ++++++++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md b/instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md index a46355ba59..a839324a0d 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md @@ -7,9 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased -- Start making changes to implement the big semantic convention changes made in https://github.com/open-telemetry/semantic-conventions/pull/2179. -Now only a single event (`gen_ai.client.inference.operation.details`) is used to capture Chat History. These changes will be opt-in, -users will need to set the environment variable OTEL_SEMCONV_STABILITY_OPT_IN to `gen_ai_latest_experimental` to see them ([#3386](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3386)). +- Update instrumentation to use the latest semantic convention changes made in https://github.com/open-telemetry/semantic-conventions/pull/2179. +Now only a single event and span (`gen_ai.client.inference.operation.details`) are used to capture prompt and response content. These changes are opt-in, +users will need to set the environment variable OTEL_SEMCONV_STABILITY_OPT_IN to `gen_ai_latest_experimental` to see them ([#3799](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3799)) and ([#3709](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3709)). - Implement uninstrument for `opentelemetry-instrumentation-vertexai` ([#3328](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3328)) - VertexAI support for async calling diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/manual/.env b/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/manual/.env index 9ca033983f..4fc56894f3 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/manual/.env +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/manual/.env @@ -6,3 +6,17 @@ OTEL_SERVICE_NAME=opentelemetry-python-vertexai # Change to 'false' to hide prompt and completion content OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true + +# Alternatively set this env var to enable the latest semantic conventions: +OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental + +# When using the latest experimental flag this env var controls which telemetry signals will have prompt and response content included in them. +# Choices are NO_CONTENT, SPAN_ONLY, EVENT_ONLY, SPAN_AND_EVENT. +OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=SPAN_AND_EVENT + +# Optional hook that will upload prompt and response content to some external destination. +# For example fsspec. +OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK = "fsspec" + +# Required if using a completion hook. The path to upload content to for example gs://my_bucket. +OTEL_INSTRUMENTATION_GENAI_UPLOAD_BASE_PATH = "gs://my_bucket" \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/zero-code/.env b/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/zero-code/.env index eefbd59500..a78d0ca6cc 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/zero-code/.env +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/zero-code/.env @@ -12,3 +12,17 @@ OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true # Change to 'false' to hide prompt and completion content OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true + +# Alternatively set this env var to enable the latest semantic conventions: +OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental + +# When using the latest experimental flag this env var controls which telemetry signals will have prompt and response content included in them. +# Choices are NO_CONTENT, SPAN_ONLY, EVENT_ONLY, SPAN_AND_EVENT. +OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=SPAN_AND_EVENT + +# Optional hook that will upload prompt and response content to some external destination. +# For example fsspec. +OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK = "fsspec" + +# Required if using a completion hook. The path to upload content to for example gs://my_bucket. +OTEL_INSTRUMENTATION_GENAI_UPLOAD_BASE_PATH = "gs://my_bucket" \ No newline at end of file From c31ad9fafddceed7fa303c31851c675b418772c2 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Tue, 30 Sep 2025 19:38:13 +0000 Subject: [PATCH 04/14] Fix linter --- .../tests/test_function_calling_experimental.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py index 597412d3ee..8b6a0855e4 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py @@ -251,7 +251,8 @@ def test_tool_events_no_content( def assert_fsspec_equal(path: str, value: Any) -> None: - __tracebackhide__ = True + # Hide this function and its calls from traceback. + __tracebackhide__ = True # pylint: disable=unused-variable with fsspec.open(path, "r") as file: assert json.load(file) == value From 7f03e05c2c2f96dbf5ff924381de05a77caa386d Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Thu, 2 Oct 2025 18:05:39 +0000 Subject: [PATCH 05/14] Respond to comments --- cassettes/test_tool_events_no_content | 83 ------------------- cassettes/test_tool_events_no_content.yaml | 83 ------------------- .../examples/manual/.env | 2 +- .../examples/zero-code/.env | 2 +- .../instrumentation/vertexai/patch.py | 25 +++--- .../tests/conftest.py | 2 +- 6 files changed, 15 insertions(+), 182 deletions(-) delete mode 100644 cassettes/test_tool_events_no_content delete mode 100644 cassettes/test_tool_events_no_content.yaml diff --git a/cassettes/test_tool_events_no_content b/cassettes/test_tool_events_no_content deleted file mode 100644 index a44eef4320..0000000000 --- a/cassettes/test_tool_events_no_content +++ /dev/null @@ -1,83 +0,0 @@ -interactions: -- request: - body: "{\n \"contents\": [\n {\n \"role\": \"user\",\n \"parts\": - [\n {\n \"text\": \"Get weather details in New Delhi and San - Francisco?\"\n }\n ]\n },\n {\n \"role\": \"model\",\n - \ \"parts\": [\n {\n \"functionCall\": {\n \"name\": - \"get_current_weather\",\n \"args\": {\n \"location\": - \"New Delhi\"\n }\n }\n },\n {\n \"functionCall\": - {\n \"name\": \"get_current_weather\",\n \"args\": {\n - \ \"location\": \"San Francisco\"\n }\n }\n - \ }\n ]\n },\n {\n \"role\": \"user\",\n \"parts\": - [\n {\n \"functionResponse\": {\n \"name\": \"get_current_weather\",\n - \ \"response\": {\n \"content\": \"{\\\"temperature\\\": - 35, \\\"unit\\\": \\\"C\\\"}\"\n }\n }\n },\n {\n - \ \"functionResponse\": {\n \"name\": \"get_current_weather\",\n - \ \"response\": {\n \"content\": \"{\\\"temperature\\\": - 25, \\\"unit\\\": \\\"C\\\"}\"\n }\n }\n }\n ]\n - \ }\n ],\n \"tools\": [\n {\n \"functionDeclarations\": [\n {\n - \ \"name\": \"get_current_weather\",\n \"description\": \"Get - the current weather in a given location\",\n \"parameters\": {\n \"type\": - 6,\n \"properties\": {\n \"location\": {\n \"type\": - 1,\n \"description\": \"The location for which to get the weather. - It can be a city name, a city name and state, or a zip code. Examples: 'San - Francisco', 'San Francisco, CA', '95616', etc.\"\n }\n },\n - \ \"propertyOrdering\": [\n \"location\"\n ]\n - \ }\n }\n ]\n }\n ]\n}" - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '1731' - Content-Type: - - application/json - User-Agent: - - python-requests/2.32.3 - x-goog-api-client: - - model-builder/1.79.0+top_google_constructor_method+vertexai.generative_models.GenerativeModel.generate_content - gl-python/3.11.9 grpc/1.68.1 gax/2.23.0 gapic/1.79.0+top_google_constructor_method+vertexai.generative_models.GenerativeModel.generate_content - x-goog-request-params: - - model=projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro - method: POST - uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint - response: - body: - string: !!binary | - H4sIAAAAAAAC/91TwU7rMBC89ytWPRcHECdOVG3EqwQFkXJCCBlnSRccb7A3BYT679ghQODAgeM7 - OTueHc+OndcRwBi9Zz8+hNdYxNJwibE62N2bvAM1hqCrhI0v8LHFIEABagqBXAU+QuSxBN3KGp2Q - 0ULswEQsldoqyJ8bNBI5Z9NIgn3QxkRREH5ANwHLFcUG5gdCYA8chTxstKVfVQtEWIs04TDLStyg - 5QZ9UBVzZVEZrjPquPKSBarcDrnsCW8T1bALbHGn8XwfjalxP2kQLW1Ig14up5erf/lytZhNV/n8 - g1CiaLKJcdUB0IfWbR7JS9OllNbehm4odFZ6V74xao63bbVwd9yrDpT/54z7WbeTvyeXp5f6MzmP - OrBL3bOLfJ6ubHpS3JwuimKxPP4WMdeaOuL3E4acOl5DqUV//g5f+JrLr15lLLel0tRYLXfsa7XZ - U+cxODIpwwL9hgyqY3ToteCMncSYBiel1/ZOSqIDnR/ePhu2o+F6PUpf29EbabJOVcEDAAA= - headers: - Content-Encoding: - - gzip - Content-Type: - - application/json; charset=UTF-8 - Date: - - Tue, 30 Sep 2025 15:26:53 GMT - Server: - - scaffolding on HTTPServer2 - Transfer-Encoding: - - chunked - Vary: - - Origin - - X-Origin - - Referer - WWW-Authenticate: - - Bearer realm="https://accounts.google.com/" - X-Content-Type-Options: - - nosniff - X-Frame-Options: - - SAMEORIGIN - X-XSS-Protection: - - '0' - status: - code: 401 - message: Unauthorized -version: 1 diff --git a/cassettes/test_tool_events_no_content.yaml b/cassettes/test_tool_events_no_content.yaml deleted file mode 100644 index cb70bab7dd..0000000000 --- a/cassettes/test_tool_events_no_content.yaml +++ /dev/null @@ -1,83 +0,0 @@ -interactions: -- request: - body: "{\n \"contents\": [\n {\n \"role\": \"user\",\n \"parts\": - [\n {\n \"text\": \"Get weather details in New Delhi and San - Francisco?\"\n }\n ]\n },\n {\n \"role\": \"model\",\n - \ \"parts\": [\n {\n \"functionCall\": {\n \"name\": - \"get_current_weather\",\n \"args\": {\n \"location\": - \"New Delhi\"\n }\n }\n },\n {\n \"functionCall\": - {\n \"name\": \"get_current_weather\",\n \"args\": {\n - \ \"location\": \"San Francisco\"\n }\n }\n - \ }\n ]\n },\n {\n \"role\": \"user\",\n \"parts\": - [\n {\n \"functionResponse\": {\n \"name\": \"get_current_weather\",\n - \ \"response\": {\n \"content\": \"{\\\"temperature\\\": - 35, \\\"unit\\\": \\\"C\\\"}\"\n }\n }\n },\n {\n - \ \"functionResponse\": {\n \"name\": \"get_current_weather\",\n - \ \"response\": {\n \"content\": \"{\\\"temperature\\\": - 25, \\\"unit\\\": \\\"C\\\"}\"\n }\n }\n }\n ]\n - \ }\n ],\n \"tools\": [\n {\n \"functionDeclarations\": [\n {\n - \ \"name\": \"get_current_weather\",\n \"description\": \"Get - the current weather in a given location\",\n \"parameters\": {\n \"type\": - 6,\n \"properties\": {\n \"location\": {\n \"type\": - 1,\n \"description\": \"The location for which to get the weather. - It can be a city name, a city name and state, or a zip code. Examples: 'San - Francisco', 'San Francisco, CA', '95616', etc.\"\n }\n },\n - \ \"propertyOrdering\": [\n \"location\"\n ]\n - \ }\n }\n ]\n }\n ]\n}" - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '1731' - Content-Type: - - application/json - User-Agent: - - python-requests/2.32.3 - x-goog-api-client: - - model-builder/1.79.0+top_google_constructor_method+vertexai.generative_models.GenerativeModel.generate_content - gl-python/3.11.9 grpc/1.68.1 gax/2.23.0 gapic/1.79.0+top_google_constructor_method+vertexai.generative_models.GenerativeModel.generate_content - x-goog-request-params: - - model=projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro - method: POST - uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint - response: - body: - string: !!binary | - H4sIAAAAAAAC/91TwU7rMBC89ytWPRcHECdOVG3EqwQFkXJCCBlnSRccb7A3BYT679ghQODAgeM7 - OTueHc+OndcRwBi9Zz8+hNdYxNJwibE62N2bvAM1hqCrhI0v8LHFIEABagqBXAU+QuSxBN3KGp2Q - 0ULswEQsldoqyJ8bNBI5Z9NIgn3QxkRREH5ANwHLFcUG5gdCYA8chTxstKVfVQtEWIs04TDLStyg - 5QZ9UBVzZVEZrjPquPKSBarcDrnsCW8T1bALbHGn8XwfjalxP2kQLW1Ig14up5erf/lytZhNV/n8 - g1CiaLKJcdUB0IfWbR7JS9OllNbehm4odFZ6V74xao63bbVwd9yrDpT/54z7WbeTvyeXp5f6MzmP - OrBL3bOLfJ6ubHpS3JwuimKxPP4WMdeaOuL3E4acOl5DqUV//g5f+JrLr15lLLel0tRYLXfsa7XZ - U+cxODIpwwL9hgyqY3ToteCMncSYBiel1/ZOSqIDnR/ePhu2o+F6PUpf29EbabJOVcEDAAA= - headers: - Content-Encoding: - - gzip - Content-Type: - - application/json; charset=UTF-8 - Date: - - Tue, 30 Sep 2025 15:27:34 GMT - Server: - - scaffolding on HTTPServer2 - Transfer-Encoding: - - chunked - Vary: - - Origin - - X-Origin - - Referer - WWW-Authenticate: - - Bearer realm="https://accounts.google.com/" - X-Content-Type-Options: - - nosniff - X-Frame-Options: - - SAMEORIGIN - X-XSS-Protection: - - '0' - status: - code: 401 - message: Unauthorized -version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/manual/.env b/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/manual/.env index 4fc56894f3..9f5d5d11db 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/manual/.env +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/manual/.env @@ -16,7 +16,7 @@ OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=SPAN_AND_EVENT # Optional hook that will upload prompt and response content to some external destination. # For example fsspec. -OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK = "fsspec" +OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK = "upload" # Required if using a completion hook. The path to upload content to for example gs://my_bucket. OTEL_INSTRUMENTATION_GENAI_UPLOAD_BASE_PATH = "gs://my_bucket" \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/zero-code/.env b/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/zero-code/.env index a78d0ca6cc..f224ac248a 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/zero-code/.env +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/zero-code/.env @@ -22,7 +22,7 @@ OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=SPAN_AND_EVENT # Optional hook that will upload prompt and response content to some external destination. # For example fsspec. -OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK = "fsspec" +OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK = "upload" # Required if using a completion hook. The path to upload content to for example gs://my_bucket. OTEL_INSTRUMENTATION_GENAI_UPLOAD_BASE_PATH = "gs://my_bucket" \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py index 639e127920..7bd9122d3e 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py @@ -193,7 +193,7 @@ def handle_response( ] if response: outputs = convert_response_to_output_messages(response) - content = { + content_attributes = { k: [asdict(x) for x in v] for k, v in [ ( @@ -207,26 +207,25 @@ def handle_response( } if span.is_recording(): span.set_attributes(attributes) - if capture_content in frozenset( - [ - ContentCapturingMode.SPAN_AND_EVENT, - ContentCapturingMode.SPAN_ONLY, - ] + if capture_content in ( + ContentCapturingMode.SPAN_AND_EVENT, + ContentCapturingMode.SPAN_ONLY, ): span.set_attributes( - {k: json.dumps(v) for k, v in content.items()} + { + k: json.dumps(v) + for k, v in content_attributes.items() + } ) event = Event( name="gen_ai.client.inference.operation.details", ) event.attributes = attributes - if capture_content in frozenset( - [ - ContentCapturingMode.SPAN_AND_EVENT, - ContentCapturingMode.EVENT_ONLY, - ] + if capture_content in ( + ContentCapturingMode.SPAN_AND_EVENT, + ContentCapturingMode.EVENT_ONLY, ): - event.attributes |= content + event.attributes |= content_attributes self.event_logger.emit(event) self.completion_hook.on_completion( inputs=inputs, diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/conftest.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/conftest.py index 1280a88339..ade078e17d 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/conftest.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/conftest.py @@ -199,7 +199,7 @@ def instrument_with_upload_hook( os.environ.update( { OTEL_SEMCONV_STABILITY_OPT_IN: "gen_ai_latest_experimental", - "OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK": "fsspec_upload", + "OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK": "upload", "OTEL_INSTRUMENTATION_GENAI_UPLOAD_BASE_PATH": "memory://", OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "SPAN_AND_EVENT", } From 35ddd5290f02a084d656d5c653871567271dfa54 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Thu, 2 Oct 2025 18:27:22 +0000 Subject: [PATCH 06/14] move decoder to utils --- .../opentelemetry/instrumentation/vertexai/patch.py | 3 ++- .../tests/conftest.py | 2 +- .../util/genai/_upload/completion_hook.py | 10 +--------- .../src/opentelemetry/util/genai/utils.py | 10 ++++++++++ 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py index 7bd9122d3e..2037da7848 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py @@ -53,6 +53,7 @@ ContentCapturingMode, Text, ) +from opentelemetry.util.genai.utils import Base64JsonEncoder if TYPE_CHECKING: from google.cloud.aiplatform_v1.services.prediction_service import client @@ -213,7 +214,7 @@ def handle_response( ): span.set_attributes( { - k: json.dumps(v) + k: json.dumps(v, cls=Base64JsonEncoder) for k, v in content_attributes.items() } ) diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/conftest.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/conftest.py index ade078e17d..349bf17820 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/conftest.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/conftest.py @@ -192,7 +192,7 @@ def instrument_with_experimental_semconvs( @pytest.fixture(scope="function") def instrument_with_upload_hook( - tracer_provider, event_logger_provider, meter_provider, request + tracer_provider, event_logger_provider, meter_provider ): # Reset global state.. _OpenTelemetrySemanticConventionStability._initialized = False diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_upload/completion_hook.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_upload/completion_hook.py index 351b74cc3b..de4aa23534 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_upload/completion_hook.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_upload/completion_hook.py @@ -19,7 +19,6 @@ import logging import posixpath import threading -from base64 import b64encode from concurrent.futures import Future, ThreadPoolExecutor from contextlib import ExitStack from dataclasses import asdict, dataclass @@ -34,7 +33,7 @@ from opentelemetry._logs import LogRecord from opentelemetry.semconv._incubating.attributes import gen_ai_attributes from opentelemetry.trace import Span -from opentelemetry.util.genai import types +from opentelemetry.util.genai import Base64JsonEncoder, types from opentelemetry.util.genai.completion_hook import CompletionHook from opentelemetry.util.genai.environment_variables import ( OTEL_INSTRUMENTATION_GENAI_UPLOAD_FORMAT, @@ -281,10 +280,3 @@ def shutdown(self, *, timeout_sec: float = 10.0) -> None: # Queue is flushed and blocked, start shutdown self._executor.shutdown(wait=False) - - -class Base64JsonEncoder(json.JSONEncoder): - def default(self, o: Any) -> Any: - if isinstance(o, bytes): - return b64encode(o).decode() - return super().default(o) diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py index 91cb9221f1..a2d604bda6 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py @@ -12,8 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import logging import os +from base64 import b64encode +from typing import Any from opentelemetry.instrumentation._semconv import ( _OpenTelemetrySemanticConventionStability, @@ -54,3 +57,10 @@ def get_content_capturing_mode() -> ContentCapturingMode: ", ".join(e.name for e in ContentCapturingMode), ) return ContentCapturingMode.NO_CONTENT + + +class Base64JsonEncoder(json.JSONEncoder): + def default(self, o: Any) -> Any: + if isinstance(o, bytes): + return b64encode(o).decode() + return super().default(o) From d17cf1d0d872eab8263a34d402bbb2d55e193140 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Thu, 2 Oct 2025 18:44:49 +0000 Subject: [PATCH 07/14] fix import --- .../src/opentelemetry/util/genai/_upload/completion_hook.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_upload/completion_hook.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_upload/completion_hook.py index de4aa23534..cc6a3fdbe5 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_upload/completion_hook.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_upload/completion_hook.py @@ -33,11 +33,12 @@ from opentelemetry._logs import LogRecord from opentelemetry.semconv._incubating.attributes import gen_ai_attributes from opentelemetry.trace import Span -from opentelemetry.util.genai import Base64JsonEncoder, types +from opentelemetry.util.genai import types from opentelemetry.util.genai.completion_hook import CompletionHook from opentelemetry.util.genai.environment_variables import ( OTEL_INSTRUMENTATION_GENAI_UPLOAD_FORMAT, ) +from opentelemetry.util.genai.utils import Base64JsonEncoder GEN_AI_INPUT_MESSAGES_REF: Final = ( gen_ai_attributes.GEN_AI_INPUT_MESSAGES + "_ref" From dbd54ff3baa299462e6cde106176a92592fe710b Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Thu, 2 Oct 2025 19:01:57 +0000 Subject: [PATCH 08/14] Respond to comments --- .../test_function_calling_experimental.py | 3 + .../test_tool_events_no_content.yaml | 83 ------------------- 2 files changed, 3 insertions(+), 83 deletions(-) delete mode 100644 tests/cassettes/test_tool_events_no_content.yaml diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py index 8b6a0855e4..71551b7e96 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py @@ -1,4 +1,5 @@ import json +import time from typing import Any import fsspec @@ -271,6 +272,8 @@ def test_tool_events_with_completion_hook( assert len(spans) == 1 logs = log_exporter.get_finished_logs() assert len(logs) == 1 + # File upload takes a few seconds sometimes. + time.sleep(3) assert_fsspec_equal( spans[0].attributes["gen_ai.output.messages_ref"], [ diff --git a/tests/cassettes/test_tool_events_no_content.yaml b/tests/cassettes/test_tool_events_no_content.yaml deleted file mode 100644 index a39a0dbf40..0000000000 --- a/tests/cassettes/test_tool_events_no_content.yaml +++ /dev/null @@ -1,83 +0,0 @@ -interactions: -- request: - body: "{\n \"contents\": [\n {\n \"role\": \"user\",\n \"parts\": - [\n {\n \"text\": \"Get weather details in New Delhi and San - Francisco?\"\n }\n ]\n },\n {\n \"role\": \"model\",\n - \ \"parts\": [\n {\n \"functionCall\": {\n \"name\": - \"get_current_weather\",\n \"args\": {\n \"location\": - \"New Delhi\"\n }\n }\n },\n {\n \"functionCall\": - {\n \"name\": \"get_current_weather\",\n \"args\": {\n - \ \"location\": \"San Francisco\"\n }\n }\n - \ }\n ]\n },\n {\n \"role\": \"user\",\n \"parts\": - [\n {\n \"functionResponse\": {\n \"name\": \"get_current_weather\",\n - \ \"response\": {\n \"content\": \"{\\\"temperature\\\": - 35, \\\"unit\\\": \\\"C\\\"}\"\n }\n }\n },\n {\n - \ \"functionResponse\": {\n \"name\": \"get_current_weather\",\n - \ \"response\": {\n \"content\": \"{\\\"temperature\\\": - 25, \\\"unit\\\": \\\"C\\\"}\"\n }\n }\n }\n ]\n - \ }\n ],\n \"tools\": [\n {\n \"functionDeclarations\": [\n {\n - \ \"name\": \"get_current_weather\",\n \"description\": \"Get - the current weather in a given location\",\n \"parameters\": {\n \"type\": - 6,\n \"properties\": {\n \"location\": {\n \"type\": - 1,\n \"description\": \"The location for which to get the weather. - It can be a city name, a city name and state, or a zip code. Examples: 'San - Francisco', 'San Francisco, CA', '95616', etc.\"\n }\n },\n - \ \"propertyOrdering\": [\n \"location\"\n ]\n - \ }\n }\n ]\n }\n ]\n}" - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '1731' - Content-Type: - - application/json - User-Agent: - - python-requests/2.32.3 - x-goog-api-client: - - model-builder/1.79.0+top_google_constructor_method+vertexai.generative_models.GenerativeModel.generate_content - gl-python/3.11.9 grpc/1.68.1 gax/2.23.0 gapic/1.79.0+top_google_constructor_method+vertexai.generative_models.GenerativeModel.generate_content - x-goog-request-params: - - model=projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro - method: POST - uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint - response: - body: - string: !!binary | - H4sIAAAAAAAC/91TwU7rMBC89ytWPRcHECdOVG3EqwQFkXJCCBlnSRccb7A3BYT679ghQODAgeM7 - OTueHc+OndcRwBi9Zz8+hNdYxNJwibE62N2bvAM1hqCrhI0v8LHFIEABagqBXAU+QuSxBN3KGp2Q - 0ULswEQsldoqyJ8bNBI5Z9NIgn3QxkRREH5ANwHLFcUG5gdCYA8chTxstKVfVQtEWIs04TDLStyg - 5QZ9UBVzZVEZrjPquPKSBarcDrnsCW8T1bALbHGn8XwfjalxP2kQLW1Ig14up5erf/lytZhNV/n8 - g1CiaLKJcdUB0IfWbR7JS9OllNbehm4odFZ6V74xao63bbVwd9yrDpT/54z7WbeTvyeXp5f6MzmP - OrBL3bOLfJ6ubHpS3JwuimKxPP4WMdeaOuL3E4acOl5DqUV//g5f+JrLr15lLLel0tRYLXfsa7XZ - U+cxODIpwwL9hgyqY3ToteCMncSYBiel1/ZOSqIDnR/ePhu2o+F6PUpf29EbabJOVcEDAAA= - headers: - Content-Encoding: - - gzip - Content-Type: - - application/json; charset=UTF-8 - Date: - - Tue, 30 Sep 2025 15:31:01 GMT - Server: - - scaffolding on HTTPServer2 - Transfer-Encoding: - - chunked - Vary: - - Origin - - X-Origin - - Referer - WWW-Authenticate: - - Bearer realm="https://accounts.google.com/" - X-Content-Type-Options: - - nosniff - X-Frame-Options: - - SAMEORIGIN - X-XSS-Protection: - - '0' - status: - code: 401 - message: Unauthorized -version: 1 From 28cad57b5fd6644f70cbb5fd77585eeac17c1318 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Thu, 2 Oct 2025 19:07:55 +0000 Subject: [PATCH 09/14] resolve docs error hopefully --- .../src/opentelemetry/util/genai/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py index a2d604bda6..62cd787dd8 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import logging import os from base64 import b64encode +from json import encoder from typing import Any from opentelemetry.instrumentation._semconv import ( @@ -59,7 +59,7 @@ def get_content_capturing_mode() -> ContentCapturingMode: return ContentCapturingMode.NO_CONTENT -class Base64JsonEncoder(json.JSONEncoder): +class Base64JsonEncoder(encoder.JSONEncoder): def default(self, o: Any) -> Any: if isinstance(o, bytes): return b64encode(o).decode() From 97d300fec41a65ffae00e288b665c89521672629 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Thu, 2 Oct 2025 19:17:01 +0000 Subject: [PATCH 10/14] Add docstring --- .../src/opentelemetry/util/genai/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py index 62cd787dd8..4d3de66d6b 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import logging import os from base64 import b64encode -from json import encoder from typing import Any from opentelemetry.instrumentation._semconv import ( @@ -59,7 +59,9 @@ def get_content_capturing_mode() -> ContentCapturingMode: return ContentCapturingMode.NO_CONTENT -class Base64JsonEncoder(encoder.JSONEncoder): +class Base64JsonEncoder(json.JSONEncoder): + """Should be used to serialize python objects to json that may contain bytes.""" + def default(self, o: Any) -> Any: if isinstance(o, bytes): return b64encode(o).decode() From 08103d8c6cc083e911ffbab7b884bc54a0956033 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Fri, 3 Oct 2025 15:13:42 +0000 Subject: [PATCH 11/14] Update system instruction from 1 text part to a list of MessageParts --- .../instrumentation/vertexai/patch.py | 36 ++++++++++++------- .../instrumentation/vertexai/utils.py | 25 ++----------- 2 files changed, 26 insertions(+), 35 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py index 2037da7848..29b2f17d80 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py @@ -35,8 +35,8 @@ ) from opentelemetry.instrumentation.vertexai.utils import ( GenerateContentParams, - convert_content_to_message, - convert_response_to_output_messages, + _map_finish_reason, + convert_content_to_message_parts, get_genai_request_attributes, get_genai_response_attributes, get_server_attributes, @@ -51,7 +51,8 @@ from opentelemetry.util.genai.completion_hook import CompletionHook from opentelemetry.util.genai.types import ( ContentCapturingMode, - Text, + InputMessage, + OutputMessage, ) from opentelemetry.util.genai.utils import Base64JsonEncoder @@ -179,21 +180,30 @@ def handle_response( ) system_instructions, inputs, outputs = [], [], [] if params.system_instruction: - system_instructions = [ - Text( - content="\n".join( - part.text - for part in params.system_instruction.parts - ) - ) - ] + system_instructions = convert_content_to_message_parts( + params.system_instruction + ) if params.contents: inputs = [ - convert_content_to_message(content) + InputMessage( + role=content.role, + parts=convert_content_to_message_parts(content), + ) for content in params.contents ] if response: - outputs = convert_response_to_output_messages(response) + outputs = [ + OutputMessage( + finish_reason=_map_finish_reason( + candidate.finish_reason + ), + role=candidate.content.role, + parts=convert_content_to_message_parts( + candidate.content + ), + ) + for candidate in response.candidates + ] content_attributes = { k: [asdict(x) for x in v] for k, v in [ diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py index e179bdd28e..9a9dd9c2d4 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py @@ -53,9 +53,7 @@ from opentelemetry.util.genai.types import ( ContentCapturingMode, FinishReason, - InputMessage, MessagePart, - OutputMessage, Text, ToolCall, ToolCallResponse, @@ -310,26 +308,9 @@ def request_to_events( yield user_event(role=content.role, content=request_content) -def convert_response_to_output_messages( - response: prediction_service.GenerateContentResponse - | prediction_service_v1beta1.GenerateContentResponse, -) -> list[OutputMessage]: - output_messages: list[OutputMessage] = [] - for candidate in response.candidates: - message = convert_content_to_message(candidate.content) - output_messages.append( - OutputMessage( - finish_reason=_map_finish_reason(candidate.finish_reason), - role=message.role, - parts=message.parts, - ) - ) - return output_messages - - -def convert_content_to_message( +def convert_content_to_message_parts( content: content.Content | content_v1beta1.Content, -) -> InputMessage: +) -> list[MessagePart]: parts: MessagePart = [] for idx, part in enumerate(content.parts): if "function_response" in part: @@ -359,7 +340,7 @@ def convert_content_to_message( ) dict_part["type"] = type(part) parts.append(dict_part) - return InputMessage(role=content.role, parts=parts) + return parts def response_to_events( From 19af618662f0821fd8ec073d07fa3523f00e9717 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Fri, 3 Oct 2025 15:37:22 +0000 Subject: [PATCH 12/14] Update json.dumps call to use different separators --- .../instrumentation/vertexai/patch.py | 6 +++++- .../tests/test_chat_completions_experimental.py | 14 +++++++------- .../tests/test_function_calling_experimental.py | 8 ++++---- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py index 29b2f17d80..a66083a1ad 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py @@ -224,7 +224,11 @@ def handle_response( ): span.set_attributes( { - k: json.dumps(v, cls=Base64JsonEncoder) + k: json.dumps( + v, + cls=Base64JsonEncoder, + separators=(",", ":"), + ) for k, v in content_attributes.items() } ) diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py index 84aa9aee8b..85a0d1cee9 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py @@ -56,8 +56,8 @@ def test_generate_content( "gen_ai.usage.output_tokens": 5, "server.address": "us-central1-aiplatform.googleapis.com", "server.port": 443, - "gen_ai.input.messages": '[{"role": "user", "parts": [{"content": "Say this is a test", "type": "text"}]}]', - "gen_ai.output.messages": '[{"role": "model", "parts": [{"content": "This is a test.", "type": "text"}], "finish_reason": "stop"}]', + "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"}]}]', + "gen_ai.output.messages": '[{"role":"model","parts":[{"content":"This is a test.","type":"text"}],"finish_reason":"stop"}]', } logs = log_exporter.get_finished_logs() @@ -109,8 +109,8 @@ def test_generate_content_without_events( assert spans[0].name == "chat gemini-2.5-pro" assert dict(spans[0].attributes) == { "gen_ai.operation.name": "chat", - "gen_ai.output.messages": '[{"role": "model", "parts": [{"content": "This is a test.", "type": "text"}], "finish_reason": "stop"}]', - "gen_ai.input.messages": '[{"role": "user", "parts": [{"content": "Say this is a test", "type": "text"}]}]', + "gen_ai.output.messages": '[{"role":"model","parts":[{"content":"This is a test.","type":"text"}],"finish_reason":"stop"}]', + "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"}]}]', "gen_ai.request.model": "gemini-2.5-pro", "gen_ai.response.finish_reasons": ("stop",), "gen_ai.response.model": "gemini-2.5-pro", @@ -177,7 +177,7 @@ def test_generate_content_empty_model( "gen_ai.request.model": "", "server.address": "us-central1-aiplatform.googleapis.com", "server.port": 443, - "gen_ai.input.messages": '[{"role": "user", "parts": [{"content": "Say this is a test", "type": "text"}]}]', + "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"}]}]', } assert_span_error(spans[0]) @@ -210,7 +210,7 @@ def test_generate_content_missing_model( "gen_ai.request.model": "gemini-does-not-exist", "server.address": "us-central1-aiplatform.googleapis.com", "server.port": 443, - "gen_ai.input.messages": '[{"role": "user", "parts": [{"content": "Say this is a test", "type": "text"}]}]', + "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"}]}]', } assert_span_error(spans[0]) @@ -245,7 +245,7 @@ def test_generate_content_invalid_temperature( "gen_ai.request.temperature": 1000.0, "server.address": "us-central1-aiplatform.googleapis.com", "server.port": 443, - "gen_ai.input.messages": '[{"role": "user", "parts": [{"content": "Say this is a test", "type": "text"}]}]', + "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"}]}]', } assert_span_error(spans[0]) diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py index 71551b7e96..f0bb1307e3 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py @@ -38,8 +38,8 @@ def test_function_call_choice( "gen_ai.usage.input_tokens": 74, "gen_ai.usage.output_tokens": 16, "server.address": "us-central1-aiplatform.googleapis.com", - "gen_ai.input.messages": '[{"role": "user", "parts": [{"content": "Get weather details in New Delhi and San Francisco?", "type": "text"}]}]', - "gen_ai.output.messages": '[{"role": "model", "parts": [{"arguments": {"location": "New Delhi"}, "name": "get_current_weather", "id": "get_current_weather_0", "type": "tool_call"}, {"arguments": {"location": "San Francisco"}, "name": "get_current_weather", "id": "get_current_weather_1", "type": "tool_call"}], "finish_reason": "stop"}]', + "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Get weather details in New Delhi and San Francisco?","type":"text"}]}]', + "gen_ai.output.messages": '[{"role":"model","parts":[{"arguments":{"location":"New Delhi"},"name":"get_current_weather","id":"get_current_weather_0","type":"tool_call"},{"arguments":{"location":"San Francisco"},"name":"get_current_weather","id":"get_current_weather_1","type":"tool_call"}],"finish_reason":"stop"}]', "server.port": 443, } @@ -136,8 +136,8 @@ def test_tool_events( "gen_ai.usage.output_tokens": 26, "server.address": "us-central1-aiplatform.googleapis.com", "server.port": 443, - "gen_ai.input.messages": '[{"role": "user", "parts": [{"content": "Get weather details in New Delhi and San Francisco?", "type": "text"}]}, {"role": "model", "parts": [{"arguments": {"location": "New Delhi"}, "name": "get_current_weather", "id": "get_current_weather_0", "type": "tool_call"}, {"arguments": {"location": "San Francisco"}, "name": "get_current_weather", "id": "get_current_weather_1", "type": "tool_call"}]}, {"role": "user", "parts": [{"response": {"content": "{\\"temperature\\": 35, \\"unit\\": \\"C\\"}"}, "id": "get_current_weather_0", "type": "tool_call_response"}, {"response": {"content": "{\\"temperature\\": 25, \\"unit\\": \\"C\\"}"}, "id": "get_current_weather_1", "type": "tool_call_response"}]}]', - "gen_ai.output.messages": '[{"role": "model", "parts": [{"content": "The current temperature in New Delhi is 35\\u00b0C, and in San Francisco, it is 25\\u00b0C.", "type": "text"}], "finish_reason": "stop"}]', + "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Get weather details in New Delhi and San Francisco?","type":"text"}]},{"role":"model","parts":[{"arguments":{"location":"New Delhi"},"name":"get_current_weather","id":"get_current_weather_0","type":"tool_call"},{"arguments":{"location":"San Francisco"},"name":"get_current_weather","id":"get_current_weather_1","type":"tool_call"}]},{"role":"user","parts":[{"response":{"content":"{\\"temperature\\": 35, \\"unit\\": \\"C\\"}"},"id":"get_current_weather_0","type":"tool_call_response"},{"response":{"content":"{\\"temperature\\": 25, \\"unit\\": \\"C\\"}"},"id":"get_current_weather_1","type":"tool_call_response"}]}]', + "gen_ai.output.messages": '[{"role":"model","parts":[{"content":"The current temperature in New Delhi is 35\\u00b0C, and in San Francisco, it is 25\\u00b0C.","type":"text"}],"finish_reason":"stop"}]', } logs = log_exporter.get_finished_logs() assert len(logs) == 1 From 9390cb5be7ef4647f8da23fc41cfba46804d178f Mon Sep 17 00:00:00 2001 From: Aaron Abbott Date: Fri, 3 Oct 2025 15:52:48 +0000 Subject: [PATCH 13/14] expose just a json util --- .../instrumentation/vertexai/patch.py | 9 ++------- .../util/genai/_upload/completion_hook.py | 10 ++-------- .../src/opentelemetry/util/genai/utils.py | 19 +++++++++++++++++-- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py index a66083a1ad..e7e6621d73 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py @@ -14,7 +14,6 @@ from __future__ import annotations -import json from contextlib import contextmanager from dataclasses import asdict from typing import ( @@ -54,7 +53,7 @@ InputMessage, OutputMessage, ) -from opentelemetry.util.genai.utils import Base64JsonEncoder +from opentelemetry.util.genai.utils import gen_ai_json_dumps if TYPE_CHECKING: from google.cloud.aiplatform_v1.services.prediction_service import client @@ -224,11 +223,7 @@ def handle_response( ): span.set_attributes( { - k: json.dumps( - v, - cls=Base64JsonEncoder, - separators=(",", ":"), - ) + k: gen_ai_json_dumps(v) for k, v in content_attributes.items() } ) diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_upload/completion_hook.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_upload/completion_hook.py index cc6a3fdbe5..86cb4f0c51 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_upload/completion_hook.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_upload/completion_hook.py @@ -15,7 +15,6 @@ from __future__ import annotations -import json import logging import posixpath import threading @@ -38,7 +37,7 @@ from opentelemetry.util.genai.environment_variables import ( OTEL_INSTRUMENTATION_GENAI_UPLOAD_FORMAT, ) -from opentelemetry.util.genai.utils import Base64JsonEncoder +from opentelemetry.util.genai.utils import gen_ai_json_dump GEN_AI_INPUT_MESSAGES_REF: Final = ( gen_ai_attributes.GEN_AI_INPUT_MESSAGES + "_ref" @@ -192,12 +191,7 @@ def _do_upload( with self._fs.open(path, "w", content_type=content_type) as file: for message in message_lines: - json.dump( - message, - file, - separators=(",", ":"), - cls=Base64JsonEncoder, - ) + gen_ai_json_dump(message, file) file.write("\n") def on_completion( diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py index 4d3de66d6b..2a3c506dd9 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py @@ -16,6 +16,7 @@ import logging import os from base64 import b64encode +from functools import partial from typing import Any from opentelemetry.instrumentation._semconv import ( @@ -59,10 +60,24 @@ def get_content_capturing_mode() -> ContentCapturingMode: return ContentCapturingMode.NO_CONTENT -class Base64JsonEncoder(json.JSONEncoder): - """Should be used to serialize python objects to json that may contain bytes.""" +class _GenAiJsonEncoder(json.JSONEncoder): + """Should be used by GenAI instrumentations when serializing objects that may contain + bytes, datetimes, etc. for GenAI observability.""" def default(self, o: Any) -> Any: if isinstance(o, bytes): return b64encode(o).decode() return super().default(o) + + +gen_ai_json_dump = partial( + json.dump, separators=(",", ":"), cls=_GenAiJsonEncoder +) +"""Should be used by GenAI instrumentations when serializing objects that may contain +bytes, datetimes, etc. for GenAI observability.""" + +gen_ai_json_dumps = partial( + json.dumps, separators=(",", ":"), cls=_GenAiJsonEncoder +) +"""Should be used by GenAI instrumentations when serializing objects that may contain +bytes, datetimes, etc. for GenAI observability.""" From f2c10578f4c9a7ddddce9dc65c63e7a57db3ce06 Mon Sep 17 00:00:00 2001 From: Aaron Abbott Date: Fri, 3 Oct 2025 16:07:59 +0000 Subject: [PATCH 14/14] remove unnecessary doc --- .../src/opentelemetry/util/genai/utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py index 2a3c506dd9..0083d5144c 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py @@ -61,9 +61,6 @@ def get_content_capturing_mode() -> ContentCapturingMode: class _GenAiJsonEncoder(json.JSONEncoder): - """Should be used by GenAI instrumentations when serializing objects that may contain - bytes, datetimes, etc. for GenAI observability.""" - def default(self, o: Any) -> Any: if isinstance(o, bytes): return b64encode(o).decode()