diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md b/instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md index a46355ba59..a839324a0d 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md @@ -7,9 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased -- Start making changes to implement the big semantic convention changes made in https://github.com/open-telemetry/semantic-conventions/pull/2179. -Now only a single event (`gen_ai.client.inference.operation.details`) is used to capture Chat History. These changes will be opt-in, -users will need to set the environment variable OTEL_SEMCONV_STABILITY_OPT_IN to `gen_ai_latest_experimental` to see them ([#3386](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3386)). +- Update instrumentation to use the latest semantic convention changes made in https://github.com/open-telemetry/semantic-conventions/pull/2179. +Now only a single event and span (`gen_ai.client.inference.operation.details`) are used to capture prompt and response content. These changes are opt-in, +users will need to set the environment variable OTEL_SEMCONV_STABILITY_OPT_IN to `gen_ai_latest_experimental` to see them ([#3799](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3799)) and ([#3709](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3709)). - Implement uninstrument for `opentelemetry-instrumentation-vertexai` ([#3328](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3328)) - VertexAI support for async calling diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/manual/.env b/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/manual/.env index 9ca033983f..9f5d5d11db 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/manual/.env +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/manual/.env @@ -6,3 +6,17 @@ OTEL_SERVICE_NAME=opentelemetry-python-vertexai # Change to 'false' to hide prompt and completion content OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true + +# Alternatively set this env var to enable the latest semantic conventions: +OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental + +# When using the latest experimental flag this env var controls which telemetry signals will have prompt and response content included in them. +# Choices are NO_CONTENT, SPAN_ONLY, EVENT_ONLY, SPAN_AND_EVENT. +OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=SPAN_AND_EVENT + +# Optional hook that will upload prompt and response content to some external destination. +# For example fsspec. +OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK = "upload" + +# Required if using a completion hook. The path to upload content to for example gs://my_bucket. +OTEL_INSTRUMENTATION_GENAI_UPLOAD_BASE_PATH = "gs://my_bucket" \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/zero-code/.env b/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/zero-code/.env index eefbd59500..f224ac248a 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/zero-code/.env +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/examples/zero-code/.env @@ -12,3 +12,17 @@ OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true # Change to 'false' to hide prompt and completion content OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true + +# Alternatively set this env var to enable the latest semantic conventions: +OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental + +# When using the latest experimental flag this env var controls which telemetry signals will have prompt and response content included in them. +# Choices are NO_CONTENT, SPAN_ONLY, EVENT_ONLY, SPAN_AND_EVENT. +OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=SPAN_AND_EVENT + +# Optional hook that will upload prompt and response content to some external destination. +# For example fsspec. +OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK = "upload" + +# Required if using a completion hook. The path to upload content to for example gs://my_bucket. +OTEL_INSTRUMENTATION_GENAI_UPLOAD_BASE_PATH = "gs://my_bucket" \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/__init__.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/__init__.py index fd241d1316..98e2ed57ed 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/__init__.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/__init__.py @@ -60,6 +60,7 @@ from opentelemetry.instrumentation.vertexai.utils import is_content_enabled from opentelemetry.semconv.schemas import Schemas from opentelemetry.trace import get_tracer +from opentelemetry.util.genai.completion_hook import load_completion_hook def _methods_to_wrap( @@ -109,6 +110,9 @@ def instrumentation_dependencies(self) -> Collection[str]: def _instrument(self, **kwargs: Any): """Enable VertexAI instrumentation.""" + completion_hook = ( + kwargs.get("completion_hook") or load_completion_hook() + ) sem_conv_opt_in_mode = _OpenTelemetrySemanticConventionStability._get_opentelemetry_stability_opt_in_mode( _OpenTelemetryStabilitySignalType.GEN_AI, ) @@ -141,6 +145,7 @@ def _instrument(self, **kwargs: Any): event_logger, is_content_enabled(sem_conv_opt_in_mode), sem_conv_opt_in_mode, + completion_hook, ) elif sem_conv_opt_in_mode == _StabilityMode.GEN_AI_LATEST_EXPERIMENTAL: # Type checker now knows it's the other literal @@ -149,6 +154,7 @@ def _instrument(self, **kwargs: Any): event_logger, is_content_enabled(sem_conv_opt_in_mode), sem_conv_opt_in_mode, + completion_hook, ) else: raise RuntimeError(f"{sem_conv_opt_in_mode} mode not supported") diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py index c35a9a317a..e7e6621d73 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py @@ -15,6 +15,7 @@ from __future__ import annotations from contextlib import contextmanager +from dataclasses import asdict from typing import ( TYPE_CHECKING, Any, @@ -27,13 +28,14 @@ overload, ) -from opentelemetry._events import EventLogger +from opentelemetry._events import Event, EventLogger from opentelemetry.instrumentation._semconv import ( _StabilityMode, ) from opentelemetry.instrumentation.vertexai.utils import ( GenerateContentParams, - create_operation_details_event, + _map_finish_reason, + convert_content_to_message_parts, get_genai_request_attributes, get_genai_response_attributes, get_server_attributes, @@ -41,8 +43,17 @@ request_to_events, response_to_events, ) +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) from opentelemetry.trace import SpanKind, Tracer -from opentelemetry.util.genai.types import ContentCapturingMode +from opentelemetry.util.genai.completion_hook import CompletionHook +from opentelemetry.util.genai.types import ( + ContentCapturingMode, + InputMessage, + OutputMessage, +) +from opentelemetry.util.genai.utils import gen_ai_json_dumps if TYPE_CHECKING: from google.cloud.aiplatform_v1.services.prediction_service import client @@ -110,6 +121,7 @@ def __init__( sem_conv_opt_in_mode: Literal[ _StabilityMode.GEN_AI_LATEST_EXPERIMENTAL ], + completion_hook: CompletionHook, ) -> None: ... @overload @@ -119,6 +131,7 @@ def __init__( event_logger: EventLogger, capture_content: bool, sem_conv_opt_in_mode: Literal[_StabilityMode.DEFAULT], + completion_hook: CompletionHook, ) -> None: ... def __init__( @@ -130,11 +143,13 @@ def __init__( Literal[_StabilityMode.DEFAULT], Literal[_StabilityMode.GEN_AI_LATEST_EXPERIMENTAL], ], + completion_hook: CompletionHook, ) -> None: self.tracer = tracer self.event_logger = event_logger self.capture_content = capture_content self.sem_conv_opt_in_mode = sem_conv_opt_in_mode + self.completion_hook = completion_hook @contextmanager def _with_new_instrumentation( @@ -146,18 +161,10 @@ def _with_new_instrumentation( kwargs: Any, ): params = _extract_params(*args, **kwargs) - api_endpoint: str = instance.api_endpoint # type: ignore[reportUnknownMemberType] - span_attributes = { - **get_genai_request_attributes(False, params), - **get_server_attributes(api_endpoint), - } - - span_name = get_span_name(span_attributes) - + request_attributes = get_genai_request_attributes(True, params) with self.tracer.start_as_current_span( - name=span_name, + name=f"{GenAI.GenAiOperationNameValues.CHAT.value} {request_attributes.get(GenAI.GEN_AI_REQUEST_MODEL, '')}".strip(), kind=SpanKind.CLIENT, - attributes=span_attributes, ) as span: def handle_response( @@ -165,21 +172,77 @@ def handle_response( | prediction_service_v1beta1.GenerateContentResponse | None, ) -> None: - if span.is_recording() and response: - # When streaming, this is called multiple times so attributes would be - # overwritten. In practice, it looks the API only returns the interesting - # attributes on the last streamed response. However, I couldn't find - # documentation for this and setting attributes shouldn't be too expensive. - span.set_attributes( - get_genai_response_attributes(response) - ) - self.event_logger.emit( - create_operation_details_event( - api_endpoint=api_endpoint, - params=params, - capture_content=capture_content, - response=response, + attributes = ( + get_server_attributes(instance.api_endpoint) # type: ignore[reportUnknownMemberType] + | request_attributes + | get_genai_response_attributes(response) + ) + system_instructions, inputs, outputs = [], [], [] + if params.system_instruction: + system_instructions = convert_content_to_message_parts( + params.system_instruction ) + if params.contents: + inputs = [ + InputMessage( + role=content.role, + parts=convert_content_to_message_parts(content), + ) + for content in params.contents + ] + if response: + outputs = [ + OutputMessage( + finish_reason=_map_finish_reason( + candidate.finish_reason + ), + role=candidate.content.role, + parts=convert_content_to_message_parts( + candidate.content + ), + ) + for candidate in response.candidates + ] + content_attributes = { + k: [asdict(x) for x in v] + for k, v in [ + ( + GenAI.GEN_AI_SYSTEM_INSTRUCTIONS, + system_instructions, + ), + (GenAI.GEN_AI_INPUT_MESSAGES, inputs), + (GenAI.GEN_AI_OUTPUT_MESSAGES, outputs), + ] + if v + } + if span.is_recording(): + span.set_attributes(attributes) + if capture_content in ( + ContentCapturingMode.SPAN_AND_EVENT, + ContentCapturingMode.SPAN_ONLY, + ): + span.set_attributes( + { + k: gen_ai_json_dumps(v) + for k, v in content_attributes.items() + } + ) + event = Event( + name="gen_ai.client.inference.operation.details", + ) + event.attributes = attributes + if capture_content in ( + ContentCapturingMode.SPAN_AND_EVENT, + ContentCapturingMode.EVENT_ONLY, + ): + event.attributes |= content_attributes + self.event_logger.emit(event) + self.completion_hook.on_completion( + inputs=inputs, + outputs=outputs, + system_instruction=system_instructions, + span=span, + log_record=event, ) yield handle_response diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py index c9c370794e..9a9dd9c2d4 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py @@ -17,7 +17,7 @@ from __future__ import annotations import re -from dataclasses import asdict, dataclass +from dataclasses import dataclass from os import environ from typing import ( TYPE_CHECKING, @@ -53,9 +53,7 @@ from opentelemetry.util.genai.types import ( ContentCapturingMode, FinishReason, - InputMessage, MessagePart, - OutputMessage, Text, ToolCall, ToolCallResponse, @@ -192,8 +190,11 @@ def get_genai_request_attributes( # pylint: disable=too-many-branches def get_genai_response_attributes( response: prediction_service.GenerateContentResponse - | prediction_service_v1beta1.GenerateContentResponse, + | prediction_service_v1beta1.GenerateContentResponse + | None, ) -> dict[str, AttributeValue]: + if not response: + return {} finish_reasons: list[str] = [ _map_finish_reason(candidate.finish_reason) for candidate in response.candidates @@ -307,68 +308,9 @@ def request_to_events( yield user_event(role=content.role, content=request_content) -def create_operation_details_event( - *, - api_endpoint: str, - response: prediction_service.GenerateContentResponse - | prediction_service_v1beta1.GenerateContentResponse - | None, - params: GenerateContentParams, - capture_content: ContentCapturingMode, -) -> Event: - event = Event(name="gen_ai.client.inference.operation.details") - attributes: dict[str, AnyValue] = { - **get_genai_request_attributes(True, params), - **get_server_attributes(api_endpoint), - **(get_genai_response_attributes(response) if response else {}), - } - event.attributes = attributes - if capture_content in { - ContentCapturingMode.NO_CONTENT, - ContentCapturingMode.SPAN_ONLY, - }: - return event - if params.system_instruction: - attributes[GenAIAttributes.GEN_AI_SYSTEM_INSTRUCTIONS] = [ - { - "type": "text", - "content": "\n".join( - part.text for part in params.system_instruction.parts - ), - } - ] - if params.contents: - attributes[GenAIAttributes.GEN_AI_INPUT_MESSAGES] = [ - asdict(_convert_content_to_message(content)) - for content in params.contents - ] - if response and response.candidates: - attributes[GenAIAttributes.GEN_AI_OUTPUT_MESSAGES] = [ - asdict(x) for x in _convert_response_to_output_messages(response) - ] - return event - - -def _convert_response_to_output_messages( - response: prediction_service.GenerateContentResponse - | prediction_service_v1beta1.GenerateContentResponse, -) -> list[OutputMessage]: - output_messages: list[OutputMessage] = [] - for candidate in response.candidates: - message = _convert_content_to_message(candidate.content) - output_messages.append( - OutputMessage( - finish_reason=_map_finish_reason(candidate.finish_reason), - role=message.role, - parts=message.parts, - ) - ) - return output_messages - - -def _convert_content_to_message( +def convert_content_to_message_parts( content: content.Content | content_v1beta1.Content, -) -> InputMessage: +) -> list[MessagePart]: parts: MessagePart = [] for idx, part in enumerate(content.parts): if "function_response" in part: @@ -398,7 +340,7 @@ def _convert_content_to_message( ) dict_part["type"] = type(part) parts.append(dict_part) - return InputMessage(role=content.role, parts=parts) + return parts def response_to_events( diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_tool_events_with_completion_hook.yaml b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_tool_events_with_completion_hook.yaml new file mode 100644 index 0000000000..20a8db0761 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_tool_events_with_completion_hook.yaml @@ -0,0 +1,151 @@ +interactions: +- request: + body: |- + { + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Get weather details in New Delhi and San Francisco?" + } + ] + }, + { + "role": "model", + "parts": [ + { + "functionCall": { + "name": "get_current_weather", + "args": { + "location": "New Delhi" + } + } + }, + { + "functionCall": { + "name": "get_current_weather", + "args": { + "location": "San Francisco" + } + } + } + ] + }, + { + "role": "user", + "parts": [ + { + "functionResponse": { + "name": "get_current_weather", + "response": { + "content": "{\"temperature\": 35, \"unit\": \"C\"}" + } + } + }, + { + "functionResponse": { + "name": "get_current_weather", + "response": { + "content": "{\"temperature\": 25, \"unit\": \"C\"}" + } + } + } + ] + } + ], + "tools": [ + { + "functionDeclarations": [ + { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": 6, + "properties": { + "location": { + "type": 1, + "description": "The location for which to get the weather. It can be a city name, a city name and state, or a zip code. Examples: 'San Francisco', 'San Francisco, CA', '95616', etc." + } + }, + "propertyOrdering": [ + "location" + ] + } + } + ] + } + ] + } + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '1731' + Content-Type: + - application/json + User-Agent: + - python-requests/2.32.3 + method: POST + uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint + response: + body: + string: |- + { + "candidates": [ + { + "content": { + "role": "model", + "parts": [ + { + "text": "The weather in New Delhi is 35\u00b0C and in San Francisco is 25\u00b0C.", + "thoughtSignature": "CqcOAcu98PDUiUq32HLxu6y5JNxvlcEjIaedPcBi5V86Hbf3vRgAXC4k0aMma0v1gotZVHrinF9edI9bEAdQdFR+2xOsaV1ntNeO4o35ymNNpm1rEv2p047eWxSABiXJ3VANecxNqQuVgxZyhClOn2BNmR/xGb43REabpcMzboVGVT6iKSJ/g3sCIt4bddY1IQ5zTdSV7lvCyZLuu9736VCJjULzAslhxSb9/xBlQu/pvcak6CmFFLZuDamqeJ3RvhalpklF6qz/Eq9nhhpdRPTERmGU7mNe4fiSpql6JxelO56ksKKGzSG+USa9vxcCQRRgAoaLlMHRegwPjfoA4hHgbpLBIUk9BNXrzmW5APOpJO3rqgIvKQ/WsZzIBI3l+slI51WHgVLr4qNvDUe8VKLhbApVj3L+rb5dQ/u6U9V2oSVA5pQFt6Lqubyjg8yCl0q+0dsvIYUqJSaDRONMfS5hwVu+RTU/SOEXLwTNEiDDuj3wO8xQuyitDAHqya797sOH2ThlSoT/c7V3Du00TBCk0eq8XaxyTobPMCaKNbrqjRTfMn5fortZlPwyAJqoTZsaSeaRufCFpgBMy34/QcLiCdVwKN6rLgBpDhtzThnyBzavDP2ltu8sxWEisJXEOel0q8LvBqTqlFY9dWrRScnB/TvitGnwqlV0PVpWU60wH/2Agexw0+aFlMRYxiFRjuHPK9R2UfLbEzVpDC8M+f16QRZOnWQc8L+cxEW/xHT6hCUxcTmSrpldAs2Ss8A21NcPTlr0lNviZxB+4xDq1dxB/VMS/qOnGVbr11nEOOVj+bWJyP/LLTjdUigGs+ISAzmEr2zkUirLdlmz/Mz1YXk9CMIus8WlPVzhQZuXY8dhSKxks0w8bJ5TjJelAwPGFgz6iWSz44IeaH22Fxdvub28/mdmvy3OerI3vfwlePUC9L8n2cg7gZjIE59PXi2x1EUTJfvcBrVTO5qqPuRpJZ9/d3ryEtS9tHsMnzfcEiS2lxNb+EqKaHk74hqvt6vEA+eQaYtMqG5TO79dpGDt0UD1cJEwngfMiL85+5qWHW4g2hSSJYgg55GtdZbxdGs960p8G02FbqUOSVcDGcIBDCkvFSGHJqMATvYQzChiwP2AnDVLN0aWd43UEl3rHESdOGLOqrxetcWmVsDacowhiy6jleS/Xml7d8mqqs2483XCOVMMYPcFh9budRed7PIf7eAYhWzGwt123JLbxqwBOBf1MHUJl2OQBy62HCJLYMlRhr1s2KVRK9Een5cgUa3H07r/s46YS2A0KOP/B0Ub0MflvGBAgo8LABj5cY2Ao1hTKcnBjgJPTNS0awETbtzU3Y74SPMsJ1Ipu2o76TFS3YFq1Fbit1VsceuYxzcqU04NFBtXAU7YojLv9LBOZXk6SukOWI/2cNUOHlzlWsSi34heVFGH4AcDbhc0/QZHhjSTE9FkBY9feTtSnTR16Q1bg3K9+7rxArD51Wj8q1ZNhVtZrzpuWGqoneI0WqJ25U5f1jSzQFW2HyzmIs/B5KiFEk4Do0EiaVcCQ5/1UGReYbgtSKa8PAvUGh2Lqiev1VMl6i4cDs7s3U9swvZdBdK+SauYC7UqEVov1G8UR8Bth2YncqUPSv7W2X7ppA6PwDXSCqCSgp3wnfEW8tQIa6AQW2s4Udp0lbYWr3Vu2DSK3iu8mVa3ybvHs5Go4SR9zJ8xElMn4adprz8jokYpoXK94DVeaUetKqTw6X3FzRb+K+e/W0MHZVEQtAjk2wwcNqAfs8JdXi2Mh/l+0PFp6Hhz0rI1rzZ3B08lVR+QU+0XpN5rerUmq44oE6t+dP6Cv8yl5kVBCjvtHZ9DLUGeSEofOR3JXs34rkWeOLfQ+vRZkYoiIRKkqFhf/RnRoU18vmfsbvK9C3XcO3wmfo58Iw2uYd9L/p6VdB27pPUiBJbTAzXbEA3CWUbedMg2vDZiucS4iBkWlmN91CofwgmNfiImdcauz5ug3TC5qRWUIqX/nNMSscPjpbhp3nmja727fyH1p4ytpXQbRX0do28KpY5XQ9yF9gAWxG2zHxtBGiZVG4iXRlpZlfmFUc4kQLUTrInYUoagh9zF6Cxwrz7smRqgOkWKOYiOTiR9ofTkvpmQpdGpDOsxjcPRd3V+jjkfZwT5yzsM7IOAzzhZKKvPWgfJfzaAI43W+dUWiMWzM5KX7Wliu5J3uEXtldo0lW2WpOiuFK5GIPBh5482IbSAtA14YEplOYVCrCI9UG356TImFYyCLuNKAxbzA3U9TSJCk0tUbssWKq8aIaoEbtvE43P1G8x5teVFR5Wu1nZypgKgYFSS5tRL+niDV82yaCOTIGjdA6yFHq8gRlv9xkmZjyPIB4LfAhAIEz9X0+hNvZBfTofi1UXRh/54GIqr1YIXmJVYKb/srAU5Qne5lQdvFauFIQbnlESVUeb1RA==" + } + ] + }, + "finishReason": 1, + "avgLogprobs": -1.18875130740079 + } + ], + "usageMetadata": { + "promptTokenCount": 128, + "candidatesTokenCount": 22, + "totalTokenCount": 575, + "trafficType": 1, + "promptTokensDetails": [ + { + "modality": 1, + "tokenCount": 128 + } + ], + "candidatesTokensDetails": [ + { + "modality": 1, + "tokenCount": 22 + } + ], + "thoughtsTokenCount": 425 + }, + "modelVersion": "gemini-2.5-pro", + "createTime": "2025-08-19T14:55:39.882212Z", + "responseId": "a5CkaKTsNa3hgLUP59n1oQo" + } + headers: + Content-Type: + - application/json; charset=UTF-8 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + content-length: + - '3277' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/conftest.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/conftest.py index c642eebcf4..349bf17820 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/conftest.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/conftest.py @@ -190,6 +190,35 @@ def instrument_with_experimental_semconvs( instrumentor.uninstrument() +@pytest.fixture(scope="function") +def instrument_with_upload_hook( + tracer_provider, event_logger_provider, meter_provider +): + # Reset global state.. + _OpenTelemetrySemanticConventionStability._initialized = False + os.environ.update( + { + OTEL_SEMCONV_STABILITY_OPT_IN: "gen_ai_latest_experimental", + "OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK": "upload", + "OTEL_INSTRUMENTATION_GENAI_UPLOAD_BASE_PATH": "memory://", + OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "SPAN_AND_EVENT", + } + ) + instrumentor = VertexAIInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + event_logger_provider=event_logger_provider, + meter_provider=meter_provider, + ) + + yield instrumentor + os.environ.pop(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, None) + os.environ.pop("OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK", None) + os.environ.pop("OTEL_INSTRUMENTATION_GENAI_UPLOAD_BASE_PATH", None) + if instrumentor.is_instrumented_by_opentelemetry: + instrumentor.uninstrument() + + @pytest.fixture(scope="function") def instrument_with_content( tracer_provider, event_logger_provider, meter_provider, request diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/requirements.latest.txt b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/requirements.latest.txt index 1497f0fc3a..6098baf83c 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/requirements.latest.txt +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/requirements.latest.txt @@ -78,6 +78,7 @@ PyYAML==6.0.2 requests==2.32.3 rsa==4.9 shapely==2.0.6 +fsspec==2025.9.0 six==1.17.0 tomli==2.2.1 typing_extensions==4.12.2 diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/requirements.oldest.txt b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/requirements.oldest.txt index b38d74c562..fd9e803fb9 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/requirements.oldest.txt +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/requirements.oldest.txt @@ -69,7 +69,7 @@ opentelemetry-api==1.37 opentelemetry-sdk==1.37 opentelemetry-semantic-conventions==0.58b0 opentelemetry-instrumentation==0.58b0 - +fsspec==2025.9.0 -e instrumentation-genai/opentelemetry-instrumentation-vertexai[instruments] -e util/opentelemetry-util-genai diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py index f1bc28d5f6..85a0d1cee9 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py @@ -52,11 +52,12 @@ def test_generate_content( "gen_ai.request.model": "gemini-2.5-pro", "gen_ai.response.finish_reasons": ("stop",), "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.system": "vertex_ai", "gen_ai.usage.input_tokens": 5, "gen_ai.usage.output_tokens": 5, "server.address": "us-central1-aiplatform.googleapis.com", "server.port": 443, + "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"}]}]', + "gen_ai.output.messages": '[{"role":"model","parts":[{"content":"This is a test.","type":"text"}],"finish_reason":"stop"}]', } logs = log_exporter.get_finished_logs() @@ -108,10 +109,11 @@ def test_generate_content_without_events( assert spans[0].name == "chat gemini-2.5-pro" assert dict(spans[0].attributes) == { "gen_ai.operation.name": "chat", + "gen_ai.output.messages": '[{"role":"model","parts":[{"content":"This is a test.","type":"text"}],"finish_reason":"stop"}]', + "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"}]}]', "gen_ai.request.model": "gemini-2.5-pro", "gen_ai.response.finish_reasons": ("stop",), "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.system": "vertex_ai", "gen_ai.usage.input_tokens": 5, "gen_ai.usage.output_tokens": 5, "server.address": "us-central1-aiplatform.googleapis.com", @@ -173,9 +175,9 @@ def test_generate_content_empty_model( assert dict(spans[0].attributes) == { "gen_ai.operation.name": "chat", "gen_ai.request.model": "", - "gen_ai.system": "vertex_ai", "server.address": "us-central1-aiplatform.googleapis.com", "server.port": 443, + "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"}]}]', } assert_span_error(spans[0]) @@ -206,9 +208,9 @@ def test_generate_content_missing_model( assert dict(spans[0].attributes) == { "gen_ai.operation.name": "chat", "gen_ai.request.model": "gemini-does-not-exist", - "gen_ai.system": "vertex_ai", "server.address": "us-central1-aiplatform.googleapis.com", "server.port": 443, + "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"}]}]', } assert_span_error(spans[0]) @@ -241,9 +243,9 @@ def test_generate_content_invalid_temperature( "gen_ai.operation.name": "chat", "gen_ai.request.model": "gemini-2.5-pro", "gen_ai.request.temperature": 1000.0, - "gen_ai.system": "vertex_ai", "server.address": "us-central1-aiplatform.googleapis.com", "server.port": 443, + "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"}]}]', } assert_span_error(spans[0]) @@ -325,7 +327,7 @@ def test_generate_content_extra_params( "gen_ai.request.top_p": 0.949999988079071, "gen_ai.response.finish_reasons": ("length",), "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.system": "vertex_ai", + "gen_ai.request.seed": 12345, "gen_ai.usage.input_tokens": 5, "gen_ai.usage.output_tokens": 0, "server.address": "us-central1-aiplatform.googleapis.com", diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py index a7b61a09c7..f0bb1307e3 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py @@ -1,3 +1,8 @@ +import json +import time +from typing import Any + +import fsspec import pytest from opentelemetry.instrumentation.vertexai import VertexAIInstrumentor @@ -13,7 +18,6 @@ ) -@pytest.mark.vcr() def test_function_call_choice( span_exporter: InMemorySpanExporter, log_exporter: InMemoryLogExporter, @@ -31,10 +35,11 @@ def test_function_call_choice( "gen_ai.request.model": "gemini-2.5-pro", "gen_ai.response.finish_reasons": ("stop",), "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.system": "vertex_ai", "gen_ai.usage.input_tokens": 74, "gen_ai.usage.output_tokens": 16, "server.address": "us-central1-aiplatform.googleapis.com", + "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Get weather details in New Delhi and San Francisco?","type":"text"}]}]', + "gen_ai.output.messages": '[{"role":"model","parts":[{"arguments":{"location":"New Delhi"},"name":"get_current_weather","id":"get_current_weather_0","type":"tool_call"},{"arguments":{"location":"San Francisco"},"name":"get_current_weather","id":"get_current_weather_1","type":"tool_call"}],"finish_reason":"stop"}]', "server.port": 443, } @@ -127,11 +132,12 @@ def test_tool_events( "gen_ai.request.model": "gemini-2.5-pro", "gen_ai.response.finish_reasons": ("stop",), "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.system": "vertex_ai", "gen_ai.usage.input_tokens": 128, "gen_ai.usage.output_tokens": 26, "server.address": "us-central1-aiplatform.googleapis.com", "server.port": 443, + "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Get weather details in New Delhi and San Francisco?","type":"text"}]},{"role":"model","parts":[{"arguments":{"location":"New Delhi"},"name":"get_current_weather","id":"get_current_weather_0","type":"tool_call"},{"arguments":{"location":"San Francisco"},"name":"get_current_weather","id":"get_current_weather_1","type":"tool_call"}]},{"role":"user","parts":[{"response":{"content":"{\\"temperature\\": 35, \\"unit\\": \\"C\\"}"},"id":"get_current_weather_0","type":"tool_call_response"},{"response":{"content":"{\\"temperature\\": 25, \\"unit\\": \\"C\\"}"},"id":"get_current_weather_1","type":"tool_call_response"}]}]', + "gen_ai.output.messages": '[{"role":"model","parts":[{"content":"The current temperature in New Delhi is 35\\u00b0C, and in San Francisco, it is 25\\u00b0C.","type":"text"}],"finish_reason":"stop"}]', } logs = log_exporter.get_finished_logs() assert len(logs) == 1 @@ -225,7 +231,6 @@ def test_tool_events_no_content( "gen_ai.request.model": "gemini-2.5-pro", "gen_ai.response.finish_reasons": ("stop",), "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.system": "vertex_ai", "gen_ai.usage.input_tokens": 128, "gen_ai.usage.output_tokens": 22, "server.address": "us-central1-aiplatform.googleapis.com", @@ -244,3 +249,93 @@ def test_tool_events_no_content( "gen_ai.usage.input_tokens": 128, "gen_ai.usage.output_tokens": 22, } + + +def assert_fsspec_equal(path: str, value: Any) -> None: + # Hide this function and its calls from traceback. + __tracebackhide__ = True # pylint: disable=unused-variable + with fsspec.open(path, "r") as file: + assert json.load(file) == value + + +@pytest.mark.vcr() +def test_tool_events_with_completion_hook( + span_exporter: InMemorySpanExporter, + log_exporter: InMemoryLogExporter, + instrument_with_upload_hook: VertexAIInstrumentor, + generate_content: callable, +): + ask_about_weather_function_response(generate_content) + + # Emits span + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + logs = log_exporter.get_finished_logs() + assert len(logs) == 1 + # File upload takes a few seconds sometimes. + time.sleep(3) + assert_fsspec_equal( + spans[0].attributes["gen_ai.output.messages_ref"], + [ + { + "role": "model", + "parts": [ + { + "content": "The weather in New Delhi is 35°C and in San Francisco is 25°C.", + "type": "text", + } + ], + "finish_reason": "stop", + } + ], + ) + assert_fsspec_equal( + spans[0].attributes["gen_ai.input.messages_ref"], + [ + { + "parts": [ + { + "content": "Get weather details in New Delhi and San Francisco?", + "type": "text", + } + ], + "role": "user", + }, + { + "parts": [ + { + "arguments": {"location": "New Delhi"}, + "id": "get_current_weather_0", + "name": "get_current_weather", + "type": "tool_call", + }, + { + "arguments": {"location": "San Francisco"}, + "id": "get_current_weather_1", + "name": "get_current_weather", + "type": "tool_call", + }, + ], + "role": "model", + }, + { + "parts": [ + { + "id": "get_current_weather_0", + "response": { + "content": '{"temperature": 35, "unit": "C"}' + }, + "type": "tool_call_response", + }, + { + "id": "get_current_weather_1", + "response": { + "content": '{"temperature": 25, "unit": "C"}' + }, + "type": "tool_call_response", + }, + ], + "role": "user", + }, + ], + ) diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_upload/completion_hook.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_upload/completion_hook.py index 351b74cc3b..86cb4f0c51 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_upload/completion_hook.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_upload/completion_hook.py @@ -15,11 +15,9 @@ from __future__ import annotations -import json import logging import posixpath import threading -from base64 import b64encode from concurrent.futures import Future, ThreadPoolExecutor from contextlib import ExitStack from dataclasses import asdict, dataclass @@ -39,6 +37,7 @@ from opentelemetry.util.genai.environment_variables import ( OTEL_INSTRUMENTATION_GENAI_UPLOAD_FORMAT, ) +from opentelemetry.util.genai.utils import gen_ai_json_dump GEN_AI_INPUT_MESSAGES_REF: Final = ( gen_ai_attributes.GEN_AI_INPUT_MESSAGES + "_ref" @@ -192,12 +191,7 @@ def _do_upload( with self._fs.open(path, "w", content_type=content_type) as file: for message in message_lines: - json.dump( - message, - file, - separators=(",", ":"), - cls=Base64JsonEncoder, - ) + gen_ai_json_dump(message, file) file.write("\n") def on_completion( @@ -281,10 +275,3 @@ def shutdown(self, *, timeout_sec: float = 10.0) -> None: # Queue is flushed and blocked, start shutdown self._executor.shutdown(wait=False) - - -class Base64JsonEncoder(json.JSONEncoder): - def default(self, o: Any) -> Any: - if isinstance(o, bytes): - return b64encode(o).decode() - return super().default(o) diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py index 91cb9221f1..0083d5144c 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py @@ -12,8 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import logging import os +from base64 import b64encode +from functools import partial +from typing import Any from opentelemetry.instrumentation._semconv import ( _OpenTelemetrySemanticConventionStability, @@ -54,3 +58,23 @@ def get_content_capturing_mode() -> ContentCapturingMode: ", ".join(e.name for e in ContentCapturingMode), ) return ContentCapturingMode.NO_CONTENT + + +class _GenAiJsonEncoder(json.JSONEncoder): + def default(self, o: Any) -> Any: + if isinstance(o, bytes): + return b64encode(o).decode() + return super().default(o) + + +gen_ai_json_dump = partial( + json.dump, separators=(",", ":"), cls=_GenAiJsonEncoder +) +"""Should be used by GenAI instrumentations when serializing objects that may contain +bytes, datetimes, etc. for GenAI observability.""" + +gen_ai_json_dumps = partial( + json.dumps, separators=(",", ":"), cls=_GenAiJsonEncoder +) +"""Should be used by GenAI instrumentations when serializing objects that may contain +bytes, datetimes, etc. for GenAI observability."""