From aca3ac2aff5adc2d585ef4caaaf274204038ad2a Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Mon, 13 Oct 2025 17:52:30 +0000 Subject: [PATCH 1/3] Initial commit --- .../instrumentation/vertexai/utils.py | 36 ++++++- .../test_generate_content_with_files.yaml | 102 ++++++++++++++++++ .../test_chat_completions_experimental.py | 38 +++++-- 3 files changed, 163 insertions(+), 13 deletions(-) create mode 100644 instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_with_files.yaml diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py index 860018f908..da590851f4 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py @@ -16,6 +16,7 @@ from __future__ import annotations +import logging import re from dataclasses import dataclass from os import environ @@ -308,6 +309,23 @@ def request_to_events( yield user_event(role=content.role, content=request_content) +@dataclass +class BlobPart: + data: bytes + mime_type: str + type: Literal["blob"] = "blob" + + +@dataclass +class FileDataPart: + mime_type: str + uri: str + type: Literal["file_data"] = "file_data" + + class Config: + extra = "allow" + + def convert_content_to_message_parts( content: content.Content | content_v1beta1.Content, ) -> list[MessagePart]: @@ -334,12 +352,20 @@ def convert_content_to_message_parts( ) elif "text" in part: parts.append(Text(content=part.text)) - else: - dict_part = type(part).to_dict( # type: ignore[reportUnknownMemberType] - part, always_print_fields_with_no_presence=False + elif "inline_data" in part: + part = part.inline_data + parts.append( + BlobPart(mime_type=part.mime_type or "", data=part.data or b"") + ) + elif "file_data" in part: + part = part.file_data + parts.append( + FileDataPart( + mime_type=part.mime_type or "", uri=part.file_uri or "" + ) ) - dict_part["type"] = type(part) - parts.append(dict_part) + else: + logging.warning("Unknown part dropped from telemetry %s", part) return parts diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_with_files.yaml b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_with_files.yaml new file mode 100644 index 0000000000..98c6fe9729 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_with_files.yaml @@ -0,0 +1,102 @@ +interactions: +- request: + body: |- + { + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Say this is a test" + }, + { + "fileData": { + "mimeType": "image/jpeg", + "fileUri": "gs://a-test-testing-testboy/app/2021/12/10/download.jpeg" + } + }, + { + "inlineData": { + "mimeType": "image/jpeg", + "data": "iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==" + } + } + ] + } + ] + } + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '554' + Content-Type: + - application/json + User-Agent: + - python-requests/2.32.3 + method: POST + uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint + response: + body: + string: |- + { + "candidates": [ + { + "content": { + "role": "model", + "parts": [ + { + "text": "This is a test." + } + ] + }, + "finishReason": 1, + "avgLogprobs": -24.462081909179688 + } + ], + "usageMetadata": { + "promptTokenCount": 521, + "candidatesTokenCount": 5, + "totalTokenCount": 950, + "trafficType": 1, + "promptTokensDetails": [ + { + "modality": 2, + "tokenCount": 516 + }, + { + "modality": 1, + "tokenCount": 5 + } + ], + "candidatesTokensDetails": [ + { + "modality": 1, + "tokenCount": 5 + } + ], + "thoughtsTokenCount": 424 + }, + "modelVersion": "gemini-2.5-pro", + "createTime": "2025-10-13T16:29:47.639271Z", + "responseId": "-yjtaKeCJ5KYmecP76S4-AI" + } + headers: + Content-Type: + - application/json; charset=UTF-8 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + content-length: + - '808' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py index 85a0d1cee9..0c655a17a4 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py @@ -6,6 +6,7 @@ Content, GenerationConfig, GenerativeModel, + Image, Part, ) from vertexai.preview.generative_models import ( @@ -24,7 +25,7 @@ @pytest.mark.vcr() -def test_generate_content( +def test_generate_content_with_files( span_exporter: InMemorySpanExporter, log_exporter: InMemoryLogExporter, generate_content: callable, @@ -38,6 +39,15 @@ def test_generate_content( role="user", parts=[ Part.from_text("Say this is a test"), + Part.from_uri( + mime_type="image/jpeg", + uri="gs://a-test-testing-testboy/app/2021/12/10/download.jpeg", + ), + Part.from_image( + Image.from_bytes( + "iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==" + ) + ), ], ), ], @@ -52,11 +62,11 @@ def test_generate_content( "gen_ai.request.model": "gemini-2.5-pro", "gen_ai.response.finish_reasons": ("stop",), "gen_ai.response.model": "gemini-2.5-pro", - "gen_ai.usage.input_tokens": 5, + "gen_ai.usage.input_tokens": 521, "gen_ai.usage.output_tokens": 5, "server.address": "us-central1-aiplatform.googleapis.com", "server.port": 443, - "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"}]}]', + "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"},{"mime_type":"image/jpeg","uri":"gs://a-test-testing-testboy/app/2021/12/10/download.jpeg","type":"file_data"},{"data":"iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==","mime_type":"image/jpeg","type":"blob"}]}]', "gen_ai.output.messages": '[{"role":"model","parts":[{"content":"This is a test.","type":"text"}],"finish_reason":"stop"}]', } @@ -64,24 +74,36 @@ def test_generate_content( assert len(logs) == 1 log = logs[0].log_record assert log.attributes == { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "gemini-2.5-pro", "server.address": "us-central1-aiplatform.googleapis.com", "server.port": 443, + "gen_ai.operation.name": "chat", + "gen_ai.request.model": "gemini-2.5-pro", "gen_ai.response.model": "gemini-2.5-pro", "gen_ai.response.finish_reasons": ("stop",), - "gen_ai.usage.input_tokens": 5, + "gen_ai.usage.input_tokens": 521, "gen_ai.usage.output_tokens": 5, "gen_ai.input.messages": ( { "role": "user", - "parts": ({"type": "text", "content": "Say this is a test"},), + "parts": ( + {"content": "Say this is a test", "type": "text"}, + { + "mime_type": "image/jpeg", + "uri": "gs://a-test-testing-testboy/app/2021/12/10/download.jpeg", + "type": "file_data", + }, + { + "data": b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x05\x00\x00\x00\x05\x08\x06\x00\x00\x00\x8do&\xe5\x00\x00\x00\x1cIDAT\x08\xd7c\xf8\xff\xff?\xc3\x7f\x06 \x05\xc3 \x12\x84\xd01\xf1\x82X\xcd\x04\x00\x0e\xf55\xcb\xd1\x8e\x0e\x1f\x00\x00\x00\x00IEND\xaeB`\x82", + "mime_type": "image/jpeg", + "type": "blob", + }, + ), }, ), "gen_ai.output.messages": ( { "role": "model", - "parts": ({"type": "text", "content": "This is a test."},), + "parts": ({"content": "This is a test.", "type": "text"},), "finish_reason": "stop", }, ), From 992dd0cd724e490528ea3515e1d0500cbbdebb09 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Mon, 13 Oct 2025 17:58:04 +0000 Subject: [PATCH 2/3] Update changelog --- .../opentelemetry-instrumentation-vertexai/CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md b/instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md index cba7e190fa..bc409ea347 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md @@ -14,8 +14,10 @@ users will need to set the environment variable OTEL_SEMCONV_STABILITY_OPT_IN to ([#3328](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3328)) - VertexAI support for async calling ([#3386](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3386)) - - `opentelemetry-instrumentation-vertexai`: migrate off the deprecated events API to use the logs API + - Migrate off the deprecated events API to use the logs API ([#3625](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3626)) + - Update `gen_ai_latest_experimental` instrumentation to record files being passed to the model + ([#3840](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3840)). ## Version 2.0b0 (2025-02-24) From 677d4e1eabd451020e532dee46a0e0330d653f77 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Tue, 14 Oct 2025 13:31:22 +0000 Subject: [PATCH 3/3] Switch to public URI --- .../tests/cassettes/test_generate_content_with_files.yaml | 2 +- .../tests/test_chat_completions_experimental.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_with_files.yaml b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_with_files.yaml index 98c6fe9729..613f27ad25 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_with_files.yaml +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_with_files.yaml @@ -12,7 +12,7 @@ interactions: { "fileData": { "mimeType": "image/jpeg", - "fileUri": "gs://a-test-testing-testboy/app/2021/12/10/download.jpeg" + "fileUri": "https://images.pdimagearchive.org/collections/microscopic-delights/1lede-0021.jpg" } }, { diff --git a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py index 0c655a17a4..fa37f77d0b 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py +++ b/instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py @@ -41,7 +41,7 @@ def test_generate_content_with_files( Part.from_text("Say this is a test"), Part.from_uri( mime_type="image/jpeg", - uri="gs://a-test-testing-testboy/app/2021/12/10/download.jpeg", + uri="https://images.pdimagearchive.org/collections/microscopic-delights/1lede-0021.jpg", ), Part.from_image( Image.from_bytes( @@ -66,7 +66,7 @@ def test_generate_content_with_files( "gen_ai.usage.output_tokens": 5, "server.address": "us-central1-aiplatform.googleapis.com", "server.port": 443, - "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"},{"mime_type":"image/jpeg","uri":"gs://a-test-testing-testboy/app/2021/12/10/download.jpeg","type":"file_data"},{"data":"iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==","mime_type":"image/jpeg","type":"blob"}]}]', + "gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"},{"mime_type":"image/jpeg","uri":"https://images.pdimagearchive.org/collections/microscopic-delights/1lede-0021.jpg","type":"file_data"},{"data":"iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==","mime_type":"image/jpeg","type":"blob"}]}]', "gen_ai.output.messages": '[{"role":"model","parts":[{"content":"This is a test.","type":"text"}],"finish_reason":"stop"}]', } @@ -89,7 +89,7 @@ def test_generate_content_with_files( {"content": "Say this is a test", "type": "text"}, { "mime_type": "image/jpeg", - "uri": "gs://a-test-testing-testboy/app/2021/12/10/download.jpeg", + "uri": "https://images.pdimagearchive.org/collections/microscopic-delights/1lede-0021.jpg", "type": "file_data", }, {