Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,10 @@ users will need to set the environment variable OTEL_SEMCONV_STABILITY_OPT_IN to
([#3328](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3328))
- VertexAI support for async calling
([#3386](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3386))
- `opentelemetry-instrumentation-vertexai`: migrate off the deprecated events API to use the logs API
- Migrate off the deprecated events API to use the logs API
([#3625](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3626))
- Update `gen_ai_latest_experimental` instrumentation to record files being passed to the model
([#3840](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3840)).

## Version 2.0b0 (2025-02-24)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

from __future__ import annotations

import logging
import re
from dataclasses import dataclass
from os import environ
Expand Down Expand Up @@ -308,6 +309,23 @@ def request_to_events(
yield user_event(role=content.role, content=request_content)


@dataclass
class BlobPart:
data: bytes
mime_type: str
type: Literal["blob"] = "blob"


@dataclass
class FileDataPart:
mime_type: str
uri: str
type: Literal["file_data"] = "file_data"

class Config:
extra = "allow"


def convert_content_to_message_parts(
content: content.Content | content_v1beta1.Content,
) -> list[MessagePart]:
Expand All @@ -334,12 +352,20 @@ def convert_content_to_message_parts(
)
elif "text" in part:
parts.append(Text(content=part.text))
else:
dict_part = type(part).to_dict( # type: ignore[reportUnknownMemberType]
part, always_print_fields_with_no_presence=False
elif "inline_data" in part:
part = part.inline_data
parts.append(
BlobPart(mime_type=part.mime_type or "", data=part.data or b"")
)
elif "file_data" in part:
part = part.file_data
parts.append(
FileDataPart(
mime_type=part.mime_type or "", uri=part.file_uri or ""
)
)
dict_part["type"] = type(part)
parts.append(dict_part)
else:
logging.warning("Unknown part dropped from telemetry %s", part)
return parts


Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
interactions:
- request:
body: |-
{
"contents": [
{
"role": "user",
"parts": [
{
"text": "Say this is a test"
},
{
"fileData": {
"mimeType": "image/jpeg",
"fileUri": "gs://a-test-testing-testboy/app/2021/12/10/download.jpeg"
}
},
{
"inlineData": {
"mimeType": "image/jpeg",
"data": "iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg=="
}
}
]
}
]
}
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Length:
- '554'
Content-Type:
- application/json
User-Agent:
- python-requests/2.32.3
method: POST
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint
response:
body:
string: |-
{
"candidates": [
{
"content": {
"role": "model",
"parts": [
{
"text": "This is a test."
}
]
},
"finishReason": 1,
"avgLogprobs": -24.462081909179688
}
],
"usageMetadata": {
"promptTokenCount": 521,
"candidatesTokenCount": 5,
"totalTokenCount": 950,
"trafficType": 1,
"promptTokensDetails": [
{
"modality": 2,
"tokenCount": 516
},
{
"modality": 1,
"tokenCount": 5
}
],
"candidatesTokensDetails": [
{
"modality": 1,
"tokenCount": 5
}
],
"thoughtsTokenCount": 424
},
"modelVersion": "gemini-2.5-pro",
"createTime": "2025-10-13T16:29:47.639271Z",
"responseId": "-yjtaKeCJ5KYmecP76S4-AI"
}
headers:
Content-Type:
- application/json; charset=UTF-8
Transfer-Encoding:
- chunked
Vary:
- Origin
- X-Origin
- Referer
content-length:
- '808'
status:
code: 200
message: OK
version: 1
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
Content,
GenerationConfig,
GenerativeModel,
Image,
Part,
)
from vertexai.preview.generative_models import (
Expand All @@ -24,7 +25,7 @@


@pytest.mark.vcr()
def test_generate_content(
def test_generate_content_with_files(
span_exporter: InMemorySpanExporter,
log_exporter: InMemoryLogExporter,
generate_content: callable,
Expand All @@ -38,6 +39,15 @@ def test_generate_content(
role="user",
parts=[
Part.from_text("Say this is a test"),
Part.from_uri(
mime_type="image/jpeg",
uri="gs://a-test-testing-testboy/app/2021/12/10/download.jpeg",
),
Part.from_image(
Image.from_bytes(
"iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg=="
)
),
],
),
],
Expand All @@ -52,36 +62,48 @@ def test_generate_content(
"gen_ai.request.model": "gemini-2.5-pro",
"gen_ai.response.finish_reasons": ("stop",),
"gen_ai.response.model": "gemini-2.5-pro",
"gen_ai.usage.input_tokens": 5,
"gen_ai.usage.input_tokens": 521,
"gen_ai.usage.output_tokens": 5,
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
"gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"}]}]',
"gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"},{"mime_type":"image/jpeg","uri":"gs://a-test-testing-testboy/app/2021/12/10/download.jpeg","type":"file_data"},{"data":"iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==","mime_type":"image/jpeg","type":"blob"}]}]',
"gen_ai.output.messages": '[{"role":"model","parts":[{"content":"This is a test.","type":"text"}],"finish_reason":"stop"}]',
}

logs = log_exporter.get_finished_logs()
assert len(logs) == 1
log = logs[0].log_record
assert log.attributes == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-2.5-pro",
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-2.5-pro",
"gen_ai.response.model": "gemini-2.5-pro",
"gen_ai.response.finish_reasons": ("stop",),
"gen_ai.usage.input_tokens": 5,
"gen_ai.usage.input_tokens": 521,
"gen_ai.usage.output_tokens": 5,
"gen_ai.input.messages": (
{
"role": "user",
"parts": ({"type": "text", "content": "Say this is a test"},),
"parts": (
{"content": "Say this is a test", "type": "text"},
{
"mime_type": "image/jpeg",
"uri": "gs://a-test-testing-testboy/app/2021/12/10/download.jpeg",
"type": "file_data",
},
{
"data": b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x05\x00\x00\x00\x05\x08\x06\x00\x00\x00\x8do&\xe5\x00\x00\x00\x1cIDAT\x08\xd7c\xf8\xff\xff?\xc3\x7f\x06 \x05\xc3 \x12\x84\xd01\xf1\x82X\xcd\x04\x00\x0e\xf55\xcb\xd1\x8e\x0e\x1f\x00\x00\x00\x00IEND\xaeB`\x82",
"mime_type": "image/jpeg",
"type": "blob",
},
),
},
),
"gen_ai.output.messages": (
{
"role": "model",
"parts": ({"type": "text", "content": "This is a test."},),
"parts": ({"content": "This is a test.", "type": "text"},),
"finish_reason": "stop",
},
),
Expand Down