VertexAI emit user events

Author: aabmass

instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/events.py

@@ -0,0 +1,69 @@
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Factories for event types described in
+https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-events.md#system-event.
+
+Hopefully this code can be autogenerated by Weaver once Gen AI semantic conventions are
+schematized in YAML and the Weaver tool supports it.
+"""
+
+from opentelemetry._events import Event
+from opentelemetry.semconv._incubating.attributes import gen_ai_attributes
+from opentelemetry.util.types import AnyValue
+
+
+def user_event(
+    *,
+    role: str = "user",
+    content: AnyValue = None,
+) -> Event:
+    """Creates a User event
+    https://github.com/open-telemetry/semantic-conventions/blob/v1.28.0/docs/gen-ai/gen-ai-events.md#user-event
+    """
+    body: dict[str, AnyValue] = {
+        "role": role,
+    }
+    if content is not None:
+        body["content"] = content
+    return Event(
+        name="gen_ai.user.message",
+        attributes={
+            gen_ai_attributes.GEN_AI_SYSTEM: gen_ai_attributes.GenAiSystemValues.VERTEX_AI.value,
+        },
+        body=body,
+    )
+
+
+def assistant_event(
+    *,
+    role: str = "model",
+    content: AnyValue = None,
+) -> Event:
+    """Creates an Assistant event
+    https://github.com/open-telemetry/semantic-conventions/blob/v1.28.0/docs/gen-ai/gen-ai-events.md#assistant-event
+    """
+    # TODO: add tool_calls once instrumentation supports it
+
+    return Event(
+        name="gen_ai.assistant.message",
+        attributes={
+            gen_ai_attributes.GEN_AI_SYSTEM: gen_ai_attributes.GenAiSystemValues.VERTEX_AI.value,
+        },
+        body={
+            "role": role,
+            "content": content,
+        },
+    )

instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py

@@ -26,6 +26,7 @@
     GenerateContentParams,
     get_genai_request_attributes,
     get_span_name,
+    request_to_events,
 )
 from opentelemetry.trace import SpanKind, Tracer
 
@@ -107,13 +108,12 @@ def traced_method(
             name=span_name,
             kind=SpanKind.CLIENT,
             attributes=span_attributes,
-        ) as _span:
-            # TODO: emit request events
-            # if span.is_recording():
-            #     for message in kwargs.get("messages", []):
-            #         event_logger.emit(
-            #             message_to_event(message, capture_content)
-            #         )
+        ) as span:
+            if span.is_recording():
+                for event in request_to_events(
+                    params=params, capture_content=capture_content
+                ):
+                    event_logger.emit(event)
 
             # TODO: set error.type attribute
             # https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-spans.md

instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py

@@ -16,17 +16,22 @@
 
 import re
 from dataclasses import dataclass
+from enum import Enum
 from os import environ
 from typing import (
     TYPE_CHECKING,
+    Iterable,
     Mapping,
     Sequence,
+    cast,
 )
 
+from opentelemetry._events import Event
+from opentelemetry.instrumentation.vertexai.events import user_event
 from opentelemetry.semconv._incubating.attributes import (
     gen_ai_attributes as GenAIAttributes,
 )
-from opentelemetry.util.types import AttributeValue
+from opentelemetry.util.types import AnyValue, AttributeValue
 
 if TYPE_CHECKING:
     from google.cloud.aiplatform_v1.types import content, tool
@@ -137,3 +142,21 @@ def get_span_name(span_attributes: Mapping[str, AttributeValue]) -> str:
     if not model:
         return f"{name}"
     return f"{name} {model}"
+
+
+def request_to_events(
+    *, params: GenerateContentParams, capture_content: bool
+) -> Iterable[Event]:
+    for content in params.contents or []:
+        if content.role == "model":
+            # TODO: handle assistant message
+            pass
+        # Assume user event but role should be "user"
+        else:
+            request_content = None
+            if capture_content:
+                request_content = [
+                    cast(dict[str, AnyValue], type(part).to_dict(part))  # type: ignore[reportUnknownMemberType]
+                    for part in content.parts
+                ]
+            yield user_event(role=content.role, content=request_content)

instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_invalid_temperature.yaml

@@ -4,17 +4,14 @@ interactions:
       {
         "contents": [
           {
-            "role": "user",
+            "role": "my_invalid_role",
             "parts": [
               {
                 "text": "Say this is a test"
               }
             ]
           }
-        ],
-        "generationConfig": {
-          "temperature": 1000.0
-        }
+        ]
       }
     headers:
       Accept:
@@ -24,7 +21,7 @@ interactions:
       Connection:
       - keep-alive
       Content-Length:
-      - '196'
+      - '152'
       Content-Type:
       - application/json
       User-Agent:
@@ -37,7 +34,7 @@ interactions:
         {
           "error": {
             "code": 400,
-            "message": "Unable to submit request because it has a temperature value of 1000 but the supported range is from 0 (inclusive) to 2.0001 (exclusive). Update the value and try again.",
+            "message": "Please use a valid role: user, model.",
             "status": "INVALID_ARGUMENT",
             "details": []
           }
@@ -52,7 +49,7 @@ interactions:
       - X-Origin
       - Referer
       content-length:
-      - '809'
+      - '416'
     status:
       code: 400
       message: Bad Request

instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions.py

@@ -8,6 +8,9 @@
 )
 
 from opentelemetry.instrumentation.vertexai import VertexAIInstrumentor
+from opentelemetry.sdk._logs._internal.export.in_memory_log_exporter import (
+    InMemoryLogExporter,
+)
 from opentelemetry.sdk.trace import ReadableSpan
 from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
     InMemorySpanExporter,
@@ -18,6 +21,7 @@
 @pytest.mark.vcr
 def test_generate_content(
     span_exporter: InMemorySpanExporter,
+    log_exporter: InMemoryLogExporter,
     instrument_with_content: VertexAIInstrumentor,
 ):
     model = GenerativeModel("gemini-1.5-flash-002")
@@ -27,6 +31,7 @@ def test_generate_content(
         ]
     )
 
+    # Emits span
     spans = span_exporter.get_finished_spans()
     assert len(spans) == 1
     assert spans[0].name == "chat gemini-1.5-flash-002"
@@ -36,6 +41,23 @@ def test_generate_content(
         "gen_ai.system": "vertex_ai",
     }
 
+    # Emits content event
+    logs = log_exporter.get_finished_logs()
+    assert len(logs) == 1
+    log_record = logs[0].log_record
+    span_context = spans[0].get_span_context()
+    assert log_record.trace_id == span_context.trace_id
+    assert log_record.span_id == span_context.span_id
+    assert log_record.trace_flags == span_context.trace_flags
+    assert log_record.attributes == {
+        "gen_ai.system": "vertex_ai",
+        "event.name": "gen_ai.user.message",
+    }
+    assert log_record.body == {
+        "content": [{"text": "Say this is a test"}],
+        "role": "user",
+    }
+
 
 @pytest.mark.vcr
 def test_generate_content_empty_model(
@@ -98,18 +120,19 @@ def test_generate_content_missing_model(
 @pytest.mark.vcr
 def test_generate_content_invalid_temperature(
     span_exporter: InMemorySpanExporter,
+    log_exporter: InMemoryLogExporter,
     instrument_with_content: VertexAIInstrumentor,
 ):
     model = GenerativeModel("gemini-1.5-flash-002")
     try:
-        # Temperature out of range causes error
+        # Fails because role must be "user" or "model"
         model.generate_content(
             [
                 Content(
-                    role="user", parts=[Part.from_text("Say this is a test")]
+                    role="my_invalid_role",
+                    parts=[Part.from_text("Say this is a test")],
                 )
             ],
-            generation_config=GenerationConfig(temperature=1000),
         )
     except BadRequest:
         pass
@@ -120,11 +143,22 @@ def test_generate_content_invalid_temperature(
     assert dict(spans[0].attributes) == {
         "gen_ai.operation.name": "chat",
         "gen_ai.request.model": "gemini-1.5-flash-002",
-        "gen_ai.request.temperature": 1000.0,
         "gen_ai.system": "vertex_ai",
     }
     assert_span_error(spans[0])
 
+    # Emits the faulty content which caused the request to fail
+    logs = log_exporter.get_finished_logs()
+    assert len(logs) == 1
+    assert logs[0].log_record.attributes == {
+        "gen_ai.system": "vertex_ai",
+        "event.name": "gen_ai.user.message",
+    }
+    assert logs[0].log_record.body == {
+        "content": [{"text": "Say this is a test"}],
+        "role": "my_invalid_role",
+    }
+
 
 @pytest.mark.vcr()
 def test_generate_content_extra_params(span_exporter, instrument_no_content):