Emit system and assistant events

Author: aabmass

instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md

@@ -11,5 +11,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   ([#3192](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3192))
 - Initial VertexAI instrumentation
   ([#3123](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3123))
-- VertexAI emit user events
+- VertexAI emit user, system, and assistant events
   ([#3203](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3203))

instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/events.py

@@ -45,3 +45,47 @@ def user_event(
         },
         body=body,
     )
+
+
+def assistant_event(
+    *,
+    role: str = "assistant",
+    content: AnyValue = None,
+) -> Event:
+    """Creates a User event
+    https://github.com/open-telemetry/semantic-conventions/blob/v1.28.0/docs/gen-ai/gen-ai-events.md#assistant-event
+    """
+    body: dict[str, AnyValue] = {
+        "role": role,
+    }
+    if content is not None:
+        body["content"] = content
+    return Event(
+        name="gen_ai.assistant.message",
+        attributes={
+            gen_ai_attributes.GEN_AI_SYSTEM: gen_ai_attributes.GenAiSystemValues.VERTEX_AI.value,
+        },
+        body=body,
+    )
+
+
+def system_event(
+    *,
+    role: str = "system",
+    content: AnyValue = None,
+) -> Event:
+    """Creates a User event
+    https://github.com/open-telemetry/semantic-conventions/blob/v1.28.0/docs/gen-ai/gen-ai-events.md#system-event
+    """
+    body: dict[str, AnyValue] = {
+        "role": role,
+    }
+    if content is not None:
+        body["content"] = content
+    return Event(
+        name="gen_ai.system.message",
+        attributes={
+            gen_ai_attributes.GEN_AI_SYSTEM: gen_ai_attributes.GenAiSystemValues.VERTEX_AI.value,
+        },
+        body=body,
+    )

instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py

@@ -26,7 +26,11 @@
 )
 
 from opentelemetry._events import Event
-from opentelemetry.instrumentation.vertexai.events import user_event
+from opentelemetry.instrumentation.vertexai.events import (
+    assistant_event,
+    system_event,
+    user_event,
+)
 from opentelemetry.semconv._incubating.attributes import (
     gen_ai_attributes as GenAIAttributes,
 )
@@ -146,16 +150,41 @@ def get_span_name(span_attributes: Mapping[str, AttributeValue]) -> str:
 def request_to_events(
     *, params: GenerateContentParams, capture_content: bool
 ) -> Iterable[Event]:
+    # System message
+    if params.system_instruction:
+        request_content = _parts_to_any_value(
+            capture_content=capture_content,
+            parts=params.system_instruction.parts,
+        )
+        yield system_event(
+            role=params.system_instruction.role, content=request_content
+        )
+
     for content in params.contents or []:
+        # Assistant message
         if content.role == "model":
-            # TODO: handle assistant message
-            pass
+            request_content = _parts_to_any_value(
+                capture_content=capture_content, parts=content.parts
+            )
+
+            yield assistant_event(role=content.role, content=request_content)
         # Assume user event but role should be "user"
         else:
-            request_content = None
-            if capture_content:
-                request_content = [
-                    cast(dict[str, AnyValue], type(part).to_dict(part))  # type: ignore[reportUnknownMemberType]
-                    for part in content.parts
-                ]
+            request_content = _parts_to_any_value(
+                capture_content=capture_content, parts=content.parts
+            )
             yield user_event(role=content.role, content=request_content)
+
+
+def _parts_to_any_value(
+    *,
+    capture_content: bool,
+    parts: Sequence[content.Part] | Sequence[content_v1beta1.Part],
+) -> list[dict[str, AnyValue]] | None:
+    if not capture_content:
+        return None
+
+    return [
+        cast(dict[str, AnyValue], type(part).to_dict(part))  # type: ignore[reportUnknownMemberType]
+        for part in parts
+    ]

instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_all_input_events.yaml

@@ -0,0 +1,94 @@
+interactions:
+- request:
+    body: |-
+      {
+        "contents": [
+          {
+            "role": "user",
+            "parts": [
+              {
+                "text": "My name is OpenTelemetry"
+              }
+            ]
+          },
+          {
+            "role": "model",
+            "parts": [
+              {
+                "text": "Hello OpenTelemetry!"
+              }
+            ]
+          },
+          {
+            "role": "user",
+            "parts": [
+              {
+                "text": "Address me by name and say this is a test"
+              }
+            ]
+          }
+        ],
+        "systemInstruction": {
+          "role": "user",
+          "parts": [
+            {
+              "text": "You are a clever language model"
+            }
+          ]
+        }
+      }
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '548'
+      Content-Type:
+      - application/json
+      User-Agent:
+      - python-requests/2.32.3
+    method: POST
+    uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-1.5-flash-002:generateContent?%24alt=json%3Benum-encoding%3Dint
+  response:
+    body:
+      string: |-
+        {
+          "candidates": [
+            {
+              "content": {
+                "role": "model",
+                "parts": [
+                  {
+                    "text": "OpenTelemetry, this is a test.\n"
+                  }
+                ]
+              },
+              "finishReason": 1,
+              "avgLogprobs": -1.1655389850299496e-06
+            }
+          ],
+          "usageMetadata": {
+            "promptTokenCount": 25,
+            "candidatesTokenCount": 9,
+            "totalTokenCount": 34
+          },
+          "modelVersion": "gemini-1.5-flash-002"
+        }
+    headers:
+      Content-Type:
+      - application/json; charset=UTF-8
+      Transfer-Encoding:
+      - chunked
+      Vary:
+      - Origin
+      - X-Origin
+      - Referer
+      content-length:
+      - '422'
+    status:
+      code: 200
+      message: OK
+version: 1

instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions.py

@@ -259,3 +259,74 @@ def assert_span_error(span: ReadableSpan) -> None:
     # Records exception event
     error_events = [e for e in span.events if e.name == "exception"]
     assert error_events != []
+
+
+@pytest.mark.vcr
+def test_generate_content_all_input_events(
+    log_exporter: InMemoryLogExporter,
+    instrument_with_content: VertexAIInstrumentor,
+):
+    model = GenerativeModel(
+        "gemini-1.5-flash-002",
+        system_instruction=Part.from_text("You are a clever language model"),
+    )
+    model.generate_content(
+        [
+            Content(
+                role="user", parts=[Part.from_text("My name is OpenTelemetry")]
+            ),
+            Content(
+                role="model", parts=[Part.from_text("Hello OpenTelemetry!")]
+            ),
+            Content(
+                role="user",
+                parts=[
+                    Part.from_text("Address me by name and say this is a test")
+                ],
+            ),
+        ],
+    )
+
+    # Emits a system event, 2 users events, and a assistant event
+    logs = log_exporter.get_finished_logs()
+    assert len(logs) == 4
+    system_log, user_log1, assistant_log, user_log2 = [
+        log_data.log_record for log_data in logs
+    ]
+
+    assert system_log.attributes == {
+        "gen_ai.system": "vertex_ai",
+        "event.name": "gen_ai.system.message",
+    }
+    assert system_log.body == {
+        "content": [{"text": "You are a clever language model"}],
+        # The API only allows user and model, so system instruction is considered a user role
+        "role": "user",
+    }
+
+    assert user_log1.attributes == {
+        "gen_ai.system": "vertex_ai",
+        "event.name": "gen_ai.user.message",
+    }
+    assert user_log1.body == {
+        "content": [{"text": "My name is OpenTelemetry"}],
+        "role": "user",
+    }
+
+    assert assistant_log.attributes == {
+        "gen_ai.system": "vertex_ai",
+        "event.name": "gen_ai.assistant.message",
+    }
+    assert assistant_log.body == {
+        "content": [{"text": "Hello OpenTelemetry!"}],
+        "role": "model",
+    }
+
+    assert user_log2.attributes == {
+        "gen_ai.system": "vertex_ai",
+        "event.name": "gen_ai.user.message",
+    }
+    assert user_log2.body == {
+        "content": [{"text": "Address me by name and say this is a test"}],
+        "role": "user",
+    }