Add more teest cases for error conditions and fix span name bug

Author: aabmass

instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py

@@ -134,4 +134,6 @@ def is_content_enabled() -> bool:
 def get_span_name(span_attributes: Mapping[str, AttributeValue]):
     name = span_attributes[GenAIAttributes.GEN_AI_OPERATION_NAME]
     model = span_attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL]
+    if not model:
+        return name
     return f"{name} {model}"

instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_vertexai_generate_content.yaml renamed to instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content.yaml

@@ -43,7 +43,7 @@ interactions:
                 ]
               },
               "finishReason": 1,
-              "avgLogprobs": -0.0054909539850134595
+              "avgLogprobs": -0.005692833348324424
             }
           ],
           "usageMetadata": {

instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_chat_completion_extra_params.yaml renamed to instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_extra_params.yaml

@@ -0,0 +1,56 @@
+interactions:
+- request:
+    body: |-
+      {
+        "contents": [
+          {
+            "role": "user",
+            "parts": [
+              {
+                "text": "Say this is a test"
+              }
+            ]
+          }
+        ]
+      }
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '141'
+      Content-Type:
+      - application/json
+      User-Agent:
+      - python-requests/2.32.3
+    method: POST
+    uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-does-not-exist:generateContent?%24alt=json%3Benum-encoding%3Dint
+  response:
+    body:
+      string: |-
+        {
+          "error": {
+            "code": 404,
+            "message": "Publisher Model `projects/otel-starter-project/locations/us-central1/publishers/google/models/gemini-does-not-exist` not found.",
+            "status": "NOT_FOUND",
+            "details": []
+          }
+        }
+    headers:
+      Content-Type:
+      - application/json; charset=UTF-8
+      Transfer-Encoding:
+      - chunked
+      Vary:
+      - Origin
+      - X-Origin
+      - Referer
+      content-length:
+      - '672'
+    status:
+      code: 404
+      message: Not Found
+version: 1

instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_vertexai_generate_content_error.yaml renamed to instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_invalid_temperature.yaml

@@ -1,5 +1,5 @@
 import pytest
-from google.api_core.exceptions import BadRequest
+from google.api_core.exceptions import BadRequest, NotFound
 from vertexai.generative_models import (
     Content,
     GenerationConfig,
@@ -8,14 +8,15 @@
 )
 
 from opentelemetry.instrumentation.vertexai import VertexAIInstrumentor
+from opentelemetry.sdk.trace import ReadableSpan
 from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
     InMemorySpanExporter,
 )
 from opentelemetry.trace import StatusCode
 
 
 @pytest.mark.vcr
-def test_vertexai_generate_content(
+def test_generate_content(
     span_exporter: InMemorySpanExporter,
     instrument_with_content: VertexAIInstrumentor,
 ):
@@ -37,7 +38,65 @@ def test_vertexai_generate_content(
 
 
 @pytest.mark.vcr
-def test_vertexai_generate_content_error(
+def test_generate_content_empty_model(
+    span_exporter: InMemorySpanExporter,
+    instrument_with_content: VertexAIInstrumentor,
+):
+    model = GenerativeModel("")
+    try:
+        model.generate_content(
+            [
+                Content(
+                    role="user", parts=[Part.from_text("Say this is a test")]
+                )
+            ],
+        )
+    except ValueError:
+        pass
+
+    spans = span_exporter.get_finished_spans()
+    assert len(spans) == 1
+    assert spans[0].name == "chat"
+    # Captures invalid params
+    assert dict(spans[0].attributes) == {
+        "gen_ai.operation.name": "chat",
+        "gen_ai.request.model": "",
+        "gen_ai.system": "vertex_ai",
+    }
+    assert_span_error(spans[0])
+
+
+@pytest.mark.vcr
+def test_generate_content_missing_model(
+    span_exporter: InMemorySpanExporter,
+    instrument_with_content: VertexAIInstrumentor,
+):
+    model = GenerativeModel("gemini-does-not-exist")
+    try:
+        model.generate_content(
+            [
+                Content(
+                    role="user", parts=[Part.from_text("Say this is a test")]
+                )
+            ],
+        )
+    except NotFound:
+        pass
+
+    spans = span_exporter.get_finished_spans()
+    assert len(spans) == 1
+    assert spans[0].name == "chat gemini-does-not-exist"
+    # Captures invalid params
+    assert dict(spans[0].attributes) == {
+        "gen_ai.operation.name": "chat",
+        "gen_ai.request.model": "gemini-does-not-exist",
+        "gen_ai.system": "vertex_ai",
+    }
+    assert_span_error(spans[0])
+
+
+@pytest.mark.vcr
+def test_generate_content_invalid_temperature(
     span_exporter: InMemorySpanExporter,
     instrument_with_content: VertexAIInstrumentor,
 ):
@@ -64,16 +123,11 @@ def test_vertexai_generate_content_error(
         "gen_ai.request.temperature": 1000.0,
         "gen_ai.system": "vertex_ai",
     }
-    # Sets error status
-    assert spans[0].status.status_code == StatusCode.ERROR
-
-    # Records exception event
-    assert len(spans[0].events) == 1
-    assert spans[0].events[0].name == "exception"
+    assert_span_error(spans[0])
 
 
 @pytest.mark.vcr()
-def test_chat_completion_extra_params(span_exporter, instrument_no_content):
+def test_generate_content_extra_params(span_exporter, instrument_no_content):
     generation_config = GenerationConfig(
         top_k=2,
         top_p=0.95,
@@ -105,3 +159,11 @@ def test_chat_completion_extra_params(span_exporter, instrument_no_content):
         "gen_ai.request.top_p": 0.949999988079071,
         "gen_ai.system": "vertex_ai",
     }
+
+
+def assert_span_error(span: ReadableSpan) -> None:
+    # Sets error status
+    assert span.status.status_code == StatusCode.ERROR
+    # Records exception event
+    error_events = [e for e in span.events if e.name == "exception"]
+    assert error_events != []