fix(anthropic): serialize assistant message pydantic models (#3041)

Author: dinmukhamedm

packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/__init__.py

@@ -17,9 +17,10 @@
 from opentelemetry.instrumentation.anthropic.utils import (
     JSONEncoder,
     acount_prompt_tokens_from_request,
+    count_prompt_tokens_from_request,
     dont_throw,
     error_metrics_attributes,
-    count_prompt_tokens_from_request,
+    model_as_dict,
     run_async,
     set_span_attribute,
     shared_metrics_attributes,
@@ -134,16 +135,16 @@ async def _dump_content(message_index, content, span):
     elif isinstance(content, list):
         # If the content is a list of text blocks, concatenate them.
         # This is more commonly used in prompt caching.
-        if all([item.get("type") == "text" for item in content]):
-            return "".join([item.get("text") for item in content])
+        if all([model_as_dict(item).get("type") == "text" for item in content]):
+            return "".join([model_as_dict(item).get("text") for item in content])
 
         content = [
             (
                 await _process_image_item(
-                    item, span.context.trace_id, span.context.span_id, message_index, j
+                    model_as_dict(item), span.context.trace_id, span.context.span_id, message_index, j
                 )
-                if _is_base64_image(item)
-                else item
+                if _is_base64_image(model_as_dict(item))
+                else model_as_dict(item)
             )
             for j, item in enumerate(content)
         ]

packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/utils.py

@@ -4,12 +4,14 @@
 import logging
 import threading
 import traceback
+from importlib.metadata import version
 from opentelemetry import context as context_api
 from opentelemetry.instrumentation.anthropic.config import Config
 from opentelemetry.semconv_ai import SpanAttributes
 
 GEN_AI_SYSTEM = "gen_ai.system"
 GEN_AI_SYSTEM_ANTHROPIC = "anthropic"
+_PYDANTIC_VERSION = version("pydantic")
 
 
 def set_span_attribute(span, name, value):
@@ -150,3 +152,17 @@ def default(self, o):
             logger = logging.getLogger(__name__)
             logger.debug("Failed to serialize object of type: %s", type(o).__name__)
             return ""
+
+
+def model_as_dict(model):
+    if isinstance(model, dict):
+        return model
+    if _PYDANTIC_VERSION < "2.0.0" and hasattr(model, "dict"):
+        return model.dict()
+    if hasattr(model, "model_dump"):
+        return model.model_dump()
+    else:
+        try:
+            return dict(model)
+        except Exception:
+            return model

packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_messages/test_anthropic_image_with_history.yaml

@@ -9,6 +9,20 @@
 
 from .utils import verify_metrics
 
+image_content_block = {
+    "type": "image",
+    "source": {
+        "type": "base64",
+        "media_type": "image/jpeg",
+        "data": base64.b64encode(
+            open(
+                Path(__file__).parent.joinpath("data/logo.jpg"),
+                "rb",
+            ).read()
+        ).decode("utf-8"),
+    },
+}
+
 
 @pytest.mark.vcr
 def test_anthropic_message_create(exporter, reader):
@@ -76,19 +90,7 @@ def test_anthropic_multi_modal(exporter):
                         "type": "text",
                         "text": "What do you see?",
                     },
-                    {
-                        "type": "image",
-                        "source": {
-                            "type": "base64",
-                            "media_type": "image/jpeg",
-                            "data": base64.b64encode(
-                                open(
-                                    Path(__file__).parent.joinpath("data/logo.jpg"),
-                                    "rb",
-                                ).read()
-                            ).decode("utf-8"),
-                        },
-                    },
+                    image_content_block,
                 ],
             },
         ],
@@ -129,6 +131,87 @@ def test_anthropic_multi_modal(exporter):
     )
 
 
+@pytest.mark.vcr
+def test_anthropic_image_with_history(exporter):
+    client = Anthropic()
+    system_message = "You are a helpful assistant. Be concise and to the point."
+    user_message1 = {
+        "role": "user",
+        "content": "Are you capable of describing an image?"
+    }
+    user_message2 = {
+        "role": "user",
+        "content": [
+            {"type": "text", "text": "What do you see?"},
+            image_content_block,
+        ]
+    }
+
+    response1 = client.messages.create(
+        max_tokens=1024,
+        model="claude-3-5-haiku-latest",
+        system=system_message,
+        messages=[
+            user_message1,
+        ],
+    )
+
+    response2 = client.messages.create(
+        max_tokens=1024,
+        model="claude-3-5-haiku-latest",
+        system=system_message,
+        messages=[
+            user_message1,
+            {"role": "assistant", "content": response1.content},
+            user_message2,
+        ],
+    )
+
+    spans = exporter.get_finished_spans()
+    assert all(span.name == "anthropic.chat" for span in spans)
+    assert (
+        spans[0].attributes[f"{SpanAttributes.LLM_PROMPTS}.0.content"] ==
+        system_message
+    )
+    assert spans[0].attributes[f"{SpanAttributes.LLM_PROMPTS}.0.role"] == "system"
+    assert (
+        spans[0].attributes[f"{SpanAttributes.LLM_PROMPTS}.1.content"]
+        == "Are you capable of describing an image?"
+    )
+    assert spans[0].attributes[f"{SpanAttributes.LLM_PROMPTS}.1.role"] == "user"
+    assert spans[0].attributes[f"{SpanAttributes.LLM_COMPLETIONS}.0.content"] == response1.content[0].text
+    assert spans[0].attributes[f"{SpanAttributes.LLM_COMPLETIONS}.0.role"] == "assistant"
+    assert (
+        spans[0].attributes.get("gen_ai.response.id")
+        == "msg_01Ctc62hUPvikvYASXZqTo9q"
+    )
+
+    assert (
+        spans[1].attributes[f"{SpanAttributes.LLM_PROMPTS}.0.content"] ==
+        system_message
+    )
+    assert spans[1].attributes[f"{SpanAttributes.LLM_PROMPTS}.0.role"] == "system"
+    assert (
+        spans[1].attributes[f"{SpanAttributes.LLM_PROMPTS}.1.content"]
+        == "Are you capable of describing an image?"
+    )
+    assert spans[1].attributes[f"{SpanAttributes.LLM_PROMPTS}.1.role"] == "user"
+    assert spans[1].attributes[f"{SpanAttributes.LLM_PROMPTS}.2.content"] == response1.content[0].text
+    assert spans[1].attributes[f"{SpanAttributes.LLM_PROMPTS}.2.role"] == "assistant"
+    assert json.loads(spans[1].attributes[f"{SpanAttributes.LLM_PROMPTS}.3.content"]) == [
+        {"type": "text", "text": "What do you see?"},
+        {"type": "image_url", "image_url": {"url": "/some/url"}},
+    ]
+    assert spans[1].attributes[f"{SpanAttributes.LLM_PROMPTS}.3.role"] == "user"
+
+    assert spans[1].attributes[f"{SpanAttributes.LLM_COMPLETIONS}.0.content"] == response2.content[0].text
+    assert spans[1].attributes[f"{SpanAttributes.LLM_COMPLETIONS}.0.role"] == "assistant"
+    assert (
+        spans[1].attributes.get("gen_ai.response.id")
+        == "msg_01EtAvxHCWn5jjdUCnG4wEAd"
+    )
+
+
 @pytest.mark.vcr
 @pytest.mark.asyncio
 async def test_anthropic_async_multi_modal(exporter):
@@ -143,19 +226,7 @@ async def test_anthropic_async_multi_modal(exporter):
                         "type": "text",
                         "text": "What do you see?",
                     },
-                    {
-                        "type": "image",
-                        "source": {
-                            "type": "base64",
-                            "media_type": "image/jpeg",
-                            "data": base64.b64encode(
-                                open(
-                                    Path(__file__).parent.joinpath("data/logo.jpg"),
-                                    "rb",
-                                ).read()
-                            ).decode("utf-8"),
-                        },
-                    },
+                    image_content_block,
                 ],
             },
         ],