Handle and test other OpenAI/Anthropic client methods (#312)

Co-authored-by: David Montague <[email protected]>

Author: alexmojaki

logfire/_internal/integrations/llm_providers/anthropic.py

@@ -13,7 +13,6 @@
 
     from ...main import LogfireSpan
 
-
 __all__ = (
     'get_endpoint_config',
     'on_response',
@@ -25,8 +24,9 @@ def get_endpoint_config(options: FinalRequestOptions) -> EndpointConfig:
     """Returns the endpoint config for Anthropic depending on the url."""
     url = options.url
     json_data = options.json_data
-    if not isinstance(json_data, dict):
-        raise ValueError('Expected `options.json_data` to be a dictionary')
+    if not isinstance(json_data, dict):  # pragma: no cover
+        # Ensure that `{request_data[model]!r}` doesn't raise an error, just a warning about `model` missing.
+        json_data = {}
 
     if url == '/v1/messages':
         return EndpointConfig(
@@ -35,7 +35,11 @@ def get_endpoint_config(options: FinalRequestOptions) -> EndpointConfig:
             content_from_stream=content_from_messages,
         )
     else:
-        raise ValueError(f'Unknown Anthropic API endpoint: `{url}`')
+        return EndpointConfig(
+            message_template='Anthropic API call to {url!r}',
+            span_data={'request_data': json_data, 'url': url},
+            content_from_stream=None,
+        )
 
 
 def content_from_messages(chunk: anthropic.types.MessageStreamEvent) -> str | None:

logfire/_internal/integrations/llm_providers/llm_provider.py

@@ -76,14 +76,7 @@ def _instrumentation_setup(**kwargs: Any) -> Any:
         if is_instrumentation_suppressed():
             return None, None, kwargs
 
-        options = kwargs['options']
-        try:
-            message_template, span_data, content_from_stream = get_endpoint_config_fn(options)
-        except ValueError as exc:
-            logfire_llm.warn(
-                'Unable to instrument {suffix} API call: {error}', suffix=scope_suffix, error=str(exc), kwargs=kwargs
-            )
-            return None, None, kwargs
+        message_template, span_data, content_from_stream = get_endpoint_config_fn(kwargs['options'])
 
         span_data['async'] = is_async
 

logfire/_internal/integrations/llm_providers/openai.py

@@ -28,12 +28,11 @@
 def get_endpoint_config(options: FinalRequestOptions) -> EndpointConfig:
     """Returns the endpoint config for OpenAI depending on the url."""
     url = options.url
+
     json_data = options.json_data
-    if not isinstance(json_data, dict):
-        raise ValueError('Expected `options.json_data` to be a dictionary')
-    if 'model' not in json_data:
-        # all OpenAI API calls have a model AFAIK
-        raise ValueError('`model` not found in request data')
+    if not isinstance(json_data, dict):  # pragma: no cover
+        # Ensure that `{request_data[model]!r}` doesn't raise an error, just a warning about `model` missing.
+        json_data = {}
 
     if url == '/chat/completions':
         return EndpointConfig(
@@ -60,7 +59,11 @@ def get_endpoint_config(options: FinalRequestOptions) -> EndpointConfig:
             content_from_stream=None,
         )
     else:
-        raise ValueError(f'Unknown OpenAI API endpoint: `{url}`')
+        return EndpointConfig(
+            message_template='OpenAI API call to {url!r}',
+            span_data={'request_data': json_data, 'url': url},
+            content_from_stream=None,
+        )
 
 
 def content_from_completions(chunk: Completion | None) -> str | None:

pyproject.toml

@@ -84,7 +84,6 @@ dev-dependencies = [
     "aiohttp",
     "redis",
     "pymongo",
-    "starlette",
     "fastapi",
     "Flask",
     "django",

requirements-dev.lock

@@ -168,7 +168,7 @@ mkdocstrings-python==1.10.5
 multidict==6.0.5
     # via aiohttp
     # via yarl
-mypy==1.10.1
+mypy==1.11.0
 mypy-extensions==1.0.0
     # via black
     # via mypy
@@ -177,7 +177,7 @@ nodeenv==1.9.1
     # via pyright
 numpy==2.0.0
     # via pandas
-openai==1.35.14
+openai==1.36.0
 opentelemetry-api==1.25.0
     # via opentelemetry-exporter-otlp-proto-http
     # via opentelemetry-instrumentation
@@ -324,7 +324,7 @@ pymdown-extensions==10.8.1
     # via mkdocstrings
 pymongo==4.8.0
 pyright==1.1.372
-pytest==8.2.2
+pytest==8.3.1
     # via pytest-django
     # via pytest-pretty
 pytest-django==4.8.0
@@ -363,8 +363,8 @@ rich==13.7.1
     # via logfire
     # via pytest-pretty
     # via typer
-ruff==0.5.2
-setuptools==71.0.1
+ruff==0.5.3
+setuptools==71.0.4
     # via opentelemetry-instrumentation
 shellingham==1.5.4
     # via typer
@@ -412,7 +412,7 @@ tzdata==2024.1
     # via pandas
 urllib3==2.2.2
     # via requests
-uvicorn==0.30.1
+uvicorn==0.30.3
     # via fastapi
 uvloop==0.19.0
     # via uvicorn

requirements.lock

@@ -57,7 +57,7 @@ requests==2.32.3
     # via opentelemetry-exporter-otlp-proto-http
 rich==13.7.1
     # via logfire
-setuptools==71.0.1
+setuptools==71.0.4
     # via opentelemetry-instrumentation
 typing-extensions==4.12.2
     # via logfire

tests/otel_integrations/test_anthropic.py

@@ -7,7 +7,6 @@
 import httpx
 import pydantic
 import pytest
-from anthropic._models import FinalRequestOptions
 from anthropic.types import (
     Completion,
     Message,
@@ -24,7 +23,6 @@
 from inline_snapshot import snapshot
 
 import logfire
-from logfire._internal.integrations.llm_providers.anthropic import get_endpoint_config
 from logfire.testing import TestExporter
 
 ANY_ADAPTER = pydantic.TypeAdapter(Any)  # type: ignore
@@ -491,35 +489,24 @@ def test_unknown_method(instrumented_client: anthropic.Anthropic, exporter: Test
     assert exporter.exported_spans_as_dict() == snapshot(
         [
             {
-                'name': 'Unable to instrument {suffix} API call: {error}',
+                'name': 'Anthropic API call to {url!r}',
                 'context': {'is_remote': False, 'span_id': 1, 'trace_id': 1},
                 'parent': None,
                 'start_time': 1000000000,
-                'end_time': 1000000000,
+                'end_time': 2000000000,
                 'attributes': {
-                    'logfire.span_type': 'log',
+                    'logfire.span_type': 'span',
                     'logfire.tags': ('LLM',),
-                    'logfire.level_num': 13,
-                    'logfire.msg_template': 'Unable to instrument {suffix} API call: {error}',
-                    'logfire.msg': 'Unable to instrument Anthropic API call: Unknown Anthropic API endpoint: `/v1/complete`',
+                    'request_data': '{"max_tokens_to_sample":1000,"model":"claude-2.1","prompt":"prompt"}',
+                    'url': '/v1/complete',
+                    'async': False,
+                    'logfire.msg_template': 'Anthropic API call to {url!r}',
+                    'logfire.msg': "Anthropic API call to '/v1/complete'",
                     'code.filepath': 'test_anthropic.py',
                     'code.function': 'test_unknown_method',
                     'code.lineno': 123,
-                    'error': 'Unknown Anthropic API endpoint: `/v1/complete`',
-                    'kwargs': IsStr(),
                     'logfire.json_schema': IsStr(),
-                    'suffix': 'Anthropic',
                 },
             }
         ]
     )
-
-
-def test_get_endpoint_config_json_not_dict():
-    with pytest.raises(ValueError, match='Expected `options.json_data` to be a dictionary'):
-        get_endpoint_config(FinalRequestOptions(method='POST', url='...'))
-
-
-def test_get_endpoint_config_unknown_url():
-    with pytest.raises(ValueError, match='Unknown Anthropic API endpoint: `/foobar/`'):
-        get_endpoint_config(FinalRequestOptions(method='POST', url='/foobar/', json_data={'model': 'foobar'}))