From e1804a8a9943d69a46bba88d771b55e4358eede3 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 23 Oct 2024 13:30:00 +0200 Subject: [PATCH 1/8] Serialize dict like attributes --- sentry_sdk/integrations/anthropic.py | 11 +++-- .../integrations/anthropic/test_anthropic.py | 49 ++++++++++--------- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index 87e69a3113..a59c96162c 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -1,4 +1,5 @@ from functools import wraps +import json from typing import TYPE_CHECKING import sentry_sdk @@ -74,9 +75,9 @@ def _calculate_token_usage(result, span): def _get_responses(content): - # type: (list[Any]) -> list[dict[str, Any]] + # type: (list[Any]) -> str """ - Get JSON of a Anthropic responses. + Get Anthropic responses as serialized JSON. """ responses = [] for item in content: @@ -87,7 +88,7 @@ def _get_responses(content): "text": item.text, } ) - return responses + return json.dumps(responses) def _collect_ai_data(event, input_tokens, output_tokens, content_blocks): @@ -126,7 +127,7 @@ def _add_ai_data_to_span( complete_message = "".join(content_blocks) span.set_data( SPANDATA.AI_RESPONSES, - [{"type": "text", "text": complete_message}], + json.dumps([{"type": "text", "text": complete_message}]), ) total_tokens = input_tokens + output_tokens record_token_usage(span, input_tokens, output_tokens, total_tokens) @@ -165,7 +166,7 @@ def _sentry_patched_create_common(f, *args, **kwargs): span.set_data(SPANDATA.AI_STREAMING, False) if should_send_default_pii() and integration.include_prompts: - span.set_data(SPANDATA.AI_INPUT_MESSAGES, messages) + span.set_data(SPANDATA.AI_INPUT_MESSAGES, json.dumps(messages)) if hasattr(result, "content"): if should_send_default_pii() and integration.include_prompts: diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 8ce12e70f5..0d44cadcab 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -1,3 +1,4 @@ +import json from unittest import mock try: @@ -115,10 +116,10 @@ def test_nonstreaming_create_message( assert span["data"][SPANDATA.AI_MODEL_ID] == "model" if send_default_pii and include_prompts: - assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == messages - assert span["data"][SPANDATA.AI_RESPONSES] == [ - {"type": "text", "text": "Hi, I'm Claude."} - ] + assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == json.dumps(messages) + assert span["data"][SPANDATA.AI_RESPONSES] == json.dumps( + [{"type": "text", "text": "Hi, I'm Claude."}] + ) else: assert SPANDATA.AI_INPUT_MESSAGES not in span["data"] assert SPANDATA.AI_RESPONSES not in span["data"] @@ -183,10 +184,10 @@ async def test_nonstreaming_create_message_async( assert span["data"][SPANDATA.AI_MODEL_ID] == "model" if send_default_pii and include_prompts: - assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == messages - assert span["data"][SPANDATA.AI_RESPONSES] == [ - {"type": "text", "text": "Hi, I'm Claude."} - ] + assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == json.dumps(messages) + assert span["data"][SPANDATA.AI_RESPONSES] == json.dumps( + [{"type": "text", "text": "Hi, I'm Claude."}] + ) else: assert SPANDATA.AI_INPUT_MESSAGES not in span["data"] assert SPANDATA.AI_RESPONSES not in span["data"] @@ -282,10 +283,10 @@ def test_streaming_create_message( assert span["data"][SPANDATA.AI_MODEL_ID] == "model" if send_default_pii and include_prompts: - assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == messages - assert span["data"][SPANDATA.AI_RESPONSES] == [ - {"type": "text", "text": "Hi! I'm Claude!"} - ] + assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == json.dumps(messages) + assert span["data"][SPANDATA.AI_RESPONSES] == json.dumps( + [{"type": "text", "text": "Hi! I'm Claude!"}] + ) else: assert SPANDATA.AI_INPUT_MESSAGES not in span["data"] @@ -385,10 +386,10 @@ async def test_streaming_create_message_async( assert span["data"][SPANDATA.AI_MODEL_ID] == "model" if send_default_pii and include_prompts: - assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == messages - assert span["data"][SPANDATA.AI_RESPONSES] == [ - {"type": "text", "text": "Hi! I'm Claude!"} - ] + assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == json.dumps(messages) + assert span["data"][SPANDATA.AI_RESPONSES] == json.dumps( + [{"type": "text", "text": "Hi! I'm Claude!"}] + ) else: assert SPANDATA.AI_INPUT_MESSAGES not in span["data"] @@ -515,10 +516,10 @@ def test_streaming_create_message_with_input_json_delta( assert span["data"][SPANDATA.AI_MODEL_ID] == "model" if send_default_pii and include_prompts: - assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == messages - assert span["data"][SPANDATA.AI_RESPONSES] == [ - {"text": "", "type": "text"} - ] # we do not record InputJSONDelta because it could contain PII + assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == json.dumps(messages) + assert span["data"][SPANDATA.AI_RESPONSES] == json.dumps( + [{"type": "text", "text": ""}] + ) # we do not record InputJSONDelta because it could contain PII else: assert SPANDATA.AI_INPUT_MESSAGES not in span["data"] @@ -652,10 +653,10 @@ async def test_streaming_create_message_with_input_json_delta_async( assert span["data"][SPANDATA.AI_MODEL_ID] == "model" if send_default_pii and include_prompts: - assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == messages - assert span["data"][SPANDATA.AI_RESPONSES] == [ - {"text": "", "type": "text"} - ] # we do not record InputJSONDelta because it could contain PII + assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == json.dumps(messages) + assert span["data"][SPANDATA.AI_RESPONSES] == json.dumps( + [{"type": "text", "text": ""}] + ) # we do not record InputJSONDelta because it could contain PII else: assert SPANDATA.AI_INPUT_MESSAGES not in span["data"] From bbac36328a4124666d5aa7af2eef6161f5b496f1 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 23 Oct 2024 13:45:09 +0200 Subject: [PATCH 2/8] Use util for serialization --- sentry_sdk/integrations/anthropic.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index a59c96162c..620a8bc519 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -1,5 +1,4 @@ from functools import wraps -import json from typing import TYPE_CHECKING import sentry_sdk @@ -8,6 +7,7 @@ from sentry_sdk.integrations import DidNotEnable, Integration from sentry_sdk.scope import should_send_default_pii from sentry_sdk.utils import ( + _serialize_span_attribute, capture_internal_exceptions, event_from_exception, package_version, @@ -88,7 +88,7 @@ def _get_responses(content): "text": item.text, } ) - return json.dumps(responses) + return _serialize_span_attribute(responses) def _collect_ai_data(event, input_tokens, output_tokens, content_blocks): @@ -127,7 +127,7 @@ def _add_ai_data_to_span( complete_message = "".join(content_blocks) span.set_data( SPANDATA.AI_RESPONSES, - json.dumps([{"type": "text", "text": complete_message}]), + _serialize_span_attribute([{"type": "text", "text": complete_message}]), ) total_tokens = input_tokens + output_tokens record_token_usage(span, input_tokens, output_tokens, total_tokens) @@ -166,7 +166,7 @@ def _sentry_patched_create_common(f, *args, **kwargs): span.set_data(SPANDATA.AI_STREAMING, False) if should_send_default_pii() and integration.include_prompts: - span.set_data(SPANDATA.AI_INPUT_MESSAGES, json.dumps(messages)) + span.set_data(SPANDATA.AI_INPUT_MESSAGES, _serialize_span_attribute(messages)) if hasattr(result, "content"): if should_send_default_pii() and integration.include_prompts: From 4d54beba7d681a530db1786cfe7d4f6c8ab4fb54 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 23 Oct 2024 13:46:23 +0200 Subject: [PATCH 3/8] Formatting --- sentry_sdk/integrations/anthropic.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index 620a8bc519..ee062994c8 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -166,7 +166,9 @@ def _sentry_patched_create_common(f, *args, **kwargs): span.set_data(SPANDATA.AI_STREAMING, False) if should_send_default_pii() and integration.include_prompts: - span.set_data(SPANDATA.AI_INPUT_MESSAGES, _serialize_span_attribute(messages)) + span.set_data( + SPANDATA.AI_INPUT_MESSAGES, _serialize_span_attribute(messages) + ) if hasattr(result, "content"): if should_send_default_pii() and integration.include_prompts: From 0c703ebf70b48b02e967da49374032e8d78d858d Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 23 Oct 2024 13:49:10 +0200 Subject: [PATCH 4/8] Cleanup --- sentry_sdk/integrations/anthropic.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index ee062994c8..b96015cd8e 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -75,9 +75,9 @@ def _calculate_token_usage(result, span): def _get_responses(content): - # type: (list[Any]) -> str + # type: (list[Any]) -> list[dict[str, Any]] """ - Get Anthropic responses as serialized JSON. + Get Anthropic responses as JSON. """ responses = [] for item in content: @@ -88,7 +88,7 @@ def _get_responses(content): "text": item.text, } ) - return _serialize_span_attribute(responses) + return responses def _collect_ai_data(event, input_tokens, output_tokens, content_blocks): @@ -172,7 +172,10 @@ def _sentry_patched_create_common(f, *args, **kwargs): if hasattr(result, "content"): if should_send_default_pii() and integration.include_prompts: - span.set_data(SPANDATA.AI_RESPONSES, _get_responses(result.content)) + span.set_data( + SPANDATA.AI_RESPONSES, + _serialize_span_attribute(_get_responses(result.content)), + ) _calculate_token_usage(result, span) span.__exit__(None, None, None) From 47e4c26dcafc2812a4ca87d3bdd6b45dca9c1fba Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 23 Oct 2024 13:49:42 +0200 Subject: [PATCH 5/8] minimize diff --- sentry_sdk/integrations/anthropic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index b96015cd8e..a41005ed20 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -77,7 +77,7 @@ def _calculate_token_usage(result, span): def _get_responses(content): # type: (list[Any]) -> list[dict[str, Any]] """ - Get Anthropic responses as JSON. + Get JSON of a Anthropic responses. """ responses = [] for item in content: From 5061b74ee59ec89e9b625442c0a99f4e61e3bdfd Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 23 Oct 2024 13:50:34 +0200 Subject: [PATCH 6/8] Update test --- .../integrations/anthropic/test_anthropic.py | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 0d44cadcab..c107aa9671 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -1,4 +1,3 @@ -import json from unittest import mock try: @@ -20,7 +19,7 @@ async def __call__(self, *args, **kwargs): from anthropic.types.message_delta_event import MessageDeltaEvent from anthropic.types.message_start_event import MessageStartEvent -from sentry_sdk.utils import package_version +from sentry_sdk.utils import _serialize_span_attribute, package_version try: from anthropic.types import InputJSONDelta @@ -116,8 +115,8 @@ def test_nonstreaming_create_message( assert span["data"][SPANDATA.AI_MODEL_ID] == "model" if send_default_pii and include_prompts: - assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == json.dumps(messages) - assert span["data"][SPANDATA.AI_RESPONSES] == json.dumps( + assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == _serialize_span_attribute(messages) + assert span["data"][SPANDATA.AI_RESPONSES] == _serialize_span_attribute( [{"type": "text", "text": "Hi, I'm Claude."}] ) else: @@ -184,8 +183,8 @@ async def test_nonstreaming_create_message_async( assert span["data"][SPANDATA.AI_MODEL_ID] == "model" if send_default_pii and include_prompts: - assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == json.dumps(messages) - assert span["data"][SPANDATA.AI_RESPONSES] == json.dumps( + assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == _serialize_span_attribute(messages) + assert span["data"][SPANDATA.AI_RESPONSES] == _serialize_span_attribute( [{"type": "text", "text": "Hi, I'm Claude."}] ) else: @@ -283,8 +282,8 @@ def test_streaming_create_message( assert span["data"][SPANDATA.AI_MODEL_ID] == "model" if send_default_pii and include_prompts: - assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == json.dumps(messages) - assert span["data"][SPANDATA.AI_RESPONSES] == json.dumps( + assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == _serialize_span_attribute(messages) + assert span["data"][SPANDATA.AI_RESPONSES] == _serialize_span_attribute( [{"type": "text", "text": "Hi! I'm Claude!"}] ) @@ -386,8 +385,8 @@ async def test_streaming_create_message_async( assert span["data"][SPANDATA.AI_MODEL_ID] == "model" if send_default_pii and include_prompts: - assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == json.dumps(messages) - assert span["data"][SPANDATA.AI_RESPONSES] == json.dumps( + assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == _serialize_span_attribute(messages) + assert span["data"][SPANDATA.AI_RESPONSES] == _serialize_span_attribute( [{"type": "text", "text": "Hi! I'm Claude!"}] ) @@ -516,8 +515,8 @@ def test_streaming_create_message_with_input_json_delta( assert span["data"][SPANDATA.AI_MODEL_ID] == "model" if send_default_pii and include_prompts: - assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == json.dumps(messages) - assert span["data"][SPANDATA.AI_RESPONSES] == json.dumps( + assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == _serialize_span_attribute(messages) + assert span["data"][SPANDATA.AI_RESPONSES] == _serialize_span_attribute( [{"type": "text", "text": ""}] ) # we do not record InputJSONDelta because it could contain PII @@ -653,8 +652,8 @@ async def test_streaming_create_message_with_input_json_delta_async( assert span["data"][SPANDATA.AI_MODEL_ID] == "model" if send_default_pii and include_prompts: - assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == json.dumps(messages) - assert span["data"][SPANDATA.AI_RESPONSES] == json.dumps( + assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == _serialize_span_attribute(messages) + assert span["data"][SPANDATA.AI_RESPONSES] == _serialize_span_attribute( [{"type": "text", "text": ""}] ) # we do not record InputJSONDelta because it could contain PII From 4292df86f52a847df63a9c6b66efd846ea94a7ab Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 5 Nov 2024 13:23:30 +0100 Subject: [PATCH 7/8] Fixed test --- tests/integrations/anthropic/test_anthropic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index c107aa9671..7764a01ede 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -667,6 +667,7 @@ async def test_streaming_create_message_with_input_json_delta_async( assert span["data"]["ai.streaming"] is True +@pytest.mark.forked def test_exception_message_create(sentry_init, capture_events): sentry_init(integrations=[AnthropicIntegration()], traces_sample_rate=1.0) events = capture_events() From 39be9c64483a9c086919516b30824c8c43ae05cc Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 5 Nov 2024 13:26:12 +0100 Subject: [PATCH 8/8] Linting --- .../integrations/anthropic/test_anthropic.py | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 7764a01ede..4a7d7ed458 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -115,7 +115,9 @@ def test_nonstreaming_create_message( assert span["data"][SPANDATA.AI_MODEL_ID] == "model" if send_default_pii and include_prompts: - assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == _serialize_span_attribute(messages) + assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == _serialize_span_attribute( + messages + ) assert span["data"][SPANDATA.AI_RESPONSES] == _serialize_span_attribute( [{"type": "text", "text": "Hi, I'm Claude."}] ) @@ -183,7 +185,9 @@ async def test_nonstreaming_create_message_async( assert span["data"][SPANDATA.AI_MODEL_ID] == "model" if send_default_pii and include_prompts: - assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == _serialize_span_attribute(messages) + assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == _serialize_span_attribute( + messages + ) assert span["data"][SPANDATA.AI_RESPONSES] == _serialize_span_attribute( [{"type": "text", "text": "Hi, I'm Claude."}] ) @@ -282,7 +286,9 @@ def test_streaming_create_message( assert span["data"][SPANDATA.AI_MODEL_ID] == "model" if send_default_pii and include_prompts: - assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == _serialize_span_attribute(messages) + assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == _serialize_span_attribute( + messages + ) assert span["data"][SPANDATA.AI_RESPONSES] == _serialize_span_attribute( [{"type": "text", "text": "Hi! I'm Claude!"}] ) @@ -385,7 +391,9 @@ async def test_streaming_create_message_async( assert span["data"][SPANDATA.AI_MODEL_ID] == "model" if send_default_pii and include_prompts: - assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == _serialize_span_attribute(messages) + assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == _serialize_span_attribute( + messages + ) assert span["data"][SPANDATA.AI_RESPONSES] == _serialize_span_attribute( [{"type": "text", "text": "Hi! I'm Claude!"}] ) @@ -515,7 +523,9 @@ def test_streaming_create_message_with_input_json_delta( assert span["data"][SPANDATA.AI_MODEL_ID] == "model" if send_default_pii and include_prompts: - assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == _serialize_span_attribute(messages) + assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == _serialize_span_attribute( + messages + ) assert span["data"][SPANDATA.AI_RESPONSES] == _serialize_span_attribute( [{"type": "text", "text": ""}] ) # we do not record InputJSONDelta because it could contain PII @@ -652,7 +662,9 @@ async def test_streaming_create_message_with_input_json_delta_async( assert span["data"][SPANDATA.AI_MODEL_ID] == "model" if send_default_pii and include_prompts: - assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == _serialize_span_attribute(messages) + assert span["data"][SPANDATA.AI_INPUT_MESSAGES] == _serialize_span_attribute( + messages + ) assert span["data"][SPANDATA.AI_RESPONSES] == _serialize_span_attribute( [{"type": "text", "text": ""}] ) # we do not record InputJSONDelta because it could contain PII