feat: add _meta annotations for breadcrumb truncataion

Author: shellmayr

sentry_sdk/scope.py

@@ -29,6 +29,7 @@
     Transaction,
 )
 from sentry_sdk.utils import (
+    AnnotatedValue,
     capture_internal_exception,
     capture_internal_exceptions,
     ContextVar,
@@ -181,6 +182,7 @@ class Scope:
         "_contexts",
         "_extras",
         "_breadcrumbs",
+        "_breadcrumb_info",
         "_event_processors",
         "_error_processors",
         "_should_capture",
@@ -205,6 +207,7 @@ def __init__(self, ty=None, client=None):
 
         self._name = None  # type: Optional[str]
         self._propagation_context = None  # type: Optional[PropagationContext]
+        self._breadcrumb_info = 0  # type: int
 
         self.client = NonRecordingClient()  # type: sentry_sdk.client.BaseClient
 
@@ -238,6 +241,7 @@ def __copy__(self):
         rv._extras = dict(self._extras)
 
         rv._breadcrumbs = copy(self._breadcrumbs)
+        rv._breadcrumb_info = copy(self._breadcrumb_info)
         rv._event_processors = list(self._event_processors)
         rv._error_processors = list(self._error_processors)
         rv._propagation_context = self._propagation_context
@@ -906,6 +910,7 @@ def clear_breadcrumbs(self):
         # type: () -> None
         """Clears breadcrumb buffer."""
         self._breadcrumbs = deque()  # type: Deque[Breadcrumb]
+        self._breadcrumb_info = 0
 
     def add_attachment(
         self,
@@ -973,6 +978,7 @@ def add_breadcrumb(self, crumb=None, hint=None, **kwargs):
 
         while len(self._breadcrumbs) > max_breadcrumbs:
             self._breadcrumbs.popleft()
+            self._breadcrumb_info += 1
 
     def start_transaction(
         self,
@@ -1339,6 +1345,12 @@ def _apply_breadcrumbs_to_event(self, event, hint, options):
             logger.debug("Error when sorting breadcrumbs", exc_info=err)
             pass
 
+        # Add annotation that breadcrumbs were truncated
+        original_length = len(event["breadcrumbs"]["values"]) + self._breadcrumb_info
+        event["breadcrumbs"]["values"] = AnnotatedValue.truncated_breadcrumbs(
+            event["breadcrumbs"]["values"], original_length
+        )
+
     def _apply_user_to_event(self, event, hint, options):
         # type: (Event, Hint, Optional[Dict[str, Any]]) -> None
         if event.get("user") is None and self._user is not None:

sentry_sdk/scrubber.py

@@ -146,9 +146,14 @@ def scrub_breadcrumbs(self, event):
         with capture_internal_exceptions():
             if "breadcrumbs" in event:
                 if "values" in event["breadcrumbs"]:
-                    for value in event["breadcrumbs"]["values"]:
-                        if "data" in value:
-                            self.scrub_dict(value["data"])
+                    if isinstance(event["breadcrumbs"]["values"], AnnotatedValue):
+                        for value in event["breadcrumbs"]["values"].value:
+                            if "data" in value:
+                                self.scrub_dict(value["data"])
+                    else:
+                        for value in event["breadcrumbs"]["values"]:
+                            if "data" in value:
+                                self.scrub_dict(value["data"])
 
     def scrub_frames(self, event):
         # type: (Event) -> None

sentry_sdk/utils.py

@@ -58,7 +58,7 @@
 
     from gevent.hub import Hub
 
-    from sentry_sdk._types import Event, ExcInfo
+    from sentry_sdk._types import Breadcrumb, Event, ExcInfo
 
     P = ParamSpec("P")
     R = TypeVar("R")
@@ -474,6 +474,15 @@ def substituted_because_contains_sensitive_data(cls):
             },
         )
 
+    @classmethod
+    def truncated_breadcrumbs(cls, breadcrumbs, n_truncated):
+        # type: (list[Breadcrumb], int) -> AnnotatedValue
+        """Breadcrumbs were removed because the number of breadcrumbs exceeded their maximum limit."""
+        return AnnotatedValue(
+            value=breadcrumbs,
+            metadata={"len": [n_truncated]},  # Remark
+        )
+
 
 if TYPE_CHECKING:
     from typing import TypeVar

tests/test_client.py

@@ -22,7 +22,7 @@
     set_tag,
 )
 from sentry_sdk.spotlight import DEFAULT_SPOTLIGHT_URL
-from sentry_sdk.utils import capture_internal_exception
+from sentry_sdk.utils import AnnotatedValue, capture_internal_exception
 from sentry_sdk.integrations.executing import ExecutingIntegration
 from sentry_sdk.transport import Transport
 from sentry_sdk.serializer import MAX_DATABAG_BREADTH
@@ -1058,8 +1058,10 @@ def test_max_breadcrumbs_option(
         add_breadcrumb({"type": "sourdough"})
 
     capture_message("dogs are great")
-
-    assert len(events[0]["breadcrumbs"]["values"]) == expected_breadcrumbs
+    if isinstance(events[0]["breadcrumbs"]["values"], AnnotatedValue):
+        assert len(events[0]["breadcrumbs"]["values"].value) == expected_breadcrumbs
+    else:
+        assert len(events[0]["breadcrumbs"]["values"]) == expected_breadcrumbs
 
 
 def test_multiple_positional_args(sentry_init):

tests/test_scrubber.py

@@ -137,7 +137,10 @@ def test_breadcrumb_extra_scrubbing(sentry_init, capture_events):
 
     assert event["_meta"]["extra"]["auth"] == {"": {"rem": [["!config", "s"]]}}
     assert event["_meta"]["breadcrumbs"] == {
-        "values": {"0": {"data": {"password": {"": {"rem": [["!config", "s"]]}}}}}
+        "values": {
+            "": {"len": [1]},
+            "0": {"data": {"password": {"": {"rem": [["!config", "s"]]}}}},
+        }
     }