chore(er): ensure max frame per trace (#12791) (#13576)

We make sure that the max frame settings is honoured for a whole trace
instead of a single span to avoid capturing too many snapshots.

(cherry picked from commit 560b7a6)
(cherry picked from commit bb68a4f)

## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

---------

Co-authored-by: Gabriele N. Tornetta <[email protected]>

Author: tylfin

ddtrace/debugging/_exception/replay.py

@@ -20,6 +20,7 @@
 from ddtrace.internal.rate_limiter import BudgetRateLimiterWithJitter as RateLimiter
 from ddtrace.internal.rate_limiter import RateLimitExceeded
 from ddtrace.internal.utils.time import HourGlass
+from ddtrace.settings.exception_replay import config
 from ddtrace.trace import Span
 
 
@@ -38,6 +39,7 @@
 
 # used to rate limit decision on the entire local trace (stored at the root span)
 CAPTURE_TRACE_TAG = "_dd.debug.error.trace_captured"
+SNAPSHOT_COUNT_TAG = "_dd.debug.error.snapshot_count"
 
 # unique exception id
 EXCEPTION_HASH_TAG = "_dd.debug.error.exception_hash"
@@ -201,6 +203,18 @@ def can_capture(span: Span) -> bool:
     raise ValueError(msg)
 
 
+def get_snapshot_count(span: Span) -> int:
+    root = span._local_root
+    if root is None:
+        return 0
+
+    count = root.get_metric(SNAPSHOT_COUNT_TAG)
+    if count is None:
+        return 0
+
+    return int(count)
+
+
 class SpanExceptionHandler:
     __uploader__ = LogsIntakeUploaderV1
 
@@ -225,20 +239,22 @@ def on_span_exception(
 
         seq = count(1)  # 1-based sequence number
 
-        while chain:
+        frames_captured = get_snapshot_count(span)
+        while chain and frames_captured <= config.max_frames:
             exc, _tb = chain.pop()  # LIFO: reverse the chain
 
             if _tb is None or _tb.tb_frame is None:
                 # If we don't have a traceback there isn't much we can do
                 continue
 
             # DEV: We go from the handler up to the root exception
-            while _tb:
+            while _tb and frames_captured < config.max_frames:
                 frame = _tb.tb_frame
                 code = frame.f_code
                 seq_nr = next(seq)
 
                 if is_user_code(Path(frame.f_code.co_filename)):
+                    snapshot = None
                     snapshot_id = frame.f_locals.get(SNAPSHOT_KEY, None)
                     if snapshot_id is None:
                         # We don't have a snapshot for the frame so we create one
@@ -263,6 +279,9 @@ def on_span_exception(
                         # Memoize
                         frame.f_locals[SNAPSHOT_KEY] = snapshot_id = snapshot.uuid
 
+                        # Count
+                        frames_captured += 1
+
                     # Add correlation tags on the span
                     span.set_tag_str(FRAME_SNAPSHOT_ID_TAG % seq_nr, snapshot_id)
                     span.set_tag_str(FRAME_FUNCTION_TAG % seq_nr, code.co_name)
@@ -276,6 +295,11 @@ def on_span_exception(
             span.set_tag_str(EXCEPTION_HASH_TAG, str(exc_ident))
             span.set_tag_str(EXCEPTION_ID_TAG, str(exc_id))
 
+            # Update the snapshot count
+            root = span._local_root
+            if root is not None:
+                root.set_metric(SNAPSHOT_COUNT_TAG, frames_captured)
+
     @classmethod
     def enable(cls) -> None:
         if cls._instance is not None:

ddtrace/settings/exception_replay.py

@@ -14,6 +14,13 @@ class ExceptionReplayConfig(En):
         help="Enable automatic capturing of exception debugging information",
         deprecations=[("debugging.enabled", None, "3.0")],
     )
+    max_frames = En.v(
+        int,
+        "replay.capture_max_frames",
+        default=8,
+        help_type="int",
+        help="The maximum number of frames to capture for each exception",
+    )
 
 
 config = ExceptionReplayConfig()

tests/debugging/exception/test_replay.py

@@ -314,3 +314,44 @@ def a(v):
             assert all(
                 s.line_capture["locals"]["nonloc"] == {"type": "int", "value": "4"} for s in uploader.collector.queue
             )
+
+    def test_debugger_max_frames(self):
+        config = ExceptionReplayConfig()
+        root = None
+
+        def r(n=config.max_frames * 2, c=None):
+            if n == 0:
+                if c is None:
+                    raise ValueError("hello")
+                else:
+                    c()
+            r(n - 1, c)
+
+        def a():
+            with self.trace("a"):
+                r()
+
+        def b():
+            with self.trace("b"):
+                r(10, a)
+
+        def c():
+            nonlocal root
+
+            with self.trace("c") as root:
+                r(10, b)
+
+        with exception_replay() as uploader:
+            rate_limiter = RateLimiter(
+                limit_rate=float("inf"),  # no rate limit
+                raise_on_exceed=False,
+            )
+            with with_rate_limiter(rate_limiter):
+                with pytest.raises(ValueError):
+                    c()
+
+            self.assert_span_count(3)
+            n = uploader.collector.queue
+            assert len(n) == config.max_frames
+
+            assert root.get_metric(replay.SNAPSHOT_COUNT_TAG) == config.max_frames

tests/telemetry/test_writer.py

@@ -354,6 +354,7 @@ def test_app_started_event_configuration_override(test_agent_session, run_python
         {"name": "DD_DYNAMIC_INSTRUMENTATION_UPLOAD_FLUSH_INTERVAL", "origin": "default", "value": 1.0},
         {"name": "DD_DYNAMIC_INSTRUMENTATION_UPLOAD_TIMEOUT", "origin": "default", "value": 30},
         {"name": "DD_ENV", "origin": "default", "value": None},
+        {"name": "DD_EXCEPTION_REPLAY_CAPTURE_MAX_FRAMES", "origin": "default", "value": 8},
         {"name": "DD_EXCEPTION_REPLAY_ENABLED", "origin": "env_var", "value": True},
         {"name": "DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED", "origin": "default", "value": False},
         {"name": "DD_HTTP_CLIENT_TAG_QUERY_STRING", "origin": "default", "value": None},