Add outer exceptions support for Exception Replay

In some cases, the innermost exception could contain only third-party
frames and no instrumentation is performed while the exception could
be re-throw up in the stack where the call originated. This could help
to root cause the inner exception.
Now in that case we are climbing up the chained of exceptions to find
user frames to instrument.
when finding the innermost exception, we are reporting the list of
all outer exceptions beginning by the innermost. The chained index is
also kept in the probe and injected into the snapshot to be able to
remap the frame index correctly and report it to the span as tag.

Author: jpbempel

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/exception/DefaultExceptionDebugger.java

@@ -14,6 +14,9 @@
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
 import datadog.trace.util.AgentTaskScheduler;
 import java.time.Duration;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
 import java.util.List;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -75,32 +78,42 @@ public void handleException(Throwable t, AgentSpan span) {
       LOGGER.debug("Unable to fingerprint exception", t);
       return;
     }
-    Throwable innerMostException = ExceptionHelper.getInnerMostThrowable(t);
+    Deque<Throwable> chainedExceptions = new ArrayDeque<>();
+    Throwable innerMostException = ExceptionHelper.getInnerMostThrowable(t, chainedExceptions);
     if (innerMostException == null) {
       LOGGER.debug("Unable to find root cause of exception");
       return;
     }
+    List<Throwable> chainedExceptionsList = new ArrayList<>(chainedExceptions);
     if (exceptionProbeManager.isAlreadyInstrumented(fingerprint)) {
       ThrowableState state = exceptionProbeManager.getStateByThrowable(innerMostException);
       if (state == null) {
         LOGGER.debug("Unable to find state for throwable: {}", innerMostException.toString());
         return;
       }
-      processSnapshotsAndSetTags(t, span, state, innerMostException, fingerprint);
+      processSnapshotsAndSetTags(t, span, state, chainedExceptionsList, fingerprint);
       exceptionProbeManager.updateLastCapture(fingerprint);
     } else {
-      ExceptionProbeManager.CreationResult creationResult =
-          exceptionProbeManager.createProbesForException(innerMostException.getStackTrace());
-      if (creationResult.probesCreated > 0) {
-        AgentTaskScheduler.INSTANCE.execute(() -> applyExceptionConfiguration(fingerprint));
-      } else {
-        if (LOGGER.isDebugEnabled()) {
-          LOGGER.debug(
-              "No probe created, nativeFrames={}, thirdPartyFrames={} for exception: {}",
-              creationResult.nativeFrames,
-              creationResult.thirdPartyFrames,
-              ExceptionHelper.foldExceptionStackTrace(innerMostException));
+      // climb up the exception chain to find the first exception that has instrumented frames
+      Throwable throwable;
+      int chainedExceptionIdx = 0;
+      while ((throwable = chainedExceptions.pollFirst()) != null) {
+        ExceptionProbeManager.CreationResult creationResult =
+            exceptionProbeManager.createProbesForException(
+                throwable.getStackTrace(), chainedExceptionIdx);
+        if (creationResult.probesCreated > 0) {
+          AgentTaskScheduler.INSTANCE.execute(() -> applyExceptionConfiguration(fingerprint));
+          break;
+        } else {
+          if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug(
+                "No probe created, nativeFrames={}, thirdPartyFrames={} for exception: {}",
+                creationResult.nativeFrames,
+                creationResult.thirdPartyFrames,
+                ExceptionHelper.foldExceptionStackTrace(throwable));
+          }
         }
+        chainedExceptionIdx++;
       }
     }
   }
@@ -114,7 +127,7 @@ private static void processSnapshotsAndSetTags(
       Throwable t,
       AgentSpan span,
       ThrowableState state,
-      Throwable innerMostException,
+      List<Throwable> chainedExceptions,
       String fingerprint) {
     if (span.getTag(DD_DEBUG_ERROR_EXCEPTION_ID) != null) {
       LOGGER.debug("Clear previous frame tags");
@@ -127,12 +140,13 @@ private static void processSnapshotsAndSetTags(
                 }
               });
     }
-    int[] mapping = createThrowableMapping(innerMostException, t);
-    StackTraceElement[] innerTrace = innerMostException.getStackTrace();
     boolean snapshotAssigned = false;
     List<Snapshot> snapshots = state.getSnapshots();
     for (int i = 0; i < snapshots.size(); i++) {
       Snapshot snapshot = snapshots.get(i);
+      Throwable currentEx = chainedExceptions.get(snapshot.getChainedExceptionIdx());
+      int[] mapping = createThrowableMapping(currentEx, t);
+      StackTraceElement[] innerTrace = currentEx.getStackTrace();
       int currentIdx = innerTrace.length - snapshot.getStack().size();
       if (!sanityCheckSnapshotAssignment(snapshot, innerTrace, currentIdx)) {
         continue;

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/exception/ExceptionProbeManager.java

@@ -79,7 +79,8 @@ public CreationResult(int probesCreated, int thirdPartyFrames, int nativeFrames)
     }
   }
 
-  public CreationResult createProbesForException(StackTraceElement[] stackTraceElements) {
+  public CreationResult createProbesForException(
+      StackTraceElement[] stackTraceElements, int chainedExceptionIdx) {
     int instrumentedFrames = 0;
     int nativeFrames = 0;
     int thirdPartyFrames = 0;
@@ -102,7 +103,7 @@ public CreationResult createProbesForException(StackTraceElement[] stackTraceEle
               stackTraceElement.getMethodName(),
               null,
               String.valueOf(stackTraceElement.getLineNumber()));
-      ExceptionProbe probe = createMethodProbe(this, where);
+      ExceptionProbe probe = createMethodProbe(this, where, chainedExceptionIdx);
       probes.putIfAbsent(probe.getId(), probe);
       instrumentedFrames++;
     }
@@ -114,10 +115,16 @@ void addFingerprint(String fingerprint) {
   }
 
   private static ExceptionProbe createMethodProbe(
-      ExceptionProbeManager exceptionProbeManager, Where where) {
+      ExceptionProbeManager exceptionProbeManager, Where where, int chainedExceptionIdx) {
     String probeId = UUID.randomUUID().toString();
     return new ExceptionProbe(
-        new ProbeId(probeId, 0), where, null, null, null, exceptionProbeManager);
+        new ProbeId(probeId, 0),
+        where,
+        null,
+        null,
+        null,
+        exceptionProbeManager,
+        chainedExceptionIdx);
   }
 
   public boolean isAlreadyInstrumented(String fingerprint) {

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/probe/ExceptionProbe.java

@@ -21,14 +21,16 @@
 public class ExceptionProbe extends LogProbe implements ForceMethodInstrumentation {
   private static final Logger LOGGER = LoggerFactory.getLogger(ExceptionProbe.class);
   private final transient ExceptionProbeManager exceptionProbeManager;
+  private final transient int chainedExceptionIdx;
 
   public ExceptionProbe(
       ProbeId probeId,
       Where where,
       ProbeCondition probeCondition,
       Capture capture,
       Sampling sampling,
-      ExceptionProbeManager exceptionProbeManager) {
+      ExceptionProbeManager exceptionProbeManager,
+      int chainedExceptionIdx) {
     super(
         LANGUAGE,
         probeId,
@@ -42,6 +44,7 @@ public ExceptionProbe(
         capture,
         sampling);
     this.exceptionProbeManager = exceptionProbeManager;
+    this.chainedExceptionIdx = chainedExceptionIdx;
   }
 
   @Override
@@ -116,6 +119,7 @@ public void commit(
       // inside the stateByThrowable map
       clearExceptionRefs(snapshot);
       // add snapshot for later to wait for triggering point (ExceptionDebugger::handleException)
+      snapshot.setChainedExceptionIdx(chainedExceptionIdx);
       exceptionProbeManager.addSnapshot(snapshot);
       LOGGER.debug(
           "committing exception probe id={}, snapshot id={}, exception id={}",

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/sink/Snapshot.java

@@ -33,6 +33,7 @@ public class Snapshot {
   private transient String message;
   private final transient int maxDepth;
   private String exceptionId;
+  private transient int chainedExceptionIdx;
 
   public Snapshot(java.lang.Thread thread, ProbeImplementation probeImplementation, int maxDepth) {
     this.version = VERSION;
@@ -91,6 +92,10 @@ public void setExceptionId(String exceptionId) {
     this.exceptionId = exceptionId;
   }
 
+  public void setChainedExceptionIdx(int chainedExceptionIdx) {
+    this.chainedExceptionIdx = chainedExceptionIdx;
+  }
+
   public void addLine(CapturedContext context, int line) {
     captures.addLine(line, context);
   }
@@ -178,6 +183,10 @@ public String getExceptionId() {
     return exceptionId;
   }
 
+  public int getChainedExceptionIdx() {
+    return chainedExceptionIdx;
+  }
+
   public void recordStackTrace(int offset) {
     stack.clear();
     int cntr = 0;

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/ExceptionHelper.java

@@ -6,6 +6,7 @@
 import java.io.Writer;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Deque;
 import java.util.List;
 import org.slf4j.Logger;
 
@@ -60,9 +61,19 @@ public void write(String s, int off, int len) {
   }
 
   public static Throwable getInnerMostThrowable(Throwable t) {
+    return getInnerMostThrowable(t, null);
+  }
+
+  public static Throwable getInnerMostThrowable(Throwable t, Deque<Throwable> chainedExceptions) {
+    if (chainedExceptions != null) {
+      chainedExceptions.addFirst(t);
+    }
     int i = 100;
     while (t.getCause() != null && i > 0) {
       t = t.getCause();
+      if (chainedExceptions != null) {
+        chainedExceptions.addFirst(t);
+      }
       i--;
     }
     if (i == 0) {