Heap dump should not be generated on handled exceptions

Patch by Paulo Motta; Reviewed by Isaac Reath, Stefan Miklosovic for CASSANDRA-20974

Author: pauloricardomg

CHANGES.txt

@@ -1,6 +1,7 @@
 5.0.7
+ * Heap dump should not be generated on handled exceptions (CASSANDRA-20974)
 Merged from 4.1:
-* ReadCommandController should close fast to avoid deadlock when building secondary index (CASSANDRA-19564)
+ * ReadCommandController should close fast to avoid deadlock when building secondary index (CASSANDRA-19564)
 Merged from 4.0:
  * Updated dtest-api to 0.0.18 and removed JMX-related classes that now live in the dtest-api (CASSANDRA-20884)
 

src/java/org/apache/cassandra/utils/HeapUtils.java

@@ -128,10 +128,6 @@ public static String maybeCreateHeapDump()
 
                     return fullPath;
                 }
-                else
-                {
-                    logger.debug("Heap dump creation on uncaught exceptions is disabled.");
-                }
             }
             catch (Throwable e)
             {

src/java/org/apache/cassandra/utils/JVMStabilityInspector.java

@@ -75,7 +75,7 @@ public static void uncaughtException(Thread thread, Throwable t)
             if (t2 != t && (t2 instanceof FSError || t2 instanceof CorruptSSTableException))
                 logger.error("Exception in thread {}", thread, t2);
         }
-        JVMStabilityInspector.inspectThrowable(t);
+        inspectThrowable(t, JVMStabilityInspector::inspectDiskError, true);
     }
 
     /**
@@ -86,12 +86,12 @@ public static void uncaughtException(Thread thread, Throwable t)
      */
     public static void inspectThrowable(Throwable t) throws OutOfMemoryError
     {
-        inspectThrowable(t, JVMStabilityInspector::inspectDiskError);
+        inspectThrowable(t, JVMStabilityInspector::inspectDiskError, false);
     }
 
     public static void inspectCommitLogThrowable(Throwable t)
     {
-        inspectThrowable(t, JVMStabilityInspector::inspectCommitLogError);
+        inspectThrowable(t, JVMStabilityInspector::inspectCommitLogError, false);
     }
 
     private static void inspectDiskError(Throwable t)
@@ -102,7 +102,7 @@ else if (t instanceof FSError)
             FileUtils.handleFSError((FSError) t);
     }
 
-    public static void inspectThrowable(Throwable t, Consumer<Throwable> fn) throws OutOfMemoryError
+    public static void inspectThrowable(Throwable t, Consumer<Throwable> fn, boolean isUncaughtException) throws OutOfMemoryError
     {
         boolean isUnstable = false;
         if (t instanceof OutOfMemoryError)
@@ -136,7 +136,18 @@ else if (t instanceof UnrecoverableIllegalStateException)
         }
 
         // Anything other than an OOM, we should try and heap dump to capture what's going on if configured to do so
-        HeapUtils.maybeCreateHeapDump();
+        if (isUncaughtException && DatabaseDescriptor.getDumpHeapOnUncaughtException())
+        {
+            try
+            {
+                // Avoid entering maybeCreateHeapDump unless the setting is enabled to avoid expensive lock
+                HeapUtils.maybeCreateHeapDump();
+            }
+            catch (Throwable sub)
+            {
+                t.addSuppressed(sub);
+            }
+        }
 
         if (t instanceof InterruptedException)
             throw new UncheckedInterruptedException((InterruptedException) t);
@@ -167,7 +178,7 @@ else if (t instanceof UnrecoverableIllegalStateException)
         }
 
         if (t.getCause() != null)
-            inspectThrowable(t.getCause(), fn);
+            inspectThrowable(t.getCause(), fn, isUncaughtException);
     }
 
     private static final Set<String> FORCE_HEAP_OOM_IGNORE_SET = ImmutableSet.of("Java heap space", "GC Overhead limit exceeded");