Do the best to handle the OOME when computing children of a node.

Author: jisedlac

visualvm/heapviewer/src/org/graalvm/visualvm/heapviewer/model/ErrorNode.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.visualvm.heapviewer.model;
+
+import org.openide.util.NbBundle;
+
+/**
+ *
+ * @author Jiri Sedlacek
+ */
+@NbBundle.Messages({
+    "ErrorNode_OOME=<not enough memory>"
+})
+public class ErrorNode extends TextNode {
+    
+    public ErrorNode(String text) {
+        super(text);
+    }
+    
+    
+    public static final class OOME extends ErrorNode {
+        
+        OOME() { super(Bundle.ErrorNode_OOME()); }
+        
+    }
+    
+}

visualvm/heapviewer/src/org/graalvm/visualvm/heapviewer/model/HeapViewerNode.java

@@ -138,11 +138,16 @@ private HeapViewerNode[] resolveChildren() {
         RootNode root = RootNode.get(this);
         if (root == null) return NO_NODES;
 
-        children = root.retrieveChildren(this);
-        
-        if (children == null) {
-            HeapViewerNode[] ch = computeChildren(root);
-            setChildren(ch == null ? NO_NODES : ch);
+        try {
+            children = root.retrieveChildren(this);
+
+            if (children == null) {
+                HeapViewerNode[] ch = computeChildren(root);
+                setChildren(ch == null ? NO_NODES : ch);
+            }
+        } catch (OutOfMemoryError e) {
+            handleOOME(e);
+            setChildren(new HeapViewerNode[] { new ErrorNode.OOME() });
         }
 
         return children;
@@ -175,6 +180,9 @@ protected HeapViewerNode[] doInBackground() throws Exception {
                     ret = lazilyComputeChildren(root.getContext().getFragment().getHeap(), root.getViewID(), root.getViewFilter(), root.getDataTypes(), root.getSortOrders(), progress);
                 } catch (InterruptedException ex) {
                     ret = null;
+                } catch (OutOfMemoryError e) {
+                    handleOOME(e);
+                    ret = new HeapViewerNode[] { new ErrorNode.OOME() };
                 }
                 if (Thread.interrupted()) ret = null; // make sure the interrupted flag is handled & reset in all circumstances
 
@@ -258,6 +266,13 @@ private static HeapViewerNode[] checkForLoops(HeapViewerNode parent, HeapViewerN
     }
 
 
+    private void handleOOME(OutOfMemoryError e) {
+        RootNode root = RootNode.get(this);
+        if (root != null) root.handleOOME(e);
+        else System.err.println("Out of memory in " + toString() + ": " + e.getMessage()); // NOI18N
+    }
+    
+    
     // --- Values --------------------------------------------------------------
 
     private static final Object NO_VALUE = new Object();

visualvm/heapviewer/src/org/graalvm/visualvm/heapviewer/model/RootNode.java

@@ -70,6 +70,11 @@ public abstract class RootNode extends HeapViewerNode {
     public abstract HeapViewerRenderer resolveRenderer(HeapViewerNode node);
 
 
+    protected void handleOOME(OutOfMemoryError e) {
+        System.err.println("Out of memory in " + getViewID() + ": " + e.getMessage()); // NOI18N
+    }
+    
+    
 //    public RootNode() {
 //        this(NO_NODES);
 //    }

visualvm/heapviewer/src/org/graalvm/visualvm/heapviewer/ui/TreeTableView.java

@@ -69,6 +69,7 @@
 import org.graalvm.visualvm.heapviewer.model.Progress;
 import org.graalvm.visualvm.heapviewer.model.RootNode;
 import javax.swing.RowSorter;
+import org.graalvm.visualvm.heapviewer.utils.HeapUtils;
 import org.openide.util.Lookup;
 import org.openide.util.NbBundle;
 
@@ -145,6 +146,7 @@ public TreeTableView(String viewID, HeapContext context, HeapViewerActions actio
             protected HeapViewerNode[] retrieveChildren(HeapViewerNode node) { return nodesCache.retrieveChildren(node); }
             protected HeapViewerNode[] lazilyComputeChildren(Heap heap, String viewID, HeapViewerNodeFilter viewFilter, List<DataType> dataTypes, List<SortOrder> sortOrders, Progress progress) throws InterruptedException { return TreeTableView.this.computeData(root, heap, viewID, viewFilter, dataTypes, sortOrders, progress); }
             public HeapViewerRenderer resolveRenderer(HeapViewerNode node) { return TreeTableView.this.resolveNodeRenderer(node); }
+            protected void handleOOME(OutOfMemoryError e) { super.handleOOME(e); TreeTableView.this.handleOOME(e); }
         };
         currentRoot = root;
 
@@ -420,6 +422,12 @@ public HeapViewerRenderer getNodeRenderer(HeapViewerNode node) {
         return nodeRenderer;
     }
 
+    // --- OOME handling -------------------------------------------------------
+    
+    protected void handleOOME(OutOfMemoryError e) {
+        HeapUtils.handleOOME(e);
+    }
+    
     // --- BreadCrumbs prototype -----------------------------------------------
 
     private void setRoot(HeapViewerNode newRoot) {

visualvm/heapviewer/src/org/graalvm/visualvm/heapviewer/ui/TreeTableViewRenderer.java

@@ -36,9 +36,11 @@
 import org.graalvm.visualvm.lib.profiler.api.icons.Icons;
 import org.graalvm.visualvm.lib.profiler.heapwalk.ui.icons.HeapWalkerIcons;
 import org.graalvm.visualvm.heapviewer.model.DataType;
+import org.graalvm.visualvm.heapviewer.model.ErrorNode;
 import org.graalvm.visualvm.heapviewer.model.HeapViewerNode;
 import org.graalvm.visualvm.heapviewer.model.LoopNode;
 import org.graalvm.visualvm.heapviewer.model.ProgressNode;
+import org.graalvm.visualvm.lib.profiler.api.icons.GeneralIcons;
 import org.openide.util.NbBundle;
 
 /**
@@ -61,6 +63,7 @@ class TreeTableViewRenderer implements ProfilerRenderer {
     {
         lookup.put(LoopNode.class, new LoopNodeRenderer());
         lookup.put(ProgressNode.class, new ProgressNodeRenderer());
+        lookup.put(ErrorNode.class, new ErrorNodeRenderer());
     }
 
 
@@ -170,6 +173,22 @@ public String getShortName() {
 
     }
 
+    private static class ErrorNodeRenderer extends LabelRenderer implements HeapViewerRenderer {
+    
+        public ErrorNodeRenderer() {
+            setIcon(Icons.getIcon(GeneralIcons.ERROR));
+        }
+        
+        public void setValue(Object value, int row) {
+            setText(value == null ? "" : value.toString()); // NOI18N
+        }
+        
+        public String getShortName() {
+            return getText();
+        }
+
+    }
+    
     private static class FallbackRenderer extends LabelRenderer implements HeapViewerRenderer {}
 
 }

visualvm/heapviewer/src/org/graalvm/visualvm/heapviewer/utils/HeapUtils.java

@@ -32,11 +32,13 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import javax.swing.SwingUtilities;
 import org.graalvm.visualvm.lib.jfluid.heap.FieldValue;
 import org.graalvm.visualvm.lib.jfluid.heap.Heap;
 import org.graalvm.visualvm.lib.jfluid.heap.Instance;
 import org.graalvm.visualvm.lib.jfluid.heap.JavaClass;
 import org.graalvm.visualvm.lib.jfluid.heap.ObjectFieldValue;
+import org.graalvm.visualvm.lib.profiler.api.ProfilerDialogs;
 import org.graalvm.visualvm.lib.profiler.heapwalk.details.spi.DetailsUtils;
 import org.openide.util.NbBundle;
 
@@ -47,7 +49,9 @@
 @NbBundle.Messages({
     "HeapUtils_UnknownClass=unknown class",
     "HeapUtils_UnknownInstance=unknown instance",
-    "HeapUtils_Class=class"
+    "HeapUtils_Class=class",
+    "HeapUtils_OomeCaption=Out Of Memory",
+    "HeapUtils_OomeMsg=<html><b>Not enough memory to finish the operation.</b><br/><br/>To avoid this error please increase the -Xmx value<br>in the etc/visualvm.conf file in VisualVM directory.</html>"
 }) 
 public final class HeapUtils {
 
@@ -190,6 +194,25 @@ public static String htmlize(String text) {
     }
 
 
+    // --- OOME handling -------------------------------------------------------
+    
+    private static boolean OOME_NOTIFIED = false;
+    
+    public static void handleOOME(OutOfMemoryError e) {
+        SwingUtilities.invokeLater(new Runnable() {
+            public void run() {
+                if (!OOME_NOTIFIED) {
+                    OOME_NOTIFIED = true;
+                    ProfilerDialogs.displayError(Bundle.HeapUtils_OomeMsg(), Bundle.HeapUtils_OomeCaption(), null);
+                    // NOTE: might update the Xmx automatically and/or lower MoreObjectsNode.MAX_BUFFER_SIZE
+                }
+            }
+        });
+    }
+    
+    
+    // --- Private stuff -------------------------------------------------------
+    
     private HeapUtils() {}