8269556: sun/tools/jhsdb/JShellHeapDumpTest.java fails with RuntimeException 'JShellToolProvider' missing from stdout/stderr

Backport-of: db502d30a42847f69273f48378f230d9d55eb8d6

Author: GoeLin

test/jdk/sun/tools/jhsdb/JShellHeapDumpTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -97,21 +97,33 @@ public static void launch(String expectedMessage, String... toolArgs)
         launch(expectedMessage, Arrays.asList(toolArgs));
     }
 
-    public static void printStackTraces(String file) throws IOException {
+    /* Returns false if the attempt should be retried. */
+    public static boolean printStackTraces(String file, boolean allowRetry) throws IOException {
         try {
             String output = HprofReader.getStack(file, 0);
             // We only require JShellToolProvider to be in the output if we did the
             // short sleep. If we did not, the java process may not have executed far
             // enough along to even start the main thread.
             if (doSleep && !output.contains("JShellToolProvider")) {
-                throw new RuntimeException("'JShellToolProvider' missing from stdout/stderr");
+                // This check will very rarely fail due to not be able to get the stack trace
+                // of the main thread do to it actively executing. See JDK-8269556. We retry once
+                // if that happens. This failure is so rare that this should be enough to make it
+                // extremely unlikely that we ever see this test fail again for this reason.
+                if (!allowRetry) {
+                    throw new RuntimeException("'JShellToolProvider' missing from stdout/stderr");
+                } else {
+                    System.out.println("'JShellToolProvider' missing. Allow one retry.");
+                    return true; // Allow one retry
+                }
             }
         } catch (Exception ex) {
             throw new RuntimeException("Test ERROR " + ex, ex);
         }
+        return false;
     }
 
-    public static void testHeapDump() throws IOException {
+    /* Returns false if the attempt should be retried. */
+    public static boolean testHeapDump(boolean allowRetry) throws IOException {
         File hprofFile = new File("jhsdb.jmap.heap." +
                              System.currentTimeMillis() + ".hprof");
         if (hprofFile.exists()) {
@@ -124,10 +136,12 @@ public static void testHeapDump() throws IOException {
         assertTrue(hprofFile.exists() && hprofFile.isFile(),
                    "Could not create dump file " + hprofFile.getAbsolutePath());
 
-        printStackTraces(hprofFile.getAbsolutePath());
+        boolean retry = printStackTraces(hprofFile.getAbsolutePath(), allowRetry);
 
         System.out.println("hprof file size: " + hprofFile.length());
         hprofFile.delete();
+
+        return retry;
     }
 
     public static void launchJshell() throws IOException {
@@ -149,7 +163,7 @@ public static void launchJshell() throws IOException {
         // Give jshell a chance to fully start up. This makes SA more stable for the jmap dump.
         try {
             if (doSleep) {
-                Thread.sleep(2000);
+                Thread.sleep(4000);
             }
         } catch (Exception e) {
         }
@@ -166,7 +180,12 @@ public static void main(String[] args) throws Exception {
         } else if (args.length != 0) {
             throw new RuntimeException("Too many args: " + args.length);
         }
-        testHeapDump();
+
+        boolean retry = testHeapDump(true);
+        // In case of rare failure to find 'JShellToolProvider' in the output, allow one retry.
+        if (retry) {
+            testHeapDump(false);
+        }
 
         // The test throws RuntimeException on error.
         // IOException is thrown if Jshell can't start because of some bad