8321400: java/foreign/TestStubAllocFailure.java fails with code cache exhaustion

Reviewed-by: mcimadamore

Author: JornVernee

test/jdk/java/foreign/TestAddressDereference.java

@@ -128,8 +128,8 @@ public void testNativeUpcallArgNeg(long alignment, ValueLayout layout) throws Th
         if (!badAlign) return;
         runInNewProcess(UpcallTestRunner.class, true,
                 new String[] {Long.toString(alignment), layout.toString() })
-                .assertFailed()
-                .assertStdErrContains("alignment constraint for address");
+                .shouldNotHaveExitValue(0)
+                .stderrShouldContain("alignment constraint for address");
     }
 
     public static class UpcallTestRunner {

test/jdk/java/foreign/TestStubAllocFailure.java

@@ -31,36 +31,65 @@
  */
 
 import java.lang.foreign.*;
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
 import java.io.IOException;
 import java.util.List;
+import java.util.function.Consumer;
+import java.util.stream.Stream;
 
 import org.testng.annotations.Test;
 
-import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
 
 public class TestStubAllocFailure extends UpcallTestHelper {
 
     @Test
     public void testUpcallAllocFailure() throws IOException, InterruptedException {
         runInNewProcess(UpcallRunner.class, true, List.of("-XX:ReservedCodeCacheSize=3M"), List.of())
-                .assertSuccess();
+                .shouldNotHaveExitValue(0)
+                .shouldNotHaveFatalError();
+    }
+
+    @Test
+    public void testUDowncallAllocFailure() throws IOException, InterruptedException {
+        runInNewProcess(DowncallRunner.class, true, List.of("-XX:ReservedCodeCacheSize=3M"), List.of())
+                .shouldNotHaveExitValue(0)
+                .shouldNotHaveFatalError();
     }
 
     public static class UpcallRunner extends NativeTestHelper {
         public static void main(String[] args) throws Throwable {
-            try (Arena arena = Arena.ofConfined()) {
-                while (true) {
-                    // allocate stubs until we crash
-                    upcallStub(UpcallRunner.class, "target", FunctionDescriptor.ofVoid(), arena);
-                }
-            } catch (OutOfMemoryError e) {
-                assertTrue(e.getMessage().contains("Failed to allocate upcall stub"));
+            FunctionDescriptor descriptor = FunctionDescriptor.ofVoid();
+            MethodHandle target = MethodHandles.lookup().findStatic(UpcallRunner.class, "target", descriptor.toMethodType());
+            while (true) {
+                LINKER.upcallStub(target, descriptor, Arena.ofAuto());
             }
         }
 
         public static void target() {
             fail("Should not get here");
         }
     }
+
+    public static class DowncallRunner extends NativeTestHelper {
+
+        private static final int MAX_ARITY = 5;
+
+        private static void mapper(FunctionDescriptor fd, Consumer<FunctionDescriptor> sink) {
+            for (MemoryLayout l : List.of(C_INT, C_LONG_LONG, C_DOUBLE, C_FLOAT, C_SHORT)) {
+                sink.accept(fd.appendArgumentLayouts(l));
+            }
+        }
+
+        public static void main(String[] args) throws Throwable {
+            Linker linker = Linker.nativeLinker();
+            Stream<FunctionDescriptor> stream = Stream.of(FunctionDescriptor.ofVoid());
+            for (int i = 0; i < MAX_ARITY; i++) {
+                stream = stream.mapMulti(DowncallRunner::mapper);
+            }
+
+            stream.forEach(linker::downcallHandle);
+        }
+    }
 }

test/jdk/java/foreign/TestUpcallException.java

@@ -48,8 +48,8 @@ public class TestUpcallException extends UpcallTestHelper {
     @Test(dataProvider = "exceptionCases")
     public void testException(Class<?> target, boolean useSpec) throws InterruptedException, IOException {
         runInNewProcess(target, useSpec)
-                .assertFailed()
-                .assertStdErrContains("Testing upcall exceptions");
+                .shouldNotHaveExitValue(0)
+                .stderrShouldContain("Testing upcall exceptions");
     }
 
     @DataProvider

test/jdk/java/foreign/UpcallTestHelper.java

@@ -21,54 +21,24 @@
  * questions.
  */
 
+import jdk.test.lib.process.OutputAnalyzer;
 import jdk.test.lib.process.ProcessTools;
 import jdk.test.lib.Utils;
 
-import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotEquals;
 import static org.testng.Assert.assertTrue;
 
 public class UpcallTestHelper extends NativeTestHelper {
-    public record Output(int result, List<String> stdout, List<String> stderr) {
-        private static void assertContains(List<String> lines, String shouldInclude, String name) {
-            assertTrue(lines.stream().anyMatch(line -> line.contains(shouldInclude)),
-                "Did not find '" + shouldInclude + "' in " + name);
-        }
 
-        public Output assertFailed() {
-            assertNotEquals(result, 0);
-            return this;
-        }
-
-        public Output assertSuccess() {
-            assertEquals(result, 0);
-            return this;
-        }
-
-        public Output assertStdErrContains(String shouldInclude) {
-            assertContains(stderr, shouldInclude, "stderr");
-            return this;
-        }
-
-        public Output assertStdOutContains(String shouldInclude) {
-            assertContains(stdout, shouldInclude, "stdout");
-            return this;
-        }
-    }
-
-    public Output runInNewProcess(Class<?> target, boolean useSpec, String... programArgs) throws IOException, InterruptedException {
+    public OutputAnalyzer runInNewProcess(Class<?> target, boolean useSpec, String... programArgs) throws IOException, InterruptedException {
         return runInNewProcess(target, useSpec, List.of(), List.of(programArgs));
     }
 
-    public Output runInNewProcess(Class<?> target, boolean useSpec, List<String> vmArgs, List<String> programArgs) throws IOException, InterruptedException {
+    public OutputAnalyzer runInNewProcess(Class<?> target, boolean useSpec, List<String> vmArgs, List<String> programArgs) throws IOException, InterruptedException {
         assert !target.isArray();
 
         List<String> command = new ArrayList<>(List.of(
@@ -86,17 +56,10 @@ public Output runInNewProcess(Class<?> target, boolean useSpec, List<String> vmA
         boolean completed = process.waitFor(timeOut, TimeUnit.MINUTES);
         assertTrue(completed, "Time out while waiting for process");
 
-        List<String> outLines = linesFromStream(process.getInputStream());
-        outLines.forEach(System.out::println);
-        List<String> errLines = linesFromStream(process.getErrorStream());
-        errLines.forEach(System.err::println);
-
-        return new Output(process.exitValue(), outLines, errLines);
-    }
+        OutputAnalyzer output = new OutputAnalyzer(process);
+        output.outputTo(System.out);
+        output.errorTo(System.err);
 
-    private static List<String> linesFromStream(InputStream stream) throws IOException {
-        try (BufferedReader reader = new BufferedReader(new InputStreamReader(stream))) {
-            return reader.lines().toList();
-        }
+        return output;
     }
 }

test/jdk/java/foreign/critical/TestCriticalUpcall.java

@@ -43,7 +43,9 @@ public class TestCriticalUpcall extends UpcallTestHelper {
     @Test
     public void testUpcallFailure() throws IOException, InterruptedException {
         // test to see if we catch a trivial downcall doing an upcall
-        runInNewProcess(Runner.class, true).assertFailed().assertStdOutContains("wrong thread state for upcall");
+        runInNewProcess(Runner.class, true)
+            .shouldNotHaveExitValue(0)
+            .stdoutShouldContain("wrong thread state for upcall");
     }
 
     public static class Runner extends NativeTestHelper {

test/jdk/java/foreign/passheapsegment/TestPassHeapSegment.java

@@ -53,7 +53,9 @@ public void testNoHeapArgs() throws Throwable {
 
     @Test(dataProvider = "specs")
     public void testNoHeapReturns(boolean spec) throws IOException, InterruptedException {
-        runInNewProcess(Runner.class, spec).assertFailed().assertStdErrContains("Heap segment not allowed");
+        runInNewProcess(Runner.class, spec)
+            .shouldNotHaveExitValue(0)
+            .stderrShouldContain("Heap segment not allowed");
     }
 
     public static class Runner {

test/lib/jdk/test/lib/process/OutputAnalyzer.java

@@ -42,6 +42,8 @@ public final class OutputAnalyzer {
 
     private static final String deprecatedmsg = ".* VM warning:.* deprecated.*";
 
+    private static final String FATAL_ERROR_PAT = "# A fatal error has been detected.*";
+
     private final OutputBuffer buffer;
     /**
      * Create an OutputAnalyzer, a utility class for verifying output and exit
@@ -862,4 +864,11 @@ public void shouldContainMultiLinePattern(String... needles) {
         shouldContainMultiLinePattern(needles, true);
     }
 
+    /**
+     * Assert that we did not crash with a hard VM error (generating an hs_err_pidXXX.log)
+     */
+    public void shouldNotHaveFatalError() {
+        shouldNotMatch(FATAL_ERROR_PAT);
+    }
+
 }