8361827: [TESTBUG] serviceability/HeapDump/UnmountedVThreadNativeMethodAtTop.java throws OutOfMemoryError

Reviewed-by: rrich, lmesnik
Backport-of: cbb3d23

Author: SendaoYan

test/hotspot/jtreg/serviceability/HeapDump/UnmountedVThreadNativeMethodAtTop.java

@@ -38,6 +38,7 @@
 import java.util.concurrent.CountDownLatch;
 import com.sun.management.HotSpotDiagnosticMXBean;
 
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import static org.junit.jupiter.api.Assertions.*;
 
@@ -49,6 +50,15 @@ public class UnmountedVThreadNativeMethodAtTop {
 
     boolean done;
 
+    /**
+     * The tests accumulate previous heap dumps. Trigger GC before each test to get rid of them.
+     * This makes dumps smaller, processing faster, and avoids OOMs
+     */
+    @BeforeEach
+    void doGC() {
+        System.gc();
+    }
+
     /**
      * Test dumping the heap while a virtual thread is blocked entering a synchronized native method.
      */