8367902: Allocation after Universe::before_exit() in the VM shutdown sequence

Reviewed-by: ayang, stefank, iwalulya, aboldtch, sspitsyn

Author: lmesnik

src/hotspot/share/runtime/java.cpp

@@ -476,7 +476,18 @@ void before_exit(JavaThread* thread, bool halt) {
 
   NativeHeapTrimmer::cleanup();
 
-  // Run before exit and then stop concurrent GC threads
+  if (JvmtiExport::should_post_thread_life()) {
+    JvmtiExport::post_thread_end(thread);
+  }
+
+  // Always call even when there are not JVMTI environments yet, since environments
+  // may be attached late and JVMTI must track phases of VM execution.
+  JvmtiExport::post_vm_death();
+  JvmtiAgentList::unload_agents();
+
+  // No user code can be executed in the current thread after this point.
+
+  // Run before exit and then stop concurrent GC threads.
   Universe::before_exit();
 
   if (PrintBytecodeHistogram) {
@@ -492,15 +503,6 @@ void before_exit(JavaThread* thread, bool halt) {
   }
 #endif
 
-  if (JvmtiExport::should_post_thread_life()) {
-    JvmtiExport::post_thread_end(thread);
-  }
-
-  // Always call even when there are not JVMTI environments yet, since environments
-  // may be attached late and JVMTI must track phases of VM execution
-  JvmtiExport::post_vm_death();
-  JvmtiAgentList::unload_agents();
-
   // Terminate the signal thread
   // Note: we don't wait until it actually dies.
   os::terminate_signal_thread();

test/hotspot/jtreg/serviceability/jvmti/events/VMDeath/AllocatingInVMDeath/TestAllocatingInVMDeath.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @summary Test verifies that VM still can execute java code, allocate
+ *          memory, and call GC in the VMDeath event callback.
+ *
+ * @bug 8367902
+ * @requires vm.jvmti
+ * @library /test/lib
+ * @build jdk.test.whitebox.WhiteBox
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
+ * @run main/othervm/native TestAllocatingInVMDeath
+ */
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+
+import jdk.test.lib.Asserts;
+import jdk.test.lib.process.OutputAnalyzer;
+import jdk.test.lib.process.ProcessTools;
+import jdk.test.whitebox.WhiteBox;
+
+public class TestAllocatingInVMDeath {
+    public static String UPCALL_MARKER = "Hello from upCall. ";
+
+    public static void main(String[] args) throws Exception {
+    ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder(
+                "-agentlib:TestAllocatingInVMDeath",
+                "--enable-native-access=ALL-UNNAMED",
+                "-Xbootclasspath/a:.",
+                "-XX:+UnlockDiagnosticVMOptions",
+                "-XX:+WhiteBoxAPI",
+                "DoWork");
+
+        OutputAnalyzer oa = new OutputAnalyzer(pb.start());
+        String output = oa.getOutput();
+        System.err.println("DoWork output:");
+        System.err.println(output);
+        Asserts.assertTrue(oa.getExitValue() == 0);
+        Asserts.assertTrue(output.contains(UPCALL_MARKER));
+    }
+}
+
+class DoWork {
+    static final WhiteBox WHITE_BOX = WhiteBox.getWhiteBox();
+
+    // This method is called from VMDeath event callback.
+    static void upCall() {
+        // Do some work including memory allocation, so it can't be optimized by compiler.
+        LocalDateTime now = LocalDateTime.now();
+        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS");
+        String result = TestAllocatingInVMDeath.UPCALL_MARKER + now.format(formatter);
+        WHITE_BOX.fullGC();
+        System.out.println(result);
+    }
+
+    public static void main(String argv[]) throws Exception {
+        System.out.println("Hello from DoWork main().");
+    }
+}

test/hotspot/jtreg/serviceability/jvmti/events/VMDeath/AllocatingInVMDeath/libTestAllocatingInVMDeath.cpp

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "jvmti.h"
+#include "jni.h"
+#include "jvmti_common.hpp"
+
+static void JNICALL
+cbVMDeath(jvmtiEnv* jvmti, JNIEnv* jni) {
+  jclass clz = jni->FindClass("DoWork");
+  if (clz == nullptr) {
+    fatal(jni, "Can't find DoWork class.");
+    return;
+  }
+  jmethodID mid = jni->GetStaticMethodID(clz, "upCall", "()V");
+  if (mid == nullptr) {
+    fatal(jni, "Can't find upCall method.");
+  }
+  jni->CallStaticObjectMethod(clz, mid);
+}
+
+JNIEXPORT jint JNICALL
+Agent_OnLoad(JavaVM *vm, char *options, void *reserved) {
+  jvmtiEnv *jvmti = nullptr;
+  jint res = vm->GetEnv((void **) &jvmti, JVMTI_VERSION_21);
+  if (res != JNI_OK) {
+    return JNI_ERR;
+  }
+  jvmtiError err = JVMTI_ERROR_NONE;
+
+  jvmtiEventCallbacks callbacks;
+  (void) memset(&callbacks, 0, sizeof (callbacks));
+  callbacks.VMDeath = &cbVMDeath;
+  err = jvmti->SetEventCallbacks(&callbacks, (int) sizeof (jvmtiEventCallbacks));
+  check_jvmti_error(err, "SetEventCallbacks");
+  err = jvmti->SetEventNotificationMode(JVMTI_ENABLE, JVMTI_EVENT_VM_DEATH, nullptr);
+  check_jvmti_error(err, "SetEventNotificationMode");
+  return JNI_OK;
+}