8346388: Cannot use DllMain in libawt for static builds

Reviewed-by: aivanov, prr

Author: magicus

src/java.desktop/windows/native/libawt/windows/awt_Toolkit.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1996, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -102,17 +102,36 @@ extern void DWMResetCompositionEnabled();
    first loaded */
 JavaVM *jvm = NULL;
 
+/* Return a handle to the module containing this method, either a DLL in case
+ * of a dynamic library build, or the .EXE in case of a static build.
+ */
+static HMODULE GetAwtModuleHandle() {
+    HMODULE hModule = NULL;
+    GetModuleHandleEx(
+        GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
+        (LPCTSTR) &GetAwtModuleHandle,
+        &hModule
+    );
+    return hModule;
+}
+
+extern "C" {
+
 JNIEXPORT jint JNICALL
 DEF_JNI_OnLoad(JavaVM *vm, void *reserved)
 {
     TRY;
 
+    AwtToolkit::GetInstance().SetModuleHandle(GetAwtModuleHandle());
+
     jvm = vm;
     return JNI_VERSION_1_2;
 
     CATCH_BAD_ALLOC_RET(0);
 }
 
+}
+
 extern "C" JNIEXPORT jboolean JNICALL AWTIsHeadless() {
     static JNIEnv *env = NULL;
     static jboolean isHeadless;
@@ -246,32 +265,6 @@ BOOL AwtToolkit::activateKeyboardLayout(HKL hkl) {
     return (prev != 0);
 }
 
-/************************************************************************
- * Exported functions
- */
-
-extern "C" BOOL APIENTRY DllMain(HANDLE hInstance, DWORD ul_reason_for_call,
-                                 LPVOID)
-{
-    // Don't use the TRY and CATCH_BAD_ALLOC_RET macros if we're detaching
-    // the library. Doing so causes awt.dll to call back into the VM during
-    // shutdown. This crashes the HotSpot VM.
-    switch (ul_reason_for_call) {
-    case DLL_PROCESS_ATTACH:
-        TRY;
-        AwtToolkit::GetInstance().SetModuleHandle((HMODULE)hInstance);
-        CATCH_BAD_ALLOC_RET(FALSE);
-        break;
-    case DLL_PROCESS_DETACH:
-#ifdef DEBUG
-        DTrace_DisableMutex();
-        DMem_DisableMutex();
-#endif // DEBUG
-        break;
-    }
-    return TRUE;
-}
-
 /************************************************************************
  * AwtToolkit fields
  */