Fix CoreCLR debug launch (#19730) (#19770)

Author: Mike McLaughlin

src/dlls/dbgshim/dbgshim.cpp

@@ -1054,26 +1054,27 @@ IsCoreClrWithGoodHeader(
 }
 
 static
-HMODULE*
+HRESULT
 EnumProcessModulesInternal(
     HANDLE hProcess, 
-    DWORD *pCountModules)
-{
+    DWORD *pCountModules,
+    HMODULE** ppModules)
+{    
     *pCountModules = 0;
+    *ppModules = nullptr;
 
     // Start with 1024 modules
     DWORD cbNeeded = sizeof(HMODULE) * 1024;
 
     ArrayHolder<HMODULE> modules = new (nothrow) HMODULE[cbNeeded / sizeof(HMODULE)];
     if (modules == nullptr)
     {
-        SetLastError(ERROR_OUTOFMEMORY);
-        return nullptr;
+        return HRESULT_FROM_WIN32(ERROR_OUTOFMEMORY);
     }
 
     if(!EnumProcessModules(hProcess, modules, cbNeeded, &cbNeeded))
     {
-        return nullptr;
+        return HRESULT_FROM_WIN32(GetLastError());
     }
 
     // If 1024 isn't enough, try the modules array size returned (cbNeeded)
@@ -1082,14 +1083,13 @@ EnumProcessModulesInternal(
         modules = new (nothrow) HMODULE[cbNeeded / sizeof(HMODULE)];
         if (modules == nullptr)
         {
-            SetLastError(ERROR_OUTOFMEMORY);
-            return nullptr;
+            return HRESULT_FROM_WIN32(ERROR_OUTOFMEMORY);
         }
 
         DWORD cbNeeded2;
         if(!EnumProcessModules(hProcess, modules, cbNeeded, &cbNeeded2))
         {
-            return nullptr;
+            return HRESULT_FROM_WIN32(GetLastError());
         }
 
         // The only way cbNeeded2 could change on the second call is if number of
@@ -1101,7 +1101,8 @@ EnumProcessModulesInternal(
     }
 
     *pCountModules = cbNeeded / sizeof(HMODULE);
-    return modules.Detach();
+    *ppModules = modules.Detach();
+    return S_OK;
 }
 
 //-----------------------------------------------------------------------------
@@ -1143,10 +1144,11 @@ EnumerateCLRs(
 
     // The modules in the array returned don't need to be closed
     DWORD countModules;
-    ArrayHolder<HMODULE> modules = EnumProcessModulesInternal(hProcess, &countModules);
-    if (modules == nullptr)
+    ArrayHolder<HMODULE> modules = nullptr;
+    HRESULT hr = EnumProcessModulesInternal(hProcess, &countModules, &modules);
+    if (FAILED(hr))
     {
-        return HRESULT_FROM_WIN32(GetLastError());
+        return hr;
     }
 
     //
@@ -1321,10 +1323,11 @@ GetRemoteModuleBaseAddress(
 
     // The modules in the array returned don't need to be closed
     DWORD countModules;
-    ArrayHolder<HMODULE> modules = EnumProcessModulesInternal(hProcess, &countModules);
-    if (modules == nullptr)
+    ArrayHolder<HMODULE> modules = nullptr;
+    HRESULT hr = EnumProcessModulesInternal(hProcess, &countModules, &modules);    
+    if (FAILED(hr))
     {
-        ThrowHR(HRESULT_FROM_WIN32(GetLastError()));
+        ThrowHR(hr);
     }
 
     for(DWORD i = 0; i < countModules; i++)

src/inc/clrhost.h

@@ -44,7 +44,7 @@
 //
 #ifdef _DEBUG
 
-#define LAST_ERROR_TRASH_VALUE 42424
+#define LAST_ERROR_TRASH_VALUE 42424 /* = 0xa5b8 */
 
 #define TRASH_LASTERROR \
     SetLastError(LAST_ERROR_TRASH_VALUE)