Port The Fix When Enabling Windows Compatibilty Mode (#26805)

Author: tarekgh

src/System.Private.CoreLib/shared/Interop/Windows/Kernel32/Interop.Globalization.cs

@@ -24,6 +24,9 @@ internal static unsafe partial class Kernel32
         internal const uint LOCALE_RETURN_NUMBER        = 0x20000000;
         internal const uint LOCALE_NOUSEROVERRIDE       = 0x80000000;
 
+        internal const uint LCMAP_SORTHANDLE            = 0x20000000;
+        internal const uint LCMAP_HASH                  = 0x00040000;
+
         internal const int  COMPARE_STRING              = 0x0001;
 
         internal const uint TIME_NOSECONDS = 0x00000002;
@@ -115,7 +118,7 @@ internal static extern unsafe bool IsNLSDefinedString(
 
         [DllImport("kernel32.dll", CharSet = CharSet.Unicode)]
         internal static extern bool EnumTimeFormatsEx(EnumTimeFormatsProcEx lpTimeFmtEnumProcEx, string lpLocaleName, uint dwFlags, void* lParam);
-  
+
         internal delegate BOOL EnumTimeFormatsProcEx(char* lpTimeFormatString, void* lParam);
 
         [DllImport("kernel32.dll", CharSet = CharSet.Unicode)]
@@ -126,7 +129,7 @@ internal static extern unsafe bool IsNLSDefinedString(
 
         [DllImport("kernel32.dll", CharSet = CharSet.Unicode)]
         internal static extern bool EnumCalendarInfoExEx(EnumCalendarInfoProcExEx pCalInfoEnumProcExEx, string lpLocaleName, uint Calendar, string? lpReserved, uint CalType, void* lParam);
-        
+
         internal delegate BOOL EnumCalendarInfoProcExEx(char* lpCalendarInfoString, uint Calendar, IntPtr lpReserved, void* lParam);
 
         [StructLayout(LayoutKind.Sequential)]
@@ -138,7 +141,7 @@ internal struct NlsVersionInfoEx
             internal int dwEffectiveId;
             internal Guid guidCustomVersion;
         }
-        
+
         [DllImport("kernel32.dll", CharSet = CharSet.Unicode)]
         internal static extern unsafe bool GetNLSVersionEx(int function, string localeName, NlsVersionInfoEx* lpVersionInformation);
     }

src/System.Private.CoreLib/shared/System/Globalization/CompareInfo.Windows.cs

@@ -12,22 +12,36 @@ namespace System.Globalization
 {
     public partial class CompareInfo
     {
-        private unsafe void InitSort(CultureInfo culture)
+        internal static unsafe IntPtr GetSortHandle(string cultureName)
         {
-            _sortName = culture.SortName;
-
             if (GlobalizationMode.Invariant)
             {
-                _sortHandle = IntPtr.Zero;
+                return IntPtr.Zero;
             }
-            else
-            {
-                const uint LCMAP_SORTHANDLE = 0x20000000;
 
-                IntPtr handle;
-                int ret = Interop.Kernel32.LCMapStringEx(_sortName, LCMAP_SORTHANDLE, null, 0, &handle, IntPtr.Size, null, null, IntPtr.Zero);
-                _sortHandle = ret > 0 ? handle : IntPtr.Zero;
+            IntPtr handle;
+            int ret = Interop.Kernel32.LCMapStringEx(cultureName, Interop.Kernel32.LCMAP_SORTHANDLE, null, 0, &handle, IntPtr.Size, null, null, IntPtr.Zero);
+            if (ret > 0)
+            {
+                // Even if we can get the sort handle, it is not guaranteed to work when Windows compatibility shim is applied
+                // e.g. Windows 7 compatibility mode. We need to ensure it is working before using it.
+                // otherwise the whole framework app will not start.
+                int hashValue = 0;
+                char a = 'a';
+                ret = Interop.Kernel32.LCMapStringEx(null, Interop.Kernel32.LCMAP_HASH, &a, 1, &hashValue, sizeof(int), null, null, handle);
+                if (ret > 1)
+                {
+                    return handle;
+                }
             }
+
+            return IntPtr.Zero;
+        }
+
+        private void InitSort(CultureInfo culture)
+        {
+            _sortName = culture.SortName;
+            _sortHandle = GetSortHandle(_sortName);
         }
 
         private static unsafe int FindStringOrdinal(

src/System.Private.CoreLib/shared/System/Globalization/TextInfo.Windows.cs

@@ -10,17 +10,7 @@ public partial class TextInfo
     {
         private unsafe void FinishInitialization()
         {
-            if (GlobalizationMode.Invariant)
-            {
-                _sortHandle = IntPtr.Zero;
-                return;
-            }
-
-            const uint LCMAP_SORTHANDLE = 0x20000000;
-
-            IntPtr handle;
-            int ret = Interop.Kernel32.LCMapStringEx(_textInfoName, LCMAP_SORTHANDLE, null, 0, &handle, IntPtr.Size, null, null, IntPtr.Zero);
-            _sortHandle = ret > 0 ? handle : IntPtr.Zero;
+            _sortHandle = CompareInfo.GetSortHandle(_textInfoName);
         }
 
         private unsafe void ChangeCase(char* pSource, int pSourceLen, char* pResult, int pResultLen, bool toUpper)