Merged PR 100702: Move to marvin hashing for OrdinalIgnoreCase CompareInfo & StringComparers

Move to marvin hashing for OrdinalIgnoreCase CompareInfo & StringComparers.

Author: Viktor Hofer

src/classlibnative/bcltype/stringnative.cpp

@@ -182,6 +182,25 @@ FCIMPL3(INT32, COMString::Marvin32HashString, StringObject* thisRefUNSAFE, INT32
 }
 FCIMPLEND
 
+FCIMPL2(INT32, COMString::Marvin32HashPtr, WCHAR *pRawStr, INT32 strLen) {
+    FCALL_CONTRACT;
+
+    int iReturnHash = 0;
+
+    if (pRawStr == NULL) {
+        FCThrow(kNullReferenceException);
+    }
+
+    BEGIN_SO_INTOLERANT_CODE_NOTHROW(GetThread(), FCThrow(kStackOverflowException));
+    iReturnHash = GetCurrentNlsHashProvider()->HashString(pRawStr, strLen, TRUE, 0);
+    END_SO_INTOLERANT_CODE;
+
+    FC_GC_POLL_RET();
+
+    return iReturnHash;
+}
+FCIMPLEND
+
 BOOL QCALLTYPE COMString::UseRandomizedHashing() {
     QCALL_CONTRACT;
 

src/classlibnative/bcltype/stringnative.h

@@ -87,6 +87,7 @@ class COMString {
 
 #ifdef FEATURE_RANDOMIZED_STRING_HASHING
     static FCDECL3(INT32, Marvin32HashString, StringObject* thisRefUNSAFE, INT32 strLen, INT64 additionalEntropy);
+    static FCDECL2(INT32, Marvin32HashPtr, WCHAR *pRawStr, INT32 strLen);
     static BOOL QCALLTYPE UseRandomizedHashing();
 #endif // FEATURE_RANDOMIZED_STRING_HASHING
 

src/mscorlib/shared/System/StringComparer.cs

@@ -287,7 +287,7 @@ public override int GetHashCode(string obj)
 
             if (_ignoreCase)
             {
-                return TextInfo.GetHashCodeOrdinalIgnoreCase(obj);
+                return CompareInfo.Invariant.GetHashCode(obj, CompareOptions.OrdinalIgnoreCase);
             }
 
             return obj.GetHashCode();
@@ -363,7 +363,7 @@ public override int GetHashCode(string obj)
                 throw new ArgumentNullException(nameof(obj));
 #endif
             }
-            return TextInfo.GetHashCodeOrdinalIgnoreCase(obj);
+            return CompareInfo.Invariant.GetHashCode(obj, CompareOptions.OrdinalIgnoreCase);
         }
 
         public void GetObjectData(SerializationInfo info, StreamingContext context)

src/mscorlib/src/System/Globalization/CompareInfo.cs

@@ -57,6 +57,9 @@ public partial class CompareInfo : IDeserializationCallback
             ~(CompareOptions.IgnoreCase | CompareOptions.IgnoreSymbols | CompareOptions.IgnoreNonSpace |
               CompareOptions.IgnoreWidth | CompareOptions.IgnoreKanaType | CompareOptions.StringSort);
 
+        // We cache the invariant compareinfo as we need it for OrdinalIgnoreCase hashing
+        internal static readonly CompareInfo Invariant = CultureInfo.InvariantCulture.CompareInfo;
+
         //
         // CompareInfos have an interesting identity.  They are attached to the locale that created them,
         // ie: en-US would have an en-US sort.  For haw-US (custom), then we serialize it as haw-US.
@@ -1181,21 +1184,67 @@ internal int GetHashCodeOfString(string source, CompareOptions options)
             return GetHashCodeOfStringCore(source, options);
         }
 
+        private unsafe int GetSmallAsciiStringHash(string source)
+        {
+            Debug.Assert(source.Length <= 250, "Input string is too long");
+
+            // Do not allocate on the stack if string is empty
+            if (source.Length == 0)
+            {
+                return source.GetHashCode();
+            }
+
+            char* charArr = stackalloc char[source.Length];
+            char c;
+            for (int i = 0; i < source.Length; i++)
+            {
+                c = source[i];
+
+                // If we have a lowercase character, ANDing off 0x20
+                // will make it an uppercase character.
+                if ((c - 'a') <= ('z' - 'a'))
+                {
+                    c = (char)(c & ~0x20);
+                }
+
+                charArr[i] = c;
+            }
+
+            return String.InternalMarvin32HashPtr(charArr, source.Length);
+        }
+
         public virtual int GetHashCode(string source, CompareOptions options)
         {
             if (source == null)
             {
                 throw new ArgumentNullException(nameof(source));
             }
 
+            if (_invariantMode)
+            {
+                // If invariant mode enabled we ignore all compare options except *IgnoreCase.
+                if ((options & (CompareOptions.IgnoreCase | CompareOptions.OrdinalIgnoreCase)) != 0)
+                {
+                    // For small strings we allocate on the stack
+                    if (source.Length <= 250)
+                        return GetSmallAsciiStringHash(source);
+                    else
+                        return TextInfo.Invariant.ToUpper(source).GetHashCode();
+                }
+
+                return source.GetHashCode();
+            }
+
             if (options == CompareOptions.Ordinal)
             {
                 return source.GetHashCode();
             }
 
             if (options == CompareOptions.OrdinalIgnoreCase)
             {
-                return TextInfo.GetHashCodeOrdinalIgnoreCase(source);
+                // We use native marvin hashing to avoid hash collisions. We are passing
+                // IgnoreCase as GetHashCodeOfStringCore can't handle OrdinalIgnoreCase.
+                return Invariant.GetHashCodeOfStringCore(source, CompareOptions.IgnoreCase);
             }
 
             //

src/mscorlib/src/System/Globalization/TextInfo.cs

@@ -101,13 +101,6 @@ void IDeserializationCallback.OnDeserialization(Object sender)
         // Internal ordinal comparison functions
         //
 
-        internal static int GetHashCodeOrdinalIgnoreCase(String s)
-        {
-            // This is the same as an case insensitive hash for Invariant
-            // (not necessarily true for sorting, but OK for casing & then we apply normal hash code rules)
-            return (Invariant.GetCaseInsensitiveHashCode(s));
-        }
-
         // Currently we don't have native functions to do this, so we do it the hard way
         internal static int IndexOfStringOrdinalIgnoreCase(String source, String value, int startIndex, int count)
         {
@@ -789,64 +782,5 @@ private static bool IsLetterCategory(UnicodeCategory uc)
                  || uc == UnicodeCategory.ModifierLetter
                  || uc == UnicodeCategory.OtherLetter);
         }
-
-        //
-        // Get case-insensitive hash code for the specified string.
-        //
-        internal unsafe int GetCaseInsensitiveHashCode(String str)
-        {
-            // Validate inputs
-            if (str == null)
-            {
-                throw new ArgumentNullException(nameof(str));
-            }
-
-            // This code assumes that ASCII casing is safe for whatever context is passed in.
-            // this is true today, because we only ever call these methods on Invariant.  It would be ideal to refactor
-            // these methods so they were correct by construction and we could only ever use Invariant.
-
-            uint hash = 5381;
-            uint c;
-
-            // Note: We assume that str contains only ASCII characters until
-            // we hit a non-ASCII character to optimize the common case.
-            for (int i = 0; i < str.Length; i++)
-            {
-                c = str[i];
-                if (c >= 0x80)
-                {
-                    return GetCaseInsensitiveHashCodeSlow(str);
-                }
-
-                // If we have a lowercase character, ANDing off 0x20
-                // will make it an uppercase character.
-                if ((c - 'a') <= ('z' - 'a'))
-                {
-                    c = (uint)((int)c & ~0x20);
-                }
-
-                hash = ((hash << 5) + hash) ^ c;
-            }
-
-            return (int)hash;
-        }
-
-        private unsafe int GetCaseInsensitiveHashCodeSlow(String str)
-        {
-            Debug.Assert(str != null);
-
-            string upper = ToUpper(str);
-
-            uint hash = 5381;
-            uint c;
-
-            for (int i = 0; i < upper.Length; i++)
-            {
-                c = upper[i];
-                hash = ((hash << 5) + hash) ^ c;
-            }
-
-            return (int)hash;
-        }
     }
 }

src/mscorlib/src/System/String.Comparison.cs

@@ -1014,6 +1014,9 @@ public static bool Equals(String a, String b, StringComparison comparisonType)
         [MethodImplAttribute(MethodImplOptions.InternalCall)]
         internal static extern int InternalMarvin32HashString(string s, int strLen, long additionalEntropy);
 
+        [MethodImplAttribute(MethodImplOptions.InternalCall)]
+        internal unsafe static extern int InternalMarvin32HashPtr(char* s, int strLen);
+
         internal static bool UseRandomizedHashing()
         {
             return InternalUseRandomizedHashing();

src/vm/ecalllist.h

@@ -127,6 +127,7 @@ FCFuncStart(gStringFuncs)
 #endif // FEATURE_COMINTEROP
 #ifdef FEATURE_RANDOMIZED_STRING_HASHING
     FCFuncElement("InternalMarvin32HashString", COMString::Marvin32HashString)
+    FCFuncElement("InternalMarvin32HashPtr", COMString::Marvin32HashPtr)
     QCFuncElement("InternalUseRandomizedHashing", COMString::UseRandomizedHashing) 
 #endif // FEATURE_RANDOMIZED_STRING_HASHING
 FCFuncEnd()