[stdlib]Unify String hashing implementation (swiftlang#14921)

* Add partial range subscripts to _UnmanagedOpaqueString

* Use SipHash13+_NormalizedCodeUnitIterator for String hashes on all platforms

* Remove unecessary collation algorithm shims

* Pass the buffer to the SipHasher for ASCII

* Hash the ascii parts of UTF16 strings the same way we hash pure ascii strings

* De-dupe some code that can be shared between _UnmanagedOpaqueString and _UnmanagedString<UInt16>

* ASCII strings now hash consistently for in hashASCII() and hashUTF16()

* Fix zalgo comparison regression

* Use hasher

* Fix crash when appending to an empty _FixedArray

* Compact ASCII characters into a single UInt64 for hashing

* String: Switch to _hash(into:)-based hashing

This should speed up String hashing quite a bit, as doing it through hashValue involves two rounds of SipHash nested in each other.

* Remove obsolete workaround for ARC traffic

* Ditch _FixedArray<UInt8> in favor of _UIntBuffer<UInt64, UInt8>

* Bad rebase remnants

* Fix failing benchmarks

* michael's feedback

* clarify the comment about nul-terminated string hashes

Author: Lance Parker

stdlib/public/SwiftShims/UnicodeShims.h

@@ -62,22 +62,6 @@ SWIFT_RUNTIME_STDLIB_INTERFACE
 const __swift_uint16_t *
 _swift_stdlib_ExtendedGraphemeClusterNoBoundaryRulesMatrix;
 
-SWIFT_RUNTIME_STDLIB_INTERFACE
-void *_swift_stdlib_unicodeCollationIterator_create(
-    const __swift_uint16_t *Str,
-    __swift_uint32_t Length);
-
-SWIFT_RUNTIME_STDLIB_INTERFACE
-__swift_int32_t _swift_stdlib_unicodeCollationIterator_next(
-    void *CollationIterator, __swift_bool *HitEnd);
-
-SWIFT_RUNTIME_STDLIB_INTERFACE
-void _swift_stdlib_unicodeCollationIterator_delete(
-    void *CollationIterator);
-
-SWIFT_RUNTIME_STDLIB_INTERFACE
-const __swift_int32_t *_swift_stdlib_unicode_getASCIICollationTable();
-
 SWIFT_RUNTIME_STDLIB_INTERFACE
 __swift_int32_t _swift_stdlib_unicode_strToUpper(
   __swift_uint16_t *Destination, __swift_int32_t DestinationCapacity,

stdlib/public/core/FixedArray.swift.gyb

@@ -127,8 +127,8 @@ extension _FixedArray${N} {
   @_versioned
   internal mutating func append(_ newElement: T) {
     _sanityCheck(count < capacity)
-    self[count] = newElement
     _count += 1
+    self[count-1] = newElement
   }
 }
 

stdlib/public/core/NormalizedCodeUnitIterator.swift

@@ -23,12 +23,11 @@ struct _NormalizedCodeUnitIterator: IteratorProtocol {
 
   typealias CodeUnit = UInt16
 
-  init(_ opaqueString: _UnmanagedOpaqueString, startIndex: Int = 0) {
-    source = _UnmanagedOpaqueStringSource(opaqueString, start: startIndex)
-  }
-  
-  init(_ unmanagedString: _UnmanagedString<UInt16>, startIndex: Int = 0) {
-    source = _UnmanagedStringSource(unmanagedString, start: startIndex)
+  init<Source: BidirectionalCollection>
+    (_ collection: Source)
+    where Source.Element == UInt16, Source.SubSequence == Source
+  {
+    source = _CollectionSource(collection)
   }
 
   init(_ guts: _StringGuts, _ range: Range<Int>, startIndex: Int = 0) {
@@ -60,23 +59,27 @@ struct _NormalizedCodeUnitIterator: IteratorProtocol {
     }
   }
 
-  struct _UnmanagedOpaqueStringSource: _SegmentSource {
+  struct _CollectionSource<Source: BidirectionalCollection>: _SegmentSource
+    where Source.Element == UInt16, Source.SubSequence == Source
+  {
     var remaining: Int {
-      return opaqueString.count - index
+      return collection.distance(from: index, to: collection.endIndex)
     }
-    var opaqueString: _UnmanagedOpaqueString
-    var index: Int
+    var collection: Source
+    var index: Source.Index
 
-    init(_ opaqueString: _UnmanagedOpaqueString, start: Int = 0) {
-      self.opaqueString = opaqueString
-      index = start
+    init(_ collection: Source) {
+      self.collection = collection
+      index = collection.startIndex
     }
 
+    @_specialize(where Source == _UnmanagedString<UInt16>)
+    @_specialize(where Source == _UnmanagedOpaqueString)
     mutating func tryFill(buffer: UnsafeMutableBufferPointer<UInt16>) -> Int? {
       var bufferIndex = 0
       let originalIndex = index
       repeat {
-        guard index < opaqueString.count else {
+        guard index != collection.endIndex else {
           break
         }
 
@@ -86,49 +89,11 @@ struct _NormalizedCodeUnitIterator: IteratorProtocol {
           return nil
         }
 
-        let cu = opaqueString[index]
+        let cu = collection[index]
         buffer[bufferIndex] = cu
-        index += 1
+        index = collection.index(after: index)
         bufferIndex += 1
-      } while !opaqueString.hasNormalizationBoundary(after: index - 1)
-      
-      return bufferIndex
-    }
-  }
-  
-  struct _UnmanagedStringSource: _SegmentSource {
-    var remaining: Int {
-      return unmanagedString.count - index
-    }
-    
-    var unmanagedString: _UnmanagedString<UInt16>
-    var index: Int
-    
-    init(_ unmanagedString: _UnmanagedString<UInt16>, start: Int = 0) {
-      self.unmanagedString = unmanagedString
-      index = start
-    }
-    
-    mutating func tryFill(buffer: UnsafeMutableBufferPointer<UInt16>) -> Int? {
-      var bufferIndex = 0
-      let originalIndex = index
-      repeat {
-        guard index < unmanagedString.count else {
-          break
-        }
-        
-        guard bufferIndex < buffer.count else {
-          //The buffer isn't big enough for the current segment
-          index = originalIndex
-          return nil
-        }
-        
-        let cu = unmanagedString[index]
-        buffer[bufferIndex] = cu
-        index += 1
-        bufferIndex += 1
-      } while unmanagedString.hasNormalizationBoundary(
-          after: index - 1) == false
+      } while !collection.hasNormalizationBoundary(after: collection.index(before: index))
 
       return bufferIndex
     }
@@ -266,4 +231,4 @@ extension _SegmentSource {
   ) -> Int? {
     return tryFill(buffer: _castOutputBuffer(buffer))
   }
-}
+}