[stdlib] _StringObject.isKnownUTF16 → isForeignUTF8

This fixes a compatibility issue with potential future UTF-8 encoded
foreign String forms, as well as simplifying the code a bit — we no
longer need to do an availability check on inlinable fast paths.

The isForeignUTF8 bit is never set by any past or current stdlib
version, but it allows us to introduce UTF-8 encoded foreign forms
without breaking inlinable index encoding validation introduced in
Swift 5.7.

Author: lorentey

stdlib/public/core/StringBridge.swift

@@ -616,8 +616,7 @@ extension String {
       let gutsCountAndFlags = _guts._object._countAndFlags
       let countAndFlags = _StringObject.CountAndFlags(
         sharedCount: _guts.count,
-        isASCII: gutsCountAndFlags.isASCII,
-        isUTF16: false)
+        isASCII: gutsCountAndFlags.isASCII)
       return __SharedStringStorage(
         immortal: _guts._object.fastUTF8.baseAddress!,
         countAndFlags: countAndFlags)

stdlib/public/core/StringCharacterView.swift

@@ -55,17 +55,14 @@ extension String: BidirectionalCollection {
   /// - Returns: The index value immediately after `i`.
   public func index(after i: Index) -> Index {
     let i = _guts.roundDownToNearestCharacter(_guts.validateScalarIndex(i))
-    let r = _uncheckedIndex(after: i)
-    return _guts.internalMarkEncoding(r)
+    return _uncheckedIndex(after: i)
   }
 
   /// A version of `index(after:)` that assumes that the given index:
   ///
   /// - has the right encoding,
   /// - is within bounds, and
   /// - is scalar aligned.
-  ///
-  /// It does not mark the encoding of the returned index.
   internal func _uncheckedIndex(after i: Index) -> Index {
     _internalInvariant(_guts.hasMatchingEncoding(i))
     _internalInvariant(i < endIndex)
@@ -77,7 +74,7 @@ extension String: BidirectionalCollection {
     let nextIndex = Index(_encodedOffset: nextOffset)._characterAligned
     let nextStride = _characterStride(startingAt: nextIndex)
     let r = Index(encodedOffset: nextOffset, characterStride: nextStride)
-    return _guts.internalMarkEncoding(r._characterAligned)
+    return _guts.markEncoding(r._characterAligned)
   }
 
   /// Returns the position immediately before the given index.
@@ -92,17 +89,14 @@ extension String: BidirectionalCollection {
     // the `i > startIndex` check needs to come after rounding.
     _precondition(i > startIndex, "String index is out of bounds")
 
-    let r = _uncheckedIndex(before: i)
-    return _guts.internalMarkEncoding(r)
+    return _uncheckedIndex(before: i)
   }
 
   /// A version of `index(before:)` that assumes that the given index:
   ///
   /// - has the right encoding,
   /// - is within bounds, and
   /// - is character aligned.
-  ///
-  /// It does not mark the encoding of the returned index.
   internal func _uncheckedIndex(before i: Index) -> Index {
     _internalInvariant(_guts.hasMatchingEncoding(i))
     _internalInvariant(i > startIndex && i <= endIndex)
@@ -113,7 +107,7 @@ extension String: BidirectionalCollection {
     let priorOffset = i._encodedOffset &- stride
 
     let r = Index(encodedOffset: priorOffset, characterStride: stride)
-    return r._characterAligned
+    return _guts.markEncoding(r._characterAligned)
   }
 
   /// Returns an index that is the specified distance from the given index.
@@ -158,7 +152,7 @@ extension String: BidirectionalCollection {
         i = _uncheckedIndex(before: i)
       }
     }
-    return _guts.internalMarkEncoding(i)
+    return i
   }
 
   /// Returns an index that is the specified distance from the given index,
@@ -238,7 +232,7 @@ extension String: BidirectionalCollection {
       }
       guard limit > start || i >= limit else { return nil }
     }
-    return _guts.internalMarkEncoding(i)
+    return i
   }
 
   /// Returns the distance between two indices.

stdlib/public/core/StringGuts.swift

@@ -311,55 +311,36 @@ func _isSwiftStdlib_5_7() -> Bool {
 
 // Encoding
 extension _StringGuts {
-  /// Returns whether this string is known to use UTF-16 code units.
+  /// Returns whether this string has a UTF-8 storage representation.
   ///
-  /// This always returns a value corresponding to the string's actual encoding
-  /// on stdlib versions >=5.7.
+  /// This always returns a value corresponding to the string's actual encoding.
+  @_alwaysEmitIntoClient
+  @inline(__always)
+  internal var isUTF8: Bool { _object.isUTF8 }
+
+  /// Returns whether this string has a UTF-16 storage representation.
   ///
-  /// Standard Library versions <=5.6 did not set the corresponding flag, so
-  /// this property always returns false.
+  /// This always returns a value corresponding to the string's actual encoding.
   @_alwaysEmitIntoClient
   @inline(__always)
-  internal var isKnownUTF16: Bool { _object.isKnownUTF16 }
+  internal var isUTF16: Bool { _object.isUTF16 }
 
   @_alwaysEmitIntoClient // Swift 5.7
   internal func markEncoding(_ i: String.Index) -> String.Index {
-    // In this inlinable function, we cannot assume that all foreign strings are
-    // UTF-16 encoded, as this code may run on a future stdlib that may have
-    // introduced other foreign forms.
-    if #available(macOS 9999, iOS 9999, watchOS 9999, tvOS 9999, *) { // SwiftStdlib 5.7
-      // With a >=5.7 stdlib, we can rely on `isKnownUTF16` to contain the truth.
-      return isKnownUTF16 ? i._knownUTF16 : i._knownUTF8
-    }
-    // We know that in stdlibs 5.0..<5.7, all foreign strings were UTF-16,
-    // so we can use `isForeign` to determine the encoding.
-    return isForeign ? i._knownUTF16 : i._knownUTF8
-  }
-
-  @inline(__always)
-  internal func internalMarkEncoding(_ i: String.Index) -> String.Index {
-    // This code is behind a resiliance boundary, so it always runs on a >=5.7
-    // stdlib. Note though that it doesn't match the 5.7+ case in the inlinable
-    // version above!
-    //
-    // We know that in this version of the stdlib, foreign strings happen to
-    // always be UTF-16 encoded (like they were between 5.0 and 5.6), and
-    // looking at `isForeign` instead of `isKnownUTF16` may allow the stdlib's
-    // internal code to be better optimized -- so let's do that.
-    isForeign ? i._knownUTF16 : i._knownUTF8
+    isUTF8 ? i._knownUTF8 : i._knownUTF16
   }
 
   /// Returns true if the encoding of the given index isn't known to be in
   /// conflict with this string's encoding.
   ///
-  /// If the index or the string was created by code that was built on stdlibs
-  /// below 5.7, then this check may incorrectly return true on a mismatching
-  /// index, but it is guaranteed to never incorrectly return false. If all
-  /// loaded binaries were built in 5.7+, then this method is guaranteed to
-  /// always return the correct value.
-  @_alwaysEmitIntoClient
+  /// If the index was created by code that was built on a stdlib below 5.7,
+  /// then this check may incorrectly return true on a mismatching index, but it
+  /// is guaranteed to never incorrectly return false. If all loaded binaries
+  /// were built in 5.7+, then this method is guaranteed to always return the
+  /// correct value.
+  @_alwaysEmitIntoClient @inline(__always)
   internal func hasMatchingEncoding(_ i: String.Index) -> Bool {
-    (isForeign && i._canBeUTF16) || (!isForeign && i._canBeUTF8)
+    isUTF8 ? i._canBeUTF8 : i._canBeUTF16
   }
 
   /// Return an index whose encoding can be assumed to match that of `self`.
@@ -371,22 +352,20 @@ extension _StringGuts {
   @_alwaysEmitIntoClient
   @inline(__always)
   internal func ensureMatchingEncoding(_ i: String.Index) -> String.Index {
-    if _fastPath(!isForeign && i._canBeUTF8) { return i }
+    if _fastPath(hasMatchingEncoding(i)) { return i }
     return _slowEnsureMatchingEncoding(i)
   }
 
   @_alwaysEmitIntoClient
   @inline(never)
   internal func _slowEnsureMatchingEncoding(_ i: String.Index) -> String.Index {
-    _internalInvariant(isForeign || !i._canBeUTF8)
-    if isForeign {
-      // Opportunistically detect attempts to use an UTF-8 index on a UTF-16
-      // string. Strings don't usually get converted to UTF-16 storage, so it
-      // seems okay to trap in this case -- the index most likely comes from an
-      // unrelated string. (Trapping here may still turn out to affect binary
-      // compatibility with broken code in existing binaries running with new
-      // stdlibs. If so, we can replace this with the same transcoding hack as
-      // in the UTF-16->8 case below.)
+    guard isUTF8 else {
+      // Attempt to use an UTF-8 index on a UTF-16 string. Strings don't usually
+      // get converted to UTF-16 storage, so it seems okay to trap in this case
+      // -- the index most likely comes from an unrelated string. (Trapping here
+      // may still turn out to affect binary compatibility with broken code in
+      // existing binaries running with new stdlibs. If so, we can replace this
+      // with the same transcoding hack as in the UTF-16->8 case below.)
       //
       // Note that this trap is not guaranteed to trigger when the process
       // includes client binaries compiled with a previous Swift release.
@@ -397,13 +376,9 @@ extension _StringGuts {
       //
       // This trap can never trigger on OSes that have stdlibs <= 5.6, because
       // those versions never set the `isKnownUTF16` flag in `_StringObject`.
-      //
-      _precondition(!isKnownUTF16 || i._canBeUTF16,
-        "Invalid string index")
-      return i
+      _preconditionFailure("Invalid string index")
     }
-    // If we get here, then we know for sure that this is an attempt to use an
-    // UTF-16 index on a UTF-8 string.
+    // Attempt to use an UTF-16 index on a UTF-8 string.
     //
     // This can happen if `self` was originally verbatim-bridged, and someone
     // mistakenly attempts to keep using an old index after a mutation. This is

stdlib/public/core/StringGutsRangeReplaceable.swift

@@ -466,7 +466,7 @@ extension _StringGuts {
     _internalInvariant(
       subrange.lowerBound >= startIndex && subrange.upperBound <= endIndex)
 
-    if _slowPath(isKnownUTF16) {
+    if _slowPath(isUTF16) {
       // UTF-16 (i.e., foreign) string. The mutation will convert this to the
       // native UTF-8 encoding, so we need to do some extra work to preserve our
       // bounds.
@@ -479,7 +479,7 @@ extension _StringGuts {
         from: subrange.lowerBound, to: subrange.upperBound)
 
       let newUTF8Subrange = body(&self)
-      _internalInvariant(!isKnownUTF16)
+      _internalInvariant(isUTF8)
 
       let newUTF8Count =
         oldUTF8Count + newUTF8Subrange.count - oldUTF8SubrangeCount

stdlib/public/core/StringIndex.swift

@@ -364,14 +364,14 @@ extension String.Index {
 // this way: position zero is the same no matter how what encoding is used for
 // the rest of string.)
 //
-// These two bits (along with the isKnownUTF16 flag in StringObject) allow newer
-// versions of the Standard Library to more reliably catch runtime errors where
-// client code is applying an index from a UTF-16 string to a UTF-8 one, or vice
-// versa. This typically happens when indices from a UTF-16 Cocoa string that
-// was verbatim bridged into Swift are accidentally applied to a mutated version
-// of the same string. (The mutation turns it into a UTF-8 native string, where
-// the same numerical offsets might correspond to wildly different logical
-// positions.)
+// These two bits (along with the isForeignUTF8 flag in StringObject) allow
+// newer versions of the Standard Library to more reliably catch runtime errors
+// where client code is applying an index from a UTF-16 string to a UTF-8 one,
+// or vice versa. This typically happens when indices from a UTF-16 Cocoa string
+// that was verbatim bridged into Swift are accidentally applied to a mutated
+// version of the same string. (The mutation turns it into a UTF-8 native
+// string, where the same numerical offsets might correspond to wildly different
+// logical positions.)
 //
 // Such code has always been broken, as the old indices are documented to be no
 // longer valid after the mutation; however, in previous releases such cases