[stdlib] Fix the String.decodeCString for UTF16 and UTF32 (#2681)

[stdlib] Fix the `String.decodeCString` for UTF16 and UTF32

Resolves [SR-1578](https://bugs.swift.org/browse/SR-1578]
Essentially the problem was that `strlen` is not the right way of
obtaining a length of anything but null-terminated UTF-8 sequence of
characters. Other encodings require alternative mechanisms.

Author: moiseev

stdlib/public/core/CString.swift

@@ -137,7 +137,7 @@ extension String {
     guard let cString = cString else {
       return nil
     }
-    let len = Int(_swift_stdlib_strlen(UnsafePointer(cString)))
+    let len = encoding._nullCodeUnitOffset(in: cString)
     let buffer = UnsafeBufferPointer<Encoding.CodeUnit>(
       start: cString, count: len)
 

stdlib/public/core/Unicode.swift

@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+import SwiftShims
 
 // Conversions between different Unicode encodings.  Note that UTF-16 and
 // UTF-32 decoding are *not* currently resilient to erroneous data.
@@ -132,6 +133,12 @@ public protocol UnicodeCodec {
     _ input: UnicodeScalar,
     sendingOutputTo processCodeUnit: @noescape (CodeUnit) -> Void
   )
+
+  /// Searches for the first occurrence of a `CodeUnit` that is equal to 0.
+  ///
+  /// Is an equivalent of `strlen` for C-strings.
+  /// - Complexity: O(n)
+  static func _nullCodeUnitOffset(in input: UnsafePointer<CodeUnit>) -> Int
 }
 
 /// A codec for translating between Unicode scalar values and UTF-8 code
@@ -428,6 +435,10 @@ public struct UTF8 : UnicodeCodec {
   public static func isContinuation(_ byte: CodeUnit) -> Bool {
     return byte & 0b11_00__0000 == 0b10_00__0000
   }
+
+  public static func _nullCodeUnitOffset(in input: UnsafePointer<CodeUnit>) -> Int {
+    return Int(_swift_stdlib_strlen(UnsafePointer(input)))
+  }
 }
 
 /// A codec for translating between Unicode scalar values and UTF-16 code
@@ -1149,6 +1160,22 @@ extension UnicodeScalar {
   }
 }
 
+extension UnicodeCodec where CodeUnit : UnsignedInteger {
+  public static func _nullCodeUnitOffset(in input: UnsafePointer<CodeUnit>) -> Int {
+    var length = 0
+    while input[length] != 0 {
+      length += 1
+    }
+    return length
+  }
+}
+
+extension UnicodeCodec {
+  public static func _nullCodeUnitOffset(in input: UnsafePointer<CodeUnit>) -> Int {
+    fatalError("_nullCodeUnitOffset(in:) implementation should be provided")
+  }
+}
+
 @available(*, unavailable, renamed: "UnicodeCodec")
 public typealias UnicodeCodecType = UnicodeCodec
 

validation-test/stdlib/String.swift

@@ -1252,5 +1252,35 @@ StringTests.test("String.append(_: Character)") {
   }
 }
 
-runAllTests()
+internal func decodeCString<
+  C : UnicodeCodec
+  where
+  C.CodeUnit : UnsignedInteger
+>(_ s: String, as codec: C.Type)
+  -> (result: String, repairsMade: Bool)? {
+  let units = s.unicodeScalars.map({ $0.value }) + [0]
+  return units.map({ C.CodeUnit(numericCast($0)) }).withUnsafeBufferPointer {
+    String.decodeCString($0.baseAddress, as: C.self)
+  }
+}
 
+StringTests.test("String.decodeCString/UTF8") {
+  let actual = decodeCString("foobar", as: UTF8.self)
+  expectFalse(actual!.repairsMade)
+  expectEqual("foobar", actual!.result)
+}
+
+StringTests.test("String.decodeCString/UTF16") {
+  let actual = decodeCString("foobar", as: UTF16.self)
+  expectFalse(actual!.repairsMade)
+  expectEqual("foobar", actual!.result)
+}
+
+StringTests.test("String.decodeCString/UTF32") {
+  let actual = decodeCString("foobar", as: UTF32.self)
+  expectFalse(actual!.repairsMade)
+  expectEqual("foobar", actual!.result)
+}
+
+
+runAllTests()

validation-test/stdlib/Unicode.swift renamed to validation-test/stdlib/Unicode.swift.gyb

@@ -1,7 +1,6 @@
-// RUN: rm -rf %t
-// RUN: mkdir -p %t
-// RUN: %target-build-swift %s -o %t/a.out
-// RUN: %target-run %t/a.out
+// RUN: rm -rf %t && mkdir -p %t && %S/../../utils/gyb %s -o %t/Unicode.swift
+// RUN: %S/../../utils/line-directive %t/Unicode.swift -- %target-build-swift %t/Unicode.swift -o %t/a.out -Xfrontend -disable-objc-attr-requires-foundation-module
+// RUN: %S/../../utils/line-directive %t/Unicode.swift -- %target-run %t/a.out
 // REQUIRES: executable_test
 
 import SwiftPrivate
@@ -2464,5 +2463,31 @@ StringTests.test("StreamableConformance") {
 
 #endif // _runtime(_ObjC)
 
+let nullOffsetTests = [
+  (input: "\0", expected: 0),
+  (input: "a\0", expected: 1),
+  (input: "foobar\0", expected: 6)
+]
+
+let NullCodeUnitOffsetTests = TestSuite("NullCodeUnitOffsetTests")
+
+% for (Encoding, View) in [
+%   ('UTF8', 'utf8'),
+%   ('UTF16', 'utf16'),
+%   ('UTF32', 'unicodeScalars.map{ $0.value }')
+% ]:
+
+NullCodeUnitOffsetTests.test("${Encoding}._nullCodeUnitOffset(in:)") {
+  for test in nullOffsetTests {
+    let actual = Array(test.input.${View})
+      .withUnsafeBufferPointer { p in
+        ${Encoding}._nullCodeUnitOffset(in: p.baseAddress!)
+      }
+    expectEqual(test.expected, actual)
+  }
+}
+
+% end
+
 runAllTests()