Implement isoLatin1 and macOSRoman encoding (#743)

jmschonfeld · web-flow · commit acae3d26b691 · 2024-07-25T08:52:30.000-07:00
* Implement isoLatin1 and macOSRoman encoding

* Limit new behavior to non-framework build
diff --git a/Sources/FoundationEssentials/String/String+IO.swift b/Sources/FoundationEssentials/String/String+IO.swift
@@ -14,8 +14,38 @@
 internal import _ForSwiftFoundation
 #endif
 
+internal import _FoundationCShims
+
 fileprivate let stringEncodingAttributeName = "com.apple.TextEncoding"
 
+private struct ExtendingToUTF16Sequence<Base: Sequence<UInt8>> : Sequence {
+    typealias Element = UInt16
+    
+    struct Iterator : IteratorProtocol {
+        private var base: Base.Iterator
+        
+        init(_ base: Base.Iterator) {
+            self.base = base
+        }
+        
+        mutating func next() -> Element? {
+            guard let value = base.next() else { return nil }
+            return UInt16(value)
+        }
+    }
+    
+    private let base: Base
+    
+    init(_ base: Base) {
+        self.base = base
+    }
+    
+    func makeIterator() -> Iterator {
+        Iterator(base.makeIterator())
+    }
+}
+
+
 @available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *)
 extension String {
     /// Returns a `String` initialized by converting given `data` into
@@ -139,6 +169,31 @@ extension String {
             } else {
                 return nil
             }
+        #if !FOUNDATION_FRAMEWORK
+        case .isoLatin1:
+            guard bytes.allSatisfy(\.isValidISOLatin1) else {
+                return nil
+            }
+            // isoLatin1 is an 8-bit encoding that represents a subset of UTF-16
+            // Map to 16-bit values and decode as UTF-16
+            self.init(_validating: ExtendingToUTF16Sequence(bytes), as: UTF16.self)
+        case .macOSRoman:
+            func buildString(_ bytes: UnsafeBufferPointer<UInt8>) -> String {
+                String(unsafeUninitializedCapacity: bytes.count * 3) { buffer in
+                    var next = 0
+                    for byte in bytes {
+                        if Unicode.ASCII.isASCII(byte) {
+                            buffer.initializeElement(at: next, to: byte)
+                            next += 1
+                        } else {
+                            next = buffer.suffix(from: next).initialize(fromContentsOf: byte.macRomanNonASCIIAsUTF8)
+                        }
+                    }
+                    return next
+                }
+            }
+            self = bytes.withContiguousStorageIfAvailable(buildString) ?? Array(bytes).withUnsafeBufferPointer(buildString)
+        #endif
         default:
 #if FOUNDATION_FRAMEWORK
             // In the framework, we can fall back to NS/CFString to handle more esoteric encodings.
@@ -261,6 +316,8 @@ internal func encodingFromDataForExtendedAttribute(_ value: Data) -> String.Enco
                 }
 #else
                 foundEncoding = switch enc {
+                case 0x0: .macOSRoman
+                case 0x0201: .isoLatin1
                 case 0x0600: .ascii
                 case 0x08000100: .utf8
                 case 0x0100: .utf16
@@ -304,6 +361,8 @@ internal func encodingFromDataForExtendedAttribute(_ value: Data) -> String.Enco
         case "utf-32": return .utf32
         case "utf-32be": return .utf32BigEndian
         case "utf-32le": return .utf32LittleEndian
+        case "iso-8859-1": return .isoLatin1
+        case "macintosh": return .macOSRoman
         default: return nil // Unknown encoding value
         }
 #endif
@@ -322,6 +381,8 @@ internal func extendedAttributeData(for encoding: String.Encoding) -> Data? {
     let encodingName = CFStringConvertEncodingToIANACharSetName(cfEncoding)
 #else
     let cfEncoding : UInt? = switch encoding {
+    case .macOSRoman: 0x0
+    case .isoLatin1: 0x0201
     case .ascii: 0x0600
     case .utf8: 0x08000100
     case .utf16: 0x0100
@@ -346,6 +407,8 @@ internal func extendedAttributeData(for encoding: String.Encoding) -> Data? {
     case .utf32: "utf-32"
     case .utf32BigEndian: "utf-32be"
     case .utf32LittleEndian: "utf-32le"
+    case .macOSRoman: "macintosh"
+    case .isoLatin1: "iso-8859-1"
     default: nil
     }
 #endif
diff --git a/Sources/FoundationEssentials/String/StringProtocol+Essentials.swift b/Sources/FoundationEssentials/String/StringProtocol+Essentials.swift
@@ -19,6 +19,61 @@ import Darwin
 import Glibc
 #endif
 
+internal import _FoundationCShims
+
+extension BinaryInteger {
+    var isValidISOLatin1: Bool {
+        (0x20 <= self && self <= 0x7E) || (0xA0 <= self && self <= 0xFF)
+    }
+}
+
+extension UInt8 {
+    private typealias UTF8Representation = (UInt8, UInt8, UInt8)
+    private static func withMacRomanMap<R>(_ body: (UnsafeBufferPointer<UTF8Representation>) -> R) -> R {
+        withUnsafePointer(to: _stringshims_macroman_mapping) {
+            $0.withMemoryRebound(to: UTF8Representation.self, capacity: Int(_STRINGSHIMS_MACROMAN_MAP_SIZE)) {
+                body(UnsafeBufferPointer(start: $0, count: Int(_STRINGSHIMS_MACROMAN_MAP_SIZE)))
+            }
+        }
+    }
+    
+    var macRomanNonASCIIAsUTF8: some Collection<UInt8> {
+        assert(!Unicode.ASCII.isASCII(self))
+        return Self.withMacRomanMap { map in
+            let utf8Rep = map[Int(self) - 128]
+            if utf8Rep.2 == 0 {
+                return [utf8Rep.0, utf8Rep.1]
+            } else {
+                return [utf8Rep.0, utf8Rep.1, utf8Rep.2]
+            }
+        }
+    }
+    
+    init?(macRomanFor scalar: UnicodeScalar) {
+        guard !scalar.isASCII else {
+            self.init(scalar.value)
+            return
+        }
+        
+        let utf8 = Array(scalar.utf8)
+        guard utf8.count <= 3 else {
+            return nil
+        }
+        let tuple = (utf8[0], utf8[1], utf8.count == 2 ? 0 : utf8[2])
+        
+        let value: UInt8? = Self.withMacRomanMap { map in
+            if let found = map.firstIndex(where: { $0 == tuple }) {
+                return UInt8(found) + 128
+            } else {
+                return nil
+            }
+        }
+        
+        guard let value else { return nil }
+        self = value
+    }
+}
+
 // These provides concrete implementations for String and Substring, enhancing performance over generic StringProtocol.
 
 @available(FoundationPreview 0.4, *)
@@ -156,6 +211,26 @@ extension String {
             }
             
             return data + swapped
+        #if !FOUNDATION_FRAMEWORK
+        case .isoLatin1:
+            return try? Data(capacity: self.utf16.count) { buffer in
+                for scalar in self.utf16 {
+                    guard scalar.isValidISOLatin1 else {
+                        throw CocoaError(.fileWriteInapplicableStringEncoding)
+                    }
+                    buffer.appendElement(UInt8(scalar & 0xFF))
+                }
+            }
+        case .macOSRoman:
+            return try? Data(capacity: self.unicodeScalars.count) { buffer in
+                for scalar in self.unicodeScalars {
+                    guard let value = UInt8(macRomanFor: scalar) else {
+                        throw CocoaError(.fileWriteInapplicableStringEncoding)
+                    }
+                    buffer.appendElement(value)
+                }
+            }
+        #endif
         default:
 #if FOUNDATION_FRAMEWORK
             // Other encodings, defer to the CoreFoundation implementation
diff --git a/Sources/_FoundationCShims/include/string_shims.h b/Sources/_FoundationCShims/include/string_shims.h
@@ -14,6 +14,7 @@
 #define CSHIMS_STRING_H
 
 #include "_CShimsMacros.h"
+#include "_CStdlib.h"
 
 #if __has_include(<locale.h>)
 #include <locale.h>
@@ -42,6 +43,9 @@ INTERNAL double _stringshims_strtod_l(const char * _Nullable __restrict nptr, ch
 
 INTERNAL float _stringshims_strtof_l(const char * _Nullable __restrict nptr, char * _Nullable * _Nullable __restrict endptr, locale_t _Nullable loc);
 
+#define _STRINGSHIMS_MACROMAN_MAP_SIZE 129
+INTERNAL const uint8_t _stringshims_macroman_mapping[_STRINGSHIMS_MACROMAN_MAP_SIZE][3];
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/Sources/_FoundationCShims/string_shims.c b/Sources/_FoundationCShims/string_shims.c
@@ -102,3 +102,135 @@ float _stringshims_strtof_l(const char * _Nullable restrict nptr,
     return result;
 #endif
 }
+
+const uint8_t _stringshims_macroman_mapping[_STRINGSHIMS_MACROMAN_MAP_SIZE][3] = {
+    { 0xC2, 0xA0, 0x00 }, /* NO-BREAK SPACE */
+    { 0xC2, 0xA1, 0x00 }, /* INVERTED EXCLAMATION MARK */
+    { 0xC2, 0xA2, 0x00 }, /* CENT SIGN */
+    { 0xC2, 0xA3, 0x00 }, /* POUND SIGN */
+    { 0xC2, 0xA5, 0x00 }, /* YEN SIGN */
+    { 0xC2, 0xA7, 0x00 }, /* SECTION SIGN */
+    { 0xC2, 0xA8, 0x00 }, /* DIAERESIS */
+    { 0xC2, 0xA9, 0x00 }, /* COPYRIGHT SIGN */
+    { 0xC2, 0xAA, 0x00 }, /* FEMININE ORDINAL INDICATOR */
+    { 0xC2, 0xAB, 0x00 }, /* LEFT-POINTING DOUBLE ANGLE QUOTATION MARK */
+    { 0xC2, 0xAC, 0x00 }, /* NOT SIGN */
+    { 0xC2, 0xAE, 0x00 }, /* REGISTERED SIGN */
+    { 0xC2, 0xAF, 0x00 }, /* MACRON */
+    { 0xC2, 0xB0, 0x00 }, /* DEGREE SIGN */
+    { 0xC2, 0xB1, 0x00 }, /* PLUS-MINUS SIGN */
+    { 0xC2, 0xB4, 0x00 }, /* ACUTE ACCENT */
+    { 0xC2, 0xB5, 0x00 }, /* MICRO SIGN */
+    { 0xC2, 0xB6, 0x00 }, /* PILCROW SIGN */
+    { 0xC2, 0xB7, 0x00 }, /* MIDDLE DOT */
+    { 0xC2, 0xB8, 0x00 }, /* CEDILLA */
+    { 0xC2, 0xBA, 0x00 }, /* MASCULINE ORDINAL INDICATOR */
+    { 0xC2, 0xBB, 0x00 }, /* RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK */
+    { 0xC2, 0xBF, 0x00 }, /* INVERTED QUESTION MARK */
+    { 0xC3, 0x80, 0x00 }, /* LATIN CAPITAL LETTER A WITH GRAVE */
+    { 0xC3, 0x81, 0x00 }, /* LATIN CAPITAL LETTER A WITH ACUTE */
+    { 0xC3, 0x82, 0x00 }, /* LATIN CAPITAL LETTER A WITH CIRCUMFLEX */
+    { 0xC3, 0x83, 0x00 }, /* LATIN CAPITAL LETTER A WITH TILDE */
+    { 0xC3, 0x84, 0x00 }, /* LATIN CAPITAL LETTER A WITH DIAERESIS */
+    { 0xC3, 0x85, 0x00 }, /* LATIN CAPITAL LETTER A WITH RING ABOVE */
+    { 0xC3, 0x86, 0x00 }, /* LATIN CAPITAL LIGATURE AE */
+    { 0xC3, 0x87, 0x00 }, /* LATIN CAPITAL LETTER C WITH CEDILLA */
+    { 0xC3, 0x88, 0x00 }, /* LATIN CAPITAL LETTER E WITH GRAVE */
+    { 0xC3, 0x89, 0x00 }, /* LATIN CAPITAL LETTER E WITH ACUTE */
+    { 0xC3, 0x8A, 0x00 }, /* LATIN CAPITAL LETTER E WITH CIRCUMFLEX */
+    { 0xC3, 0x8B, 0x00 }, /* LATIN CAPITAL LETTER E WITH DIAERESIS */
+    { 0xC3, 0x8C, 0x00 }, /* LATIN CAPITAL LETTER I WITH GRAVE */
+    { 0xC3, 0x8D, 0x00 }, /* LATIN CAPITAL LETTER I WITH ACUTE */
+    { 0xC3, 0x8E, 0x00 }, /* LATIN CAPITAL LETTER I WITH CIRCUMFLEX */
+    { 0xC3, 0x8F, 0x00 }, /* LATIN CAPITAL LETTER I WITH DIAERESIS */
+    { 0xC3, 0x91, 0x00 }, /* LATIN CAPITAL LETTER N WITH TILDE */
+    { 0xC3, 0x92, 0x00 }, /* LATIN CAPITAL LETTER O WITH GRAVE */
+    { 0xC3, 0x93, 0x00 }, /* LATIN CAPITAL LETTER O WITH ACUTE */
+    { 0xC3, 0x94, 0x00 }, /* LATIN CAPITAL LETTER O WITH CIRCUMFLEX */
+    { 0xC3, 0x95, 0x00 }, /* LATIN CAPITAL LETTER O WITH TILDE */
+    { 0xC3, 0x96, 0x00 }, /* LATIN CAPITAL LETTER O WITH DIAERESIS */
+    { 0xC3, 0x98, 0x00 }, /* LATIN CAPITAL LETTER O WITH STROKE */
+    { 0xC3, 0x99, 0x00 }, /* LATIN CAPITAL LETTER U WITH GRAVE */
+    { 0xC3, 0x9A, 0x00 }, /* LATIN CAPITAL LETTER U WITH ACUTE */
+    { 0xC3, 0x9B, 0x00 }, /* LATIN CAPITAL LETTER U WITH CIRCUMFLEX */
+    { 0xC3, 0x9C, 0x00 }, /* LATIN CAPITAL LETTER U WITH DIAERESIS */
+    { 0xC3, 0x9F, 0x00 }, /* LATIN SMALL LETTER SHARP S */
+    { 0xC3, 0xA0, 0x00 }, /* LATIN SMALL LETTER A WITH GRAVE */
+    { 0xC3, 0xA1, 0x00 }, /* LATIN SMALL LETTER A WITH ACUTE */
+    { 0xC3, 0xA2, 0x00 }, /* LATIN SMALL LETTER A WITH CIRCUMFLEX */
+    { 0xC3, 0xA3, 0x00 }, /* LATIN SMALL LETTER A WITH TILDE */
+    { 0xC3, 0xA4, 0x00 }, /* LATIN SMALL LETTER A WITH DIAERESIS */
+    { 0xC3, 0xA5, 0x00 }, /* LATIN SMALL LETTER A WITH RING ABOVE */
+    { 0xC3, 0xA6, 0x00 }, /* LATIN SMALL LIGATURE AE */
+    { 0xC3, 0xA7, 0x00 }, /* LATIN SMALL LETTER C WITH CEDILLA */
+    { 0xC3, 0xA8, 0x00 }, /* LATIN SMALL LETTER E WITH GRAVE */
+    { 0xC3, 0xA9, 0x00 }, /* LATIN SMALL LETTER E WITH ACUTE */
+    { 0xC3, 0xAA, 0x00 }, /* LATIN SMALL LETTER E WITH CIRCUMFLEX */
+    { 0xC3, 0xAB, 0x00 }, /* LATIN SMALL LETTER E WITH DIAERESIS */
+    { 0xC3, 0xAC, 0x00 }, /* LATIN SMALL LETTER I WITH GRAVE */
+    { 0xC3, 0xAD, 0x00 }, /* LATIN SMALL LETTER I WITH ACUTE */
+    { 0xC3, 0xAE, 0x00 }, /* LATIN SMALL LETTER I WITH CIRCUMFLEX */
+    { 0xC3, 0xAF, 0x00 }, /* LATIN SMALL LETTER I WITH DIAERESIS */
+    { 0xC3, 0xB1, 0x00 }, /* LATIN SMALL LETTER N WITH TILDE */
+    { 0xC3, 0xB2, 0x00 }, /* LATIN SMALL LETTER O WITH GRAVE */
+    { 0xC3, 0xB3, 0x00 }, /* LATIN SMALL LETTER O WITH ACUTE */
+    { 0xC3, 0xB4, 0x00 }, /* LATIN SMALL LETTER O WITH CIRCUMFLEX */
+    { 0xC3, 0xB5, 0x00 }, /* LATIN SMALL LETTER O WITH TILDE */
+    { 0xC3, 0xB6, 0x00 }, /* LATIN SMALL LETTER O WITH DIAERESIS */
+    { 0xC3, 0xB7, 0x00 }, /* DIVISION SIGN */
+    { 0xC3, 0xB8, 0x00 }, /* LATIN SMALL LETTER O WITH STROKE */
+    { 0xC3, 0xB9, 0x00 }, /* LATIN SMALL LETTER U WITH GRAVE */
+    { 0xC3, 0xBA, 0x00 }, /* LATIN SMALL LETTER U WITH ACUTE */
+    { 0xC3, 0xBB, 0x00 }, /* LATIN SMALL LETTER U WITH CIRCUMFLEX */
+    { 0xC3, 0xBC, 0x00 }, /* LATIN SMALL LETTER U WITH DIAERESIS */
+    { 0xC3, 0xBF, 0x00 }, /* LATIN SMALL LETTER Y WITH DIAERESIS */
+    { 0xC4, 0xB1, 0x00 }, /* LATIN SMALL LETTER DOTLESS I */
+    { 0xC5, 0x92, 0x00 }, /* LATIN CAPITAL LIGATURE OE */
+    { 0xC5, 0x93, 0x00 }, /* LATIN SMALL LIGATURE OE */
+    { 0xC5, 0xB8, 0x00 }, /* LATIN CAPITAL LETTER Y WITH DIAERESIS */
+    { 0xC6, 0x92, 0x00 }, /* LATIN SMALL LETTER F WITH HOOK */
+    { 0xCB, 0x86, 0x00 }, /* MODIFIER LETTER CIRCUMFLEX ACCENT */
+    { 0xCB, 0x87, 0x00 }, /* CARON */
+    { 0xCB, 0x98, 0x00 }, /* BREVE */
+    { 0xCB, 0x99, 0x00 }, /* DOT ABOVE */
+    { 0xCB, 0x9A, 0x00 }, /* RING ABOVE */
+    { 0xCB, 0x9B, 0x00 }, /* OGONEK */
+    { 0xCB, 0x9C, 0x00 }, /* SMALL TILDE */
+    { 0xCB, 0x9D, 0x00 }, /* DOUBLE ACUTE ACCENT */
+    { 0xCE, 0xA9, 0x00 }, /* OHM SIGN (Canonical ?) */
+    { 0xCF, 0x80, 0x00 }, /* GREEK SMALL LETTER PI */
+    { 0xE2, 0x80, 0x93 }, /* EN DASH */
+    { 0xE2, 0x80, 0x94 }, /* EM DASH */
+    { 0xE2, 0x80, 0x98 }, /* LEFT SINGLE QUOTATION MARK */
+    { 0xE2, 0x80, 0x99 }, /* RIGHT SINGLE QUOTATION MARK */
+    { 0xE2, 0x80, 0x9A }, /* SINGLE LOW-9 QUOTATION MARK */
+    { 0xE2, 0x80, 0x9C }, /* LEFT DOUBLE QUOTATION MARK */
+    { 0xE2, 0x80, 0x9D }, /* RIGHT DOUBLE QUOTATION MARK */
+    { 0xE2, 0x80, 0x9E }, /* DOUBLE LOW-9 QUOTATION MARK */
+    { 0xE2, 0x80, 0xA0 }, /* DAGGER */
+    { 0xE2, 0x80, 0xA1 }, /* DOUBLE DAGGER */
+    { 0xE2, 0x80, 0xA2 }, /* BULLET */
+    { 0xE2, 0x80, 0xA6 }, /* HORIZONTAL ELLIPSIS */
+    { 0xE2, 0x80, 0xB0 }, /* PER MILLE SIGN */
+    { 0xE2, 0x80, 0xB9 }, /* SINGLE LEFT-POINTING ANGLE QUOTATION MARK */
+    { 0xE2, 0x80, 0xBA }, /* SINGLE RIGHT-POINTING ANGLE QUOTATION MARK */
+    { 0xE2, 0x81, 0x84 }, /* FRACTION SLASH */
+    { 0xE2, 0x82, 0xAC }, /* EURO SIGN */
+    { 0xE2, 0x84, 0xA2 }, /* TRADE MARK SIGN */
+    { 0xE2, 0x84, 0xA6 }, /* OHM SIGN */
+    { 0xE2, 0x88, 0x82 }, /* PARTIAL DIFFERENTIAL */
+    { 0xE2, 0x88, 0x86 }, /* INCREMENT */
+    { 0xE2, 0x88, 0x8F }, /* N-ARY PRODUCT */
+    { 0xE2, 0x88, 0x91 }, /* N-ARY SUMMATION */
+    { 0xE2, 0x88, 0x9A }, /* SQUARE ROOT */
+    { 0xE2, 0x88, 0x9E }, /* INFINITY */
+    { 0xE2, 0x88, 0xAB }, /* INTEGRAL */
+    { 0xE2, 0x89, 0x88 }, /* ALMOST EQUAL TO */
+    { 0xE2, 0x89, 0xA0 }, /* NOT EQUAL TO */
+    { 0xE2, 0x89, 0xA4 }, /* LESS-THAN OR EQUAL TO */
+    { 0xE2, 0x89, 0xA5 }, /* GREATER-THAN OR EQUAL TO */
+    { 0xE2, 0x97, 0x8A }, /* LOZENGE */
+    { 0xEF, 0xA3, 0xBF }, /* Apple logo */
+    { 0xEF, 0xAC, 0x81 }, /* LATIN SMALL LIGATURE FI */
+    { 0xEF, 0xAC, 0x82 }, /* LATIN SMALL LIGATURE FL */
+};
diff --git a/Tests/FoundationEssentialsTests/StringTests.swift b/Tests/FoundationEssentialsTests/StringTests.swift
