Generate Unicode normalization data for SwiftShims

Author: Azoy

stdlib/public/SwiftShims/CMakeLists.txt

@@ -19,6 +19,7 @@ set(sources
   System.h
   Target.h
   ThreadLocalStorage.h
+  UnicodeData.h
   UnicodeShims.h
   Visibility.h
   _SwiftConcurrency.h

stdlib/public/SwiftShims/UnicodeData.h

@@ -0,0 +1,47 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2021 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_STDLIB_SHIMS_UNICODEDATA_H
+#define SWIFT_STDLIB_SHIMS_UNICODEDATA_H
+
+#include "SwiftStdint.h"
+#include "Visibility.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+SWIFT_RUNTIME_STDLIB_INTERNAL
+__swift_uint16_t _swift_stdlib_getNormData(__swift_uint32_t scalar);
+
+SWIFT_RUNTIME_STDLIB_INTERNAL
+const __swift_uint8_t * const _swift_stdlib_nfd_decompositions;
+
+SWIFT_RUNTIME_STDLIB_INTERNAL
+__swift_uint32_t _swift_stdlib_getDecompositionEntry(__swift_uint32_t scalar);
+
+SWIFT_RUNTIME_STDLIB_INTERNAL
+__swift_uint32_t _swift_stdlib_getComposition(__swift_uint32_t x,
+                                              __swift_uint32_t y);
+
+SWIFT_RUNTIME_STDLIB_INTERNAL
+__swift_intptr_t _swift_stdlib_getMphIdx(__swift_uint32_t scalar,
+                                         __swift_intptr_t levels,
+                                         const __swift_uint64_t * const *keys,
+                                         const __swift_uint16_t * const *ranks,
+                                         const __swift_uint16_t * const sizes);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif // SWIFT_STDLIB_SHIMS_UNICODEDATA_H

stdlib/public/SwiftShims/UnicodeShims.h

@@ -432,7 +432,6 @@ typedef enum __swift_stdlib_UNumericType {
 
 typedef struct __swift_stdlib_UBreakIterator __swift_stdlib_UBreakIterator;
 typedef struct __swift_stdlib_UText __swift_stdlib_UText;
-typedef struct __swift_stdlib_UNormalizer2 __swift_stdlib_UNormalizer2;
 typedef __swift_int8_t __swift_stdlib_UBool;
 typedef __swift_int32_t __swift_stdlib_UChar32;
 #if defined(__APPLE__)
@@ -489,27 +488,6 @@ SWIFT_RUNTIME_STDLIB_API
 __swift_int32_t __swift_stdlib_ubrk_following(__swift_stdlib_UBreakIterator *bi,
                                               __swift_int32_t offset);
 
-SWIFT_RUNTIME_STDLIB_API
-__swift_stdlib_UBool
-__swift_stdlib_unorm2_hasBoundaryBefore(const __swift_stdlib_UNormalizer2 *,
-                                        __swift_stdlib_UChar32);
-
-SWIFT_RUNTIME_STDLIB_API
-const __swift_stdlib_UNormalizer2 *
-__swift_stdlib_unorm2_getNFCInstance(__swift_stdlib_UErrorCode *);
-
-SWIFT_RUNTIME_STDLIB_API
-__swift_int32_t
-__swift_stdlib_unorm2_normalize(const __swift_stdlib_UNormalizer2 *,
-                                const __swift_stdlib_UChar *, __swift_int32_t,
-                                __swift_stdlib_UChar *, __swift_int32_t,
-                                __swift_stdlib_UErrorCode *);
-
-SWIFT_RUNTIME_STDLIB_API
-__swift_int32_t __swift_stdlib_unorm2_spanQuickCheckYes(
-    const __swift_stdlib_UNormalizer2 *, const __swift_stdlib_UChar *,
-    __swift_int32_t, __swift_stdlib_UErrorCode *);
-
 SWIFT_RUNTIME_STDLIB_API
 __swift_stdlib_UBool
     __swift_stdlib_u_hasBinaryProperty(__swift_stdlib_UChar32,

stdlib/public/SwiftShims/module.modulemap

@@ -17,6 +17,7 @@ module SwiftShims {
   header "SwiftStdint.h"
   header "System.h"
   header "ThreadLocalStorage.h"
+  header "UnicodeData.h"
   header "UnicodeShims.h"
   header "Visibility.h"
   export *

stdlib/public/stubs/CMakeLists.txt

@@ -7,6 +7,8 @@ set(swift_stubs_sources
     Stubs.cpp
     ThreadLocalStorage.cpp
     MathStubs.cpp
+    UnicodeData.cpp
+    UnicodeNormalization.cpp
 )
 set(swift_stubs_objc_sources
     Availability.mm
@@ -18,7 +20,7 @@ set(swift_stubs_objc_sources
 set(swift_stubs_gyb_sources
     SwiftNativeNSXXXBase.mm.gyb)
 set(swift_stubs_unicode_normalization_sources
-    UnicodeNormalization.cpp)
+    UnicodeShims.cpp)
 set(LLVM_OPTIONAL_SOURCES
     ${swift_stubs_objc_sources}
     ${swift_stubs_unicode_normalization_sources}

stdlib/public/stubs/UnicodeData.cpp

@@ -0,0 +1,80 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2021 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#include "../SwiftShims/UnicodeData.h"
+
+static inline __swift_uint32_t scramble(__swift_uint32_t scalar) {
+  scalar *= 0xCC9E2D51;
+  scalar = (scalar << 15) | (scalar >> 17);
+  scalar *= 0x1B873593;
+  return scalar;
+}
+
+// This is a reimplementation of MurMur3 hash with a modulo at the end.
+static __swift_uint32_t hash(__swift_uint32_t scalar, __swift_uint32_t level,
+                             __swift_uint32_t seed) {
+  __swift_uint32_t hash = seed;
+
+  hash ^= scramble(scalar);
+  hash = (hash << 13) | (hash >> 19);
+  hash = hash * 5 + 0xE6546B64;
+  
+  hash ^= scramble(level);
+  hash = (hash << 13) | (hash >> 19);
+  hash = hash * 5 + 0xE6546B64;
+
+  hash ^= 8;
+  hash ^= hash >> 16;
+  hash *= 0x85EBCA6B;
+  hash ^= hash >> 13;
+  hash *= 0xC2B2AE35;
+  hash ^= hash >> 16;
+
+  return hash % level;
+}
+
+SWIFT_RUNTIME_STDLIB_INTERNAL
+__swift_intptr_t _swift_stdlib_getMphIdx(__swift_uint32_t scalar,
+                                         __swift_intptr_t levels,
+                                         const __swift_uint64_t * const *keys,
+                                         const __swift_uint16_t * const *ranks,
+                                         const __swift_uint16_t * const sizes) {
+  __swift_intptr_t resultIdx = 0;
+
+  for (int i = 0; i != levels; i += 1) {
+    auto bitArray = keys[i];
+
+    auto idx = (__swift_uint64_t) hash(scalar, sizes[i], i);
+
+    auto word = bitArray[idx / 64];
+    auto mask = (__swift_uint64_t) 1 << (idx % 64);
+
+    if (word & mask) {
+      auto rank = ranks[i][idx / 512];
+
+      for (int j = (idx / 64) & ~7; j != idx / 64; j += 1) {
+        rank += __builtin_popcountll(bitArray[j]);
+      }
+
+      auto finalWord = bitArray[idx / 64];
+
+      if (idx % 64 > 0) {
+        rank += __builtin_popcountll(finalWord << (64 - (idx % 64)));
+      }
+
+      resultIdx = rank;
+      break;
+    }
+  }
+
+  return resultIdx;
+}