Introduce ICUExport: a command utility that upgrades the ICU version for the swift-foundation-icu, based on ICU releases from apple-oss-distribution

ICUExport/Package.swift

@@ -0,0 +1,28 @@
+// swift-tools-version: 6.0
+// The swift-tools-version declares the minimum version of Swift required to build this package.
+
+import PackageDescription
+
+let package = Package(
+    name: "ICUExport",
+    platforms: [.macOS(.v15)],
+    dependencies: [
+        .package(url: "https://github.com/apple/swift-argument-parser", from: "1.4.0"),
+        .package(url: "https://github.com/swiftlang/swift-subprocess", from: "0.1.0"),
+    ],
+    targets: [
+        // Targets are the basic building blocks of a package, defining a module or a test suite.
+        // Targets can depend on other targets in this package and products from dependencies.
+        .executableTarget(
+            name: "icu-export",
+            dependencies: [
+                .product(name: "ArgumentParser", package: "swift-argument-parser"),
+                .product(name: "Subprocess", package: "swift-subprocess")
+            ],
+            resources: [
+                .process("Patches/putil.patch"),
+                .process("Patches/namespace.patch"),
+            ]
+        ),
+    ]
+)

ICUExport/README.md

@@ -0,0 +1,25 @@
+# ICUExport
+
+A utility that upgrades the ICU version for the [swift-foundation-icu](https://github.com/swiftlang/swift-foundation-icu) repository.
+
+## Usage
+
+`icu-export` performs a merge between the existing `swift-foundation-icu` and the upgraded `ICU` repository to extract relevant files from each. To start, first clone the two repositories:
+
+```bash
+git clone https://github.com/apple-oss-distributions/ICU.git /path/to/ICU
+git clone https://github.com/swiftlang/swift-foundation-icu.git /path/to/swift-foundation-icu
+```
+
+Run the utility:
+
+```bash
+swift run icu-export -i /path/to/ICU -s /path/to/swift-foundation-icu -o /output/path
+```
+
+`icu-export` will create a directory `SwiftFoundationICU` under `/output/path` that contains the newly updated ICU.
+
+—
+### Known Limitations
+
+Depending on the changes made to each ICU upgrade, `icu-export` may fail to apply the patches in `Sources/Patches`. In such cases, please manually apply the changes.

ICUExport/Sources/Patches/namespace.patch

@@ -0,0 +1,88 @@
+From ff7d419b581bff5cbdd1a1b483d9389339558a00 Mon Sep 17 00:00:00 2001
+From: Charles Hu <[email protected]>
+Date: Tue, 23 Jul 2024 18:32:04 -0700
+Subject: [PATCH 1/2] Hide all ICU public C++ Symbols
+
+**Rationale:** When FoundationInternationalization tests are executed, we effectively load two ICU instances into memory: 1) The system ICU loaded by XCTest via system Foundation; 2) The package ICU SwiftFoundation utilizes.
+
+These two ICUs cause symbol collisions for dyld due to the fact that all public C++ symbols share a global namespace and coalesce across all loaded dylibs. Consequently, we encounter sporadic test failures in SwiftFoundation as dyld arbitrarily selects ICU symbols and occasionally chooses the system one.
+
+To address this issue, we resolved to hide all C++ APIs, ensuring they are not weakly referenced and potentially bound to the system ICU implementation. This solution proves effective for SwiftFoundation, as it does not actually utilize the C++ APIs.
+---
+ .../include/_foundation_unicode/utypes.h      | 32 +++++++++++++++----
+ 1 file changed, 25 insertions(+), 7 deletions(-)
+
+diff --git a/icuSources/include/_foundation_unicode/utypes.h b/icuSources/include/_foundation_unicode/utypes.h
+index e707465..10f7189 100644
+--- a/icuSources/include/_foundation_unicode/utypes.h
++++ b/icuSources/include/_foundation_unicode/utypes.h
+@@ -354,13 +354,31 @@ typedef double UDate;
+ #endif
+
+ #if defined(U_COMBINED_IMPLEMENTATION)
+-#define U_DATA_API     U_EXPORT
+-#define U_COMMON_API   U_EXPORT
+-#define U_I18N_API     U_EXPORT
+-#define U_LAYOUT_API   U_EXPORT
+-#define U_LAYOUTEX_API U_EXPORT
+-#define U_IO_API       U_EXPORT
+-#define U_TOOLUTIL_API U_EXPORT
++// SwiftFoundationICU Changes: hide all C++ public symbols
++// Rationale: When FoundationInternationalization tests are executed,
++// we effectively load two ICU instances into memory:
++//
++// 1) The system ICU loaded by XCTest via system Foundation;
++// 2) The package ICU SwiftFoundation utilizes.
++//
++// These two ICUs cause symbol collisions for dyld due to the fact that
++// all public C++ symbols share a global namespace and coalesce across all loaded dylibs.
++// Consequently, we encounter sporadic test failures in SwiftFoundation as dyld
++// arbitrarily selects ICU symbols and occasionally chooses the system one.
++//
++// To address this issue, we resolved to hide all C++ APIs,
++// ensuring they are not weakly referenced and potentially bound to
++// the system ICU implementation.
++//
++// This solution proves effective for SwiftFoundation,
++// as it does not actually utilize the C++ APIs.
++#define U_DATA_API     __attribute__((visibility("hidden")))
++#define U_COMMON_API   __attribute__((visibility("hidden")))
++#define U_I18N_API     __attribute__((visibility("hidden")))
++#define U_LAYOUT_API   __attribute__((visibility("hidden")))
++#define U_LAYOUTEX_API __attribute__((visibility("hidden")))
++#define U_IO_API       __attribute__((visibility("hidden")))
++#define U_TOOLUTIL_API __attribute__((visibility("hidden")))
+ #elif defined(U_STATIC_IMPLEMENTATION)
+ #define U_DATA_API
+ #define U_COMMON_API
+
+From b671477b99b6d4b4a32b3426a0fc42ec242d7d2a Mon Sep 17 00:00:00 2001
+From: Charles Hu <[email protected]>
+Date: Fri, 26 Jul 2024 14:02:53 -0700
+Subject: [PATCH 2/2] Use U_CAPI for uspoof_getInclusionUnicodeSet and
+ uspoof_getRecommendedUnicodeSet
+
+---
+ icuSources/i18n/uspoof.cpp | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/icuSources/i18n/uspoof.cpp b/icuSources/i18n/uspoof.cpp
+index 206e0b2..d1ec69d 100644
+--- a/icuSources/i18n/uspoof.cpp
++++ b/icuSources/i18n/uspoof.cpp
+@@ -781,13 +781,13 @@ uspoof_getRecommendedSet(UErrorCode *status) {
+     return gRecommendedSet->toUSet();
+ }
+
+-U_I18N_API const UnicodeSet * U_EXPORT2
++U_CAPI const UnicodeSet * U_EXPORT2
+ uspoof_getInclusionUnicodeSet(UErrorCode *status) {
+     umtx_initOnce(gSpoofInitStaticsOnce, &initializeStatics, *status);
+     return gInclusionSet;
+ }
+
+-U_I18N_API const UnicodeSet * U_EXPORT2
++U_CAPI const UnicodeSet * U_EXPORT2
+ uspoof_getRecommendedUnicodeSet(UErrorCode *status) {
+     umtx_initOnce(gSpoofInitStaticsOnce, &initializeStatics, *status);
+     return gRecommendedSet;

ICUExport/Sources/Patches/putil.patch

@@ -0,0 +1,77 @@
+From db45750963292c30f0eb84833b522a42fd9a2fe4 Mon Sep 17 00:00:00 2001
+From: Charles Hu <[email protected]>
+Date: Thu, 25 Jul 2024 14:52:24 -0700
+Subject: [PATCH] putil.cpp SwiftFoundationICU changes
+
+---
+ icuSources/common/putil.cpp | 40 -------------------------------------
+ 1 file changed, 40 deletions(-)
+
+diff --git a/icuSources/common/putil.cpp b/icuSources/common/putil.cpp
+index 89216be..f318f56 100644
+--- a/icuSources/common/putil.cpp
++++ b/icuSources/common/putil.cpp
+@@ -1426,31 +1426,6 @@ static BOOL U_CALLCONV getIcuDataDirectoryUnderWindowsDirectory(char* directoryB
+ }
+ #endif
+
+-#if APPLE_ICU_CHANGES
+-// rdar://102831360 (Add Swift Package Manager support to ICU)
+-#if defined(USE_PACKAGE_DATA)
+-const char* getPackageICUDataPath() {
+-    Dl_info dl_info;
+-    dladdr(reinterpret_cast<const void*>(getPackageICUDataPath), &dl_info);
+-    const char* libraryFilename = dl_info.dli_fname;
+-    if (libraryFilename != NULL && libraryFilename[0] != 0) {
+-        // Remove the executable name
+-        char path[PATH_MAX + 1];
+-        strncpy(path, libraryFilename, PATH_MAX);
+-        char *lastSlash = strrchr(path, '/');
+-        if (lastSlash) {
+-            // Terminate the string at /
+-            *lastSlash = 0;
+-        }
+-        // Append the resource bundle path // rdar://104488214
+-        strcat(path, "/FoundationICU_FoundationICU.resources");
+-        return strdup(path);
+-    }
+-    return "";
+-}
+-#endif
+-#endif // APPLE_ICU_CHANGES
+-
+ static void U_CALLCONV dataDirectoryInitFn() {
+     /* If we already have the directory, then return immediately. Will happen if user called
+      * u_setDataDirectory().
+@@ -1459,14 +1434,6 @@ static void U_CALLCONV dataDirectoryInitFn() {
+         return;
+     }
+
+-#if APPLE_ICU_CHANGES
+-// rdar://102831360 (Add Swift Package Manager support to ICU)
+-#if defined(USE_PACKAGE_DATA)
+-    u_setDataDirectory(getPackageICUDataPath());
+-    return;
+-#endif
+-#endif // APPLE_ICU_CHANGES
+-
+     const char *path = nullptr;
+ #if defined(ICU_DATA_DIR_PREFIX_ENV_VAR)
+     char datadir_path_buffer[PATH_MAX];
+@@ -1603,13 +1570,6 @@ static void U_CALLCONV TimeZoneDataDirInitFn(UErrorCode &status) {
+     dir = getenv("ICU_TIMEZONE_FILES_DIR");
+ #endif // U_PLATFORM_HAS_WINUWP_API
+
+-#if APPLE_ICU_CHANGES
+-// rdar://102831360 (Add Swift Package Manager support to ICU)
+-#if defined(USE_PACKAGE_DATA)
+-    dir = getPackageICUDataPath();
+-#endif
+-#endif // APPLE_ICU_CHANGES
+-
+ #if defined(U_TIMEZONE_FILES_DIR)
+     if (dir == nullptr) {
+         // Build time configuration setting.
+-- 
+2.39.5 (Apple Git-154)
+