Allow the "generic Unix" platforms as cross-compile targets

This allows Linux, FreeBSD, and OpenBSD to be used with Swift SDKs for cross compilation, not only as part of the toolchain on the same host as the target.

Author: jakepetroules

Sources/SWBCore/SwiftSDK.swift

@@ -85,11 +85,11 @@ public struct SwiftSDK: Sendable {
     }
 
     /// Find Swift SDKs installed by SwiftPM.
-    public static func findSDKs(targetTriples: [String], fs: any FSProxy, hostOperatingSystem: OperatingSystem) throws -> [SwiftSDK] {
+    public static func findSDKs(targetTriples: [String]?, fs: any FSProxy, hostOperatingSystem: OperatingSystem) throws -> [SwiftSDK] {
         return try findSDKs(swiftSDKsDirectory: defaultSwiftSDKsDirectory(hostOperatingSystem: hostOperatingSystem), targetTriples: targetTriples, fs: fs)
     }
 
-    private static func findSDKs(swiftSDKsDirectory: Path, targetTriples: [String], fs: any FSProxy) throws -> [SwiftSDK] {
+    private static func findSDKs(swiftSDKsDirectory: Path, targetTriples: [String]?, fs: any FSProxy) throws -> [SwiftSDK] {
         var sdks: [SwiftSDK] = []
         // Find .artifactbundle in the SDK directory (e.g. ~/Library/org.swift.swiftpm/swift-sdks)
         for artifactBundle in try fs.listdir(swiftSDKsDirectory) {
@@ -118,7 +118,7 @@ public struct SwiftSDK: Sendable {
     }
 
     /// Find Swift SDKs in an artifact bundle supporting one of the given targets.
-    private static func findSDKs(artifactBundle: Path, targetTriples: [String], fs: any FSProxy) throws -> [SwiftSDK] {
+    private static func findSDKs(artifactBundle: Path, targetTriples: [String]?, fs: any FSProxy) throws -> [SwiftSDK] {
         // Load info.json from the artifact bundle
         let infoPath = artifactBundle.join("info.json")
         guard try fs.isFile(infoPath) else { return [] }
@@ -145,7 +145,9 @@ public struct SwiftSDK: Sendable {
 
                 guard let sdk = try SwiftSDK(identifier: identifier, version: artifact.version, path: sdkPath, fs: fs) else { continue }
                 // Filter out SDKs that don't support any of the target triples.
-                guard targetTriples.contains(where: { sdk.targetTriples[$0] != nil }) else { continue }
+                if let targetTriples {
+                    guard targetTriples.contains(where: { sdk.targetTriples[$0] != nil }) else { continue }
+                }
                 sdks.append(sdk)
             }
         }

Sources/SWBGenericUnixPlatform/Plugin.swift

@@ -49,12 +49,6 @@ struct SwiftTargetInfo: Decodable {
     let target: TargetInfo
 }
 
-extension SwiftTargetInfo.TargetInfo {
-    var tripleVersion: String? {
-        triple != unversionedTriple && triple.system.hasPrefix(unversionedTriple.system) ? String(triple.system.dropFirst(unversionedTriple.system.count)).nilIfEmpty : nil
-    }
-}
-
 struct GenericUnixDeveloperDirectoryExtension: DeveloperDirectoryExtension {
     func fallbackDeveloperDirectory(hostOperatingSystem: OperatingSystem) async throws -> Core.DeveloperPath? {
         if hostOperatingSystem == .windows || hostOperatingSystem == .macOS {
@@ -82,13 +76,12 @@ struct GenericUnixPlatformSpecsExtension: SpecificationsExtension {
 
 struct GenericUnixPlatformInfoExtension: PlatformInfoExtension {
     func additionalPlatforms(context: any PlatformInfoExtensionAdditionalPlatformsContext) throws -> [(path: Path, data: [String: PropertyListItem])] {
-        let operatingSystem = context.hostOperatingSystem
-        guard operatingSystem.createFallbackSystemToolchain else {
-            return []
-        }
-
-        return try [
-            (.root, [
+        return try OperatingSystem.createFallbackSystemToolchains.compactMap { operatingSystem in
+            // Only create platforms if the host OS allows a fallback toolchain, or we're cross compiling.
+            guard operatingSystem.createFallbackSystemToolchain || operatingSystem != context.hostOperatingSystem else {
+                return nil
+            }
+            return try (.root, [
                 "Type": .plString("Platform"),
                 "Name": .plString(operatingSystem.xcodePlatformName),
                 "Identifier": .plString(operatingSystem.xcodePlatformName),
@@ -97,78 +90,156 @@ struct GenericUnixPlatformInfoExtension: PlatformInfoExtension {
                 "FamilyIdentifier": .plString(operatingSystem.xcodePlatformName),
                 "IsDeploymentPlatform": .plString("YES"),
             ])
-        ]
+        }
     }
 }
 
 struct GenericUnixSDKRegistryExtension: SDKRegistryExtension {
     let plugin: GenericUnixPlugin
 
     func additionalSDKs(context: any SDKRegistryExtensionAdditionalSDKsContext) async throws -> [(path: Path, platform: SWBCore.Platform?, data: [String: PropertyListItem])] {
-        let operatingSystem = context.hostOperatingSystem
-        guard operatingSystem.createFallbackSystemToolchain, let platform = try context.platformRegistry.lookup(name: operatingSystem.xcodePlatformName), let swift = plugin.swiftExecutablePath(fs: context.fs) else {
-            return []
-        }
+        return try await OperatingSystem.createFallbackSystemToolchains.asyncMap { operatingSystem in
+            // Only create SDKs if the host OS allows a fallback toolchain, or we're cross compiling.
+            guard operatingSystem.createFallbackSystemToolchain || operatingSystem != context.hostOperatingSystem else {
+                return nil
+            }
 
-        let defaultProperties: [String: PropertyListItem]
-        switch operatingSystem {
-        case .linux, .freebsd:
-            defaultProperties = [
-                // Workaround to avoid `-dependency_info`.
-                "LD_DEPENDENCY_INFO_FILE": .plString(""),
-
-                "GENERATE_TEXT_BASED_STUBS": "NO",
-                "GENERATE_INTERMEDIATE_TEXT_BASED_STUBS": "NO",
-
-                "CHOWN": "/usr/bin/chown",
-                "AR": "llvm-ar",
-            ]
-        default:
-            defaultProperties = [:]
-        }
+            // Don't create any SDKs for the platform if the platform itself isn't registered.
+            guard let platform = try context.platformRegistry.lookup(name: operatingSystem.xcodePlatformName) else {
+                return nil
+            }
 
-        let tripleEnvironment: String
-        switch operatingSystem {
-        case .linux:
-            tripleEnvironment = "gnu"
-        default:
-            tripleEnvironment = ""
-        }
+            var defaultProperties: [String: PropertyListItem]
+            switch operatingSystem {
+            case .linux, .freebsd:
+                defaultProperties = [
+                    // Workaround to avoid `-dependency_info`.
+                    "LD_DEPENDENCY_INFO_FILE": .plString(""),
 
-        let swiftTargetInfo = try await plugin.swiftTargetInfo(swiftExecutablePath: swift)
+                    "GENERATE_TEXT_BASED_STUBS": "NO",
+                    "GENERATE_INTERMEDIATE_TEXT_BASED_STUBS": "NO",
 
-        let deploymentTargetSettings: [String: PropertyListItem]
-        if operatingSystem == .freebsd {
-            guard let tripleVersion = swiftTargetInfo.target.tripleVersion else {
-                throw StubError.error("Unknown FreeBSD triple version")
+                    "CHOWN": "/usr/bin/chown",
+                    "AR": "llvm-ar",
+                ]
+            default:
+                defaultProperties = [:]
             }
-            deploymentTargetSettings = [
-                "DeploymentTargetSettingName": .plString("FREEBSD_DEPLOYMENT_TARGET"),
-                "DefaultDeploymentTarget": .plString(tripleVersion),
-                "MinimumDeploymentTarget": .plString(tripleVersion),
-                "MaximumDeploymentTarget": .plString(tripleVersion),
-            ]
-        } else {
-            deploymentTargetSettings = [:]
-        }
 
-        return try [(.root, platform, [
-            "Type": .plString("SDK"),
-            "Version": .plString(Version(ProcessInfo.processInfo.operatingSystemVersion).zeroTrimmed.description),
-            "CanonicalName": .plString(operatingSystem.xcodePlatformName),
-            "IsBaseSDK": .plBool(true),
-            "DefaultProperties": .plDict([
-                "PLATFORM_NAME": .plString(operatingSystem.xcodePlatformName),
-            ].merging(defaultProperties, uniquingKeysWith: { _, new in new })),
-            "SupportedTargets": .plDict([
-                operatingSystem.xcodePlatformName: .plDict([
-                    "Archs": .plArray([.plString(Architecture.hostStringValue ?? "unknown")]),
-                    "LLVMTargetTripleEnvironment": .plString(tripleEnvironment),
-                    "LLVMTargetTripleSys": .plString(operatingSystem.xcodePlatformName),
-                    "LLVMTargetTripleVendor": .plString("unknown"),
-                ].merging(deploymentTargetSettings, uniquingKeysWith: { _, new in new }))
-            ]),
-        ])]
+            if operatingSystem == .freebsd || operatingSystem != context.hostOperatingSystem {
+                // FreeBSD is always LLVM-based, and if we're cross-compiling, use lld
+                defaultProperties["ALTERNATE_LINKER"] = "lld"
+            }
+
+            let tripleEnvironment: String
+            switch operatingSystem {
+            case .linux:
+                tripleEnvironment = "gnu"
+            default:
+                tripleEnvironment = ""
+            }
+
+            let swiftSDK: SwiftSDK?
+            let sysroot: Path
+            let architectures: [String]
+            let tripleVersion: String?
+            let customProperties: [String: PropertyListItem]
+            if operatingSystem == context.hostOperatingSystem {
+                swiftSDK = nil
+                sysroot = .root
+                architectures = [Architecture.hostStringValue ?? "unknown"]
+                tripleVersion = nil
+                customProperties = [:]
+            } else {
+                do {
+                    let swiftSDKs = try SwiftSDK.findSDKs(
+                        targetTriples: nil,
+                        fs: context.fs,
+                        hostOperatingSystem: context.hostOperatingSystem
+                    ).filter { sdk in
+                        try sdk.targetTriples.keys.map {
+                            try LLVMTriple($0)
+                        }.contains {
+                            switch operatingSystem {
+                            case .linux:
+                                $0.system == "linux" && $0.environment?.hasPrefix("gnu") == true
+                            case .freebsd:
+                                $0.system == "freebsd"
+                            case .openbsd:
+                                $0.system == "openbsd"
+                            default:
+                                throw StubError.error("Unhandled operating system: \(operatingSystem)")
+                            }
+                        }
+                    }
+                    // FIXME: Do something better than just skipping the platform if more than one SDK matches
+                    swiftSDK = swiftSDKs.only
+                    guard let swiftSDK else {
+                        return nil
+                    }
+                    sysroot = swiftSDK.path
+                    architectures = try swiftSDK.targetTriples.keys.map { try LLVMTriple($0).arch }.sorted()
+                    tripleVersion = try Set(swiftSDK.targetTriples.keys.compactMap { try LLVMTriple($0).systemVersion }).only?.description
+                    customProperties = try Dictionary(uniqueKeysWithValues: swiftSDK.targetTriples.map { targetTriple in
+                        try ("__SYSROOT_\(LLVMTriple(targetTriple.key).arch)", .plString(swiftSDK.path.join(targetTriple.value.sdkRootPath).str))
+                    }).merging([
+                        "SYSROOT": "$(__SYSROOT_$(CURRENT_ARCH))",
+
+                        // ld.lld: error: -r and --export-dynamic (-rdynamic) may not be used together
+                        "LD_EXPORT_GLOBAL_SYMBOLS": "YES",
+                    ], uniquingKeysWith: { _, new in new })
+                } catch {
+                    // FIXME: Handle errors?
+                    return nil
+                }
+            }
+
+            let deploymentTargetSettings: [String: PropertyListItem]
+            if operatingSystem == .freebsd {
+                let realTripleVersion: String
+                if context.hostOperatingSystem == operatingSystem {
+                    guard let swift = plugin.swiftExecutablePath(fs: context.fs) else {
+                        throw StubError.error("Cannot locate swift executable path for determining the FreeBSD triple version")
+                    }
+                    let swiftTargetInfo = try await plugin.swiftTargetInfo(swiftExecutablePath: swift)
+                    guard let foundTripleVersion = try swiftTargetInfo.target.triple.version?.description else {
+                        throw StubError.error("Unknown FreeBSD triple version")
+                    }
+                    realTripleVersion = foundTripleVersion
+                } else if let tripleVersion {
+                    realTripleVersion = tripleVersion
+                } else {
+                    return nil // couldn't compute triple version for FreeBSD
+                }
+                deploymentTargetSettings = [
+                    "DeploymentTargetSettingName": .plString("FREEBSD_DEPLOYMENT_TARGET"),
+                    "DefaultDeploymentTarget": .plString(realTripleVersion),
+                    "MinimumDeploymentTarget": .plString(realTripleVersion),
+                    "MaximumDeploymentTarget": .plString(realTripleVersion),
+                ]
+            } else {
+                deploymentTargetSettings = [:]
+            }
+
+            return try (sysroot, platform, [
+                "Type": .plString("SDK"),
+                "Version": .plString(Version(ProcessInfo.processInfo.operatingSystemVersion).zeroTrimmed.description),
+                "CanonicalName": .plString(operatingSystem.xcodePlatformName),
+                "IsBaseSDK": .plBool(true),
+                "DefaultProperties": .plDict([
+                    "PLATFORM_NAME": .plString(operatingSystem.xcodePlatformName),
+                ].merging(defaultProperties, uniquingKeysWith: { _, new in new })),
+                "CustomProperties": .plDict(customProperties),
+                "SupportedTargets": .plDict([
+                    operatingSystem.xcodePlatformName: .plDict([
+                        "Archs": .plArray(architectures.map { .plString($0) }),
+                        "LLVMTargetTripleEnvironment": .plString(tripleEnvironment),
+                        "LLVMTargetTripleSys": .plString(operatingSystem.xcodePlatformName),
+                        "LLVMTargetTripleVendor": .plString("unknown"),
+                    ].merging(deploymentTargetSettings, uniquingKeysWith: { _, new in new }))
+                ]),
+            ])
+        }.compactMap { $0 }
     }
 }
 
@@ -218,8 +289,12 @@ struct GenericUnixToolchainRegistryExtension: ToolchainRegistryExtension {
 }
 
 extension OperatingSystem {
+    static var createFallbackSystemToolchains: [OperatingSystem] {
+        [.linux, .freebsd, .openbsd]
+    }
+
     /// Whether the Core is allowed to create a fallback toolchain, SDK, and platform for this operating system in cases where no others have been provided.
-    var createFallbackSystemToolchain: Bool {
-        return self == .linux || self == .freebsd || self == .openbsd
+    fileprivate var createFallbackSystemToolchain: Bool {
+        return Self.createFallbackSystemToolchains.contains(self)
     }
 }

Sources/SWBGenericUnixPlatform/Specs/UnixCompile.xcspec

@@ -38,6 +38,12 @@
             {
                 Name = SDKROOT;
                 Type = Path;
+                CommandLineArgs = ();
+            },
+            {
+                Name = SYSROOT;
+                DefaultValue = "$(SDKROOT)";
+                Type = Path;
                 CommandLineFlag = "--sysroot";
             },
             {
@@ -91,6 +97,12 @@
             {
                 Name = SDKROOT;
                 Type = Path;
+                CommandLineArgs = ();
+            },
+            {
+                Name = SYSROOT;
+                DefaultValue = "$(SDKROOT)";
+                Type = Path;
                 CommandLineFlag = "--sysroot";
             },
         );
@@ -106,6 +118,12 @@
             {
                 Name = SDKROOT;
                 Type = Path;
+                CommandLineArgs = ();
+            },
+            {
+                Name = SYSROOT;
+                DefaultValue = "$(SDKROOT)";
+                Type = Path;
                 CommandLineFlag = "--sysroot";
             },
         );

Sources/SWBGenericUnixPlatform/Specs/UnixLd.xcspec

@@ -53,15 +53,15 @@
             {
                 Name = CLANG_SDKROOT_LINKER_INPUT;
                 Type = Path;
-                DefaultValue = "$(SDKROOT)";
+                DefaultValue = "$(SYSROOT:default=$(SDKROOT))";
                 Condition = "$(LINKER_DRIVER) == clang";
                 CommandLineFlag = "--sysroot";
                 IsInputDependency = Yes;
             },
             {
                 Name = SWIFTC_SDKROOT_LINKER_INPUT;
                 Type = Path;
-                DefaultValue = "$(SDKROOT)";
+                DefaultValue = "$(SYSROOT:default=$(SDKROOT))";
                 Condition = "$(LINKER_DRIVER) == swiftc";
                 CommandLineFlag = "-sysroot";
                 IsInputDependency = Yes;