diff --git a/.github/scripts/windows_pre_build.ps1 b/.github/scripts/windows_pre_build.ps1 index 9932ef63..1e40dfa4 100644 --- a/.github/scripts/windows_pre_build.ps1 +++ b/.github/scripts/windows_pre_build.ps1 @@ -30,7 +30,7 @@ if ($InstallCMake) { } if (-not $SkipAndroid) { - choco install android-ndk + choco install -y android-ndk Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1 refreshenv diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 2c70054f..a0b467cd 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -17,6 +17,7 @@ jobs: linux_pre_build_command: ./.github/scripts/linux_pre_build.sh linux_build_command: 'swift test --no-parallel' linux_swift_versions: '["nightly-main", "nightly-6.2"]' + windows_pre_build_command: 'Invoke-Program .\.github\scripts\windows_pre_build.ps1' windows_swift_versions: '["nightly-main"]' windows_build_command: 'Invoke-Program swift test --no-parallel' enable_linux_static_sdk_build: true diff --git a/Sources/SWBAndroidPlatform/AndroidSDK.swift b/Sources/SWBAndroidPlatform/AndroidSDK.swift index fc5d2060..249b64cd 100644 --- a/Sources/SWBAndroidPlatform/AndroidSDK.swift +++ b/Sources/SWBAndroidPlatform/AndroidSDK.swift @@ -266,16 +266,16 @@ public import Foundation } } -fileprivate extension AndroidSDK.NDK { +extension AndroidSDK.NDK { /// The location of the Android NDK based on the `ANDROID_NDK_ROOT` environment variable (falling back to the deprecated but well known `ANDROID_NDK_HOME`). /// - seealso: [Configuring NDK Path](https://github.com/android/ndk-samples/wiki/Configure-NDK-Path#terminologies) - static var environmentOverrideLocation: AbsolutePath? { + internal static var environmentOverrideLocation: AbsolutePath? { (getEnvironmentVariable("ANDROID_NDK_ROOT") ?? getEnvironmentVariable("ANDROID_NDK_HOME"))?.nilIfEmpty.map { AbsolutePath($0) } ?? nil } /// Location of the Android NDK installed by the `google-android-ndk-*-installer` family of packages available in Debian 13 "Trixie" and Ubuntu 24.04 "Noble". /// These packages are available in non-free / multiverse and multiple versions can be installed simultaneously. - static var defaultDebianLocation: AbsolutePath? { + fileprivate static var defaultDebianLocation: AbsolutePath? { AbsolutePath("/usr/lib/android-ndk") } } diff --git a/Sources/SWBAndroidPlatform/Plugin.swift b/Sources/SWBAndroidPlatform/Plugin.swift index 87d0bee1..5c8dad11 100644 --- a/Sources/SWBAndroidPlatform/Plugin.swift +++ b/Sources/SWBAndroidPlatform/Plugin.swift @@ -26,6 +26,7 @@ public let initializePlugin: PluginInitializationFunction = { manager in @_spi(Testing) public final class AndroidPlugin: Sendable { private let androidSDKInstallations = AsyncCache() + private let androidOverrideNDKInstallation = AsyncCache() func cachedAndroidSDKInstallations(host: OperatingSystem) async throws -> [AndroidSDK] { try await androidSDKInstallations.value(forKey: host) { @@ -34,8 +35,22 @@ public let initializePlugin: PluginInitializationFunction = { manager in } } - @_spi(Testing) public func effectiveInstallation(host: OperatingSystem) async throws -> (sdk: AndroidSDK, ndk: AndroidSDK.NDK)? { + func cachedAndroidOverrideNDKInstallation(host: OperatingSystem) async throws -> AndroidSDK.NDK? { + try await androidOverrideNDKInstallation.value(forKey: host) { + if let overridePath = AndroidSDK.NDK.environmentOverrideLocation { + return try AndroidSDK.NDK(host: host, path: overridePath, fs: localFS) + } + return nil + } + } + + @_spi(Testing) public func effectiveInstallation(host: OperatingSystem) async throws -> (sdk: AndroidSDK?, ndk: AndroidSDK.NDK)? { guard let androidSdk = try? await cachedAndroidSDKInstallations(host: host).first else { + // No SDK, but we might still have a standalone NDK from the env var override + if let overrideNDK = try? await cachedAndroidOverrideNDKInstallation(host: host) { + return (nil, overrideNDK) + } + return nil } @@ -63,9 +78,9 @@ struct AndroidEnvironmentExtension: EnvironmentExtension { func additionalEnvironmentVariables(context: any EnvironmentExtensionAdditionalEnvironmentVariablesContext) async throws -> [String: String] { switch context.hostOperatingSystem { case .windows, .macOS, .linux: - if let latest = try? await plugin.cachedAndroidSDKInstallations(host: context.hostOperatingSystem).first { - let sdkPath = latest.path.path.str - let ndkPath = latest.preferredNDK?.path.path.str + if let (sdk, ndk) = try? await plugin.effectiveInstallation(host: context.hostOperatingSystem) { + let sdkPath = sdk?.path.path.str + let ndkPath = ndk.path.path.str return [ "ANDROID_HOME": sdkPath, "ANDROID_SDK_ROOT": sdkPath, @@ -220,7 +235,7 @@ struct AndroidToolchainRegistryExtension: ToolchainRegistryExtension { let plugin: AndroidPlugin func additionalToolchains(context: any ToolchainRegistryExtensionAdditionalToolchainsContext) async throws -> [Toolchain] { - guard let toolchainPath = try? await plugin.cachedAndroidSDKInstallations(host: context.hostOperatingSystem).first?.preferredNDK?.toolchainPath else { + guard let toolchainPath = try? await plugin.effectiveInstallation(host: context.hostOperatingSystem)?.ndk.toolchainPath else { return [] } diff --git a/Sources/SWBBuildService/Tools.swift b/Sources/SWBBuildService/Tools.swift index cf5863df..0d6bfe3b 100644 --- a/Sources/SWBBuildService/Tools.swift +++ b/Sources/SWBBuildService/Tools.swift @@ -569,11 +569,12 @@ private class SerializedDiagnosticsTool { return false } - let toolchain = core.toolchainRegistry.defaultToolchain - guard let libclangPath = toolchain?.librarySearchPaths.findLibrary(operatingSystem: core.hostOperatingSystem, basename: "clang") ?? toolchain?.fallbackLibrarySearchPaths.findLibrary(operatingSystem: core.hostOperatingSystem, basename: "clang") else { - throw StubError.error("unable to find libclang") + guard let toolchain = core.toolchainRegistry.defaultToolchain else { + throw StubError.error("unable to find libclang (no default toolchain)") } + let libclangPath = try toolchain.lookup(subject: .library(basename: "clang"), operatingSystem: core.hostOperatingSystem) + guard let libclang = Libclang(path: libclangPath.str) else { emitError("unable to open libclang: \(libclangPath)") return false diff --git a/Sources/SWBCore/Core.swift b/Sources/SWBCore/Core.swift index 4b6e59bb..36991f28 100644 --- a/Sources/SWBCore/Core.swift +++ b/Sources/SWBCore/Core.swift @@ -269,35 +269,35 @@ public final class Core: Sendable { self.stopAfterOpeningLibClang = UserDefaults.stopAfterOpeningLibClang self.toolchainPaths = { - var toolchainPaths = [(Path, strict: Bool)]() + var toolchainPaths = [ToolchainRegistry.SearchPath]() switch developerPath { case .xcode(let path): - toolchainPaths.append((path.join("Toolchains"), strict: path.str.hasSuffix(".app/Contents/Developer"))) + toolchainPaths.append(.init(path: path.join("Toolchains"), strict: path.str.hasSuffix(".app/Contents/Developer"))) case .swiftToolchain(let path, xcodeDeveloperPath: let xcodeDeveloperPath): if hostOperatingSystem == .windows { - toolchainPaths.append((path.join("Toolchains"), strict: true)) + toolchainPaths.append(.init(path: path.join("Toolchains"), strict: true, aliases: ["default"])) } else { - toolchainPaths.append((path, strict: true)) + toolchainPaths.append(.init(path: path, strict: true)) } if let xcodeDeveloperPath { - toolchainPaths.append((xcodeDeveloperPath.join("Toolchains"), strict: xcodeDeveloperPath.str.hasSuffix(".app/Contents/Developer"))) + toolchainPaths.append(.init(path: xcodeDeveloperPath.join("Toolchains"), strict: xcodeDeveloperPath.str.hasSuffix(".app/Contents/Developer"))) } } // FIXME: We should support building the toolchain locally (for `inferiorProductsPath`). - toolchainPaths.append((Path("/Library/Developer/Toolchains"), strict: false)) + toolchainPaths.append(.init(path: Path("/Library/Developer/Toolchains"), strict: false)) if let homeString = getEnvironmentVariable("HOME")?.nilIfEmpty { let userToolchainsPath = Path(homeString).join("Library/Developer/Toolchains") - toolchainPaths.append((userToolchainsPath, strict: false)) + toolchainPaths.append(.init(path: userToolchainsPath, strict: false)) } if let externalToolchainDirs = getEnvironmentVariable("EXTERNAL_TOOLCHAINS_DIR") ?? environment["EXTERNAL_TOOLCHAINS_DIR"] { let envPaths = externalToolchainDirs.split(separator: Path.pathEnvironmentSeparator) for envPath in envPaths { - toolchainPaths.append((Path(envPath), strict: false)) + toolchainPaths.append(.init(path: Path(envPath), strict: false)) } } @@ -367,7 +367,7 @@ public final class Core: Sendable { }() /// The list of toolchain search paths. - @_spi(Testing) public var toolchainPaths: [(Path, strict: Bool)] + @_spi(Testing) public var toolchainPaths: [ToolchainRegistry.SearchPath] /// The platform registry. let _platformRegistry: UnsafeDelayedInitializationSendableWrapper = .init() diff --git a/Sources/SWBCore/Settings/Settings.swift b/Sources/SWBCore/Settings/Settings.swift index a1ccf9de..1844fd71 100644 --- a/Sources/SWBCore/Settings/Settings.swift +++ b/Sources/SWBCore/Settings/Settings.swift @@ -48,7 +48,7 @@ fileprivate struct PreOverridesSettings { if let toolchain = core.toolchainRegistry.lookup("default") { self.defaultToolchain = toolchain } else { - core.delegate.error("missing required default toolchain") + core.delegate.error("missing required default toolchain (\(core.toolchainRegistry.toolchains.count) loaded toolchain(s): \(core.toolchainRegistry.toolchains.map { $0.identifier }.joined(separator: " "))") self.defaultToolchain = nil } diff --git a/Sources/SWBCore/Settings/StackedSearchPaths.swift b/Sources/SWBCore/Settings/StackedSearchPaths.swift index e0e70ab5..47df20cc 100644 --- a/Sources/SWBCore/Settings/StackedSearchPaths.swift +++ b/Sources/SWBCore/Settings/StackedSearchPaths.swift @@ -68,11 +68,43 @@ public final class StackedSearchPath: Sendable { } extension StackedSearchPath { + public func lookup(subject: StackedSearchPathLookupSubject, operatingSystem: OperatingSystem) -> Path? { + lookup(subject.fileName(operatingSystem: operatingSystem)) + } + public func findExecutable(operatingSystem: OperatingSystem, basename: String) -> Path? { - lookup(Path(operatingSystem.imageFormat.executableName(basename: basename))) + lookup(subject: .executable(basename: basename), operatingSystem: operatingSystem) } public func findLibrary(operatingSystem: OperatingSystem, basename: String) -> Path? { - lookup(Path("lib\(basename).\(operatingSystem.imageFormat.dynamicLibraryExtension)")) + lookup(subject: .library(basename: basename), operatingSystem: operatingSystem) + } +} + +public enum StackedSearchPathLookupSubject { + case executable(basename: String) + case library(basename: String) + + func fileName(operatingSystem: OperatingSystem) -> Path { + switch self { + case let .executable(basename): + Path(operatingSystem.imageFormat.executableName(basename: basename)) + case let .library(basename): + Path("lib\(basename).\(operatingSystem.imageFormat.dynamicLibraryExtension)") + } + } +} + +public enum StackedSearchPathLookupError: Error { + case unableToFind(subject: StackedSearchPathLookupSubject, operatingSystem: OperatingSystem, searchPaths: [StackedSearchPath]) +} + +extension StackedSearchPathLookupError: CustomStringConvertible { + public var description: String { + switch self { + case let .unableToFind(subject, operatingSystem, searchPaths): + let candidates = searchPaths.flatMap { $0.paths.map { $0.join(subject.fileName(operatingSystem: operatingSystem)).str }} + return "unable to find \(subject.fileName(operatingSystem: operatingSystem)) among search paths: \(candidates.joined(separator: ", "))" + } } } diff --git a/Sources/SWBCore/TaskGeneration.swift b/Sources/SWBCore/TaskGeneration.swift index 53db7891..f49f17e2 100644 --- a/Sources/SWBCore/TaskGeneration.swift +++ b/Sources/SWBCore/TaskGeneration.swift @@ -1308,10 +1308,10 @@ extension TaskOutputParserDelegate { func readSerializedDiagnostics(at path: Path, workingDirectory: Path, workspaceContext: WorkspaceContext) -> [Diagnostic] { do { // Using the default toolchain's libclang regardless of context should be sufficient, since we assume serialized diagnostics to be a stable format. - let toolchain = workspaceContext.core.toolchainRegistry.defaultToolchain - guard let libclangPath = toolchain?.librarySearchPaths.findLibrary(operatingSystem: workspaceContext.core.hostOperatingSystem, basename: "clang") ?? toolchain?.fallbackLibrarySearchPaths.findLibrary(operatingSystem: workspaceContext.core.hostOperatingSystem, basename: "clang") else { - throw StubError.error("unable to find libclang") + guard let toolchain = workspaceContext.core.toolchainRegistry.defaultToolchain else { + throw StubError.error("unable to find libclang (no default toolchain)") } + let libclangPath = try toolchain.lookup(subject: .library(basename: "clang"), operatingSystem: workspaceContext.core.hostOperatingSystem) guard let libclang = workspaceContext.core.lookupLibclang(path: libclangPath).libclang else { throw StubError.error("unable to open libclang: '\(libclangPath.str)'") } diff --git a/Sources/SWBCore/ToolchainRegistry.swift b/Sources/SWBCore/ToolchainRegistry.swift index f3e1a081..28afaf07 100644 --- a/Sources/SWBCore/ToolchainRegistry.swift +++ b/Sources/SWBCore/ToolchainRegistry.swift @@ -105,7 +105,7 @@ public final class Toolchain: Hashable, Sendable { self.testingLibraryPlatformNames = testingLibraryPlatformNames } - convenience init(path: Path, operatingSystem: OperatingSystem, fs: any FSProxy, pluginManager: any PluginManager, platformRegistry: PlatformRegistry?) async throws { + convenience init(path: Path, operatingSystem: OperatingSystem, aliases additionalAliases: Set, fs: any FSProxy, pluginManager: any PluginManager, platformRegistry: PlatformRegistry?) async throws { let data: PropertyListItem do { @@ -216,6 +216,8 @@ public final class Toolchain: Hashable, Sendable { aliases = Toolchain.deriveAliases(path: path, identifier: identifier) } + aliases.formUnion(additionalAliases) + // Framework Search Paths var frameworkSearchPaths = Array() if let infoFrameworkSearchPaths = items["FallbackFrameworkSearchPaths"] { @@ -412,8 +414,32 @@ extension Array where Element == Toolchain { } } +extension Toolchain { + public func lookup(subject: StackedSearchPathLookupSubject, operatingSystem: OperatingSystem) throws(StackedSearchPathLookupError) -> Path { + let searchPathsList = [librarySearchPaths, fallbackLibrarySearchPaths] + for searchPaths in searchPathsList { + if let library = searchPaths.lookup(subject: subject, operatingSystem: operatingSystem) { + return library + } + } + throw .unableToFind(subject: subject, operatingSystem: operatingSystem, searchPaths: searchPathsList) + } +} + /// The ToolchainRegistry manages the set of registered toolchains. public final class ToolchainRegistry: @unchecked Sendable { + @_spi(Testing) public struct SearchPath: Sendable { + public var path: Path + public var strict: Bool + public var aliases: Set = [] + + public init(path: Path, strict: Bool, aliases: Set = []) { + self.path = path + self.strict = strict + self.aliases = aliases + } + } + let fs: any FSProxy let hostOperatingSystem: OperatingSystem @@ -427,17 +453,19 @@ public final class ToolchainRegistry: @unchecked Sendable { public static let appleToolchainIdentifierPrefix: String = "com.apple.dt.toolchain." - @_spi(Testing) public init(delegate: any ToolchainRegistryDelegate, searchPaths: [(Path, strict: Bool)], fs: any FSProxy, hostOperatingSystem: OperatingSystem) async { + @_spi(Testing) public init(delegate: any ToolchainRegistryDelegate, searchPaths: [SearchPath], fs: any FSProxy, hostOperatingSystem: OperatingSystem) async { self.fs = fs self.hostOperatingSystem = hostOperatingSystem - for (path, strict) in searchPaths { + for searchPath in searchPaths { + let path = searchPath.path + let strict = searchPath.strict if !strict && !fs.exists(path) { continue } do { - try await registerToolchainsInDirectory(path, strict: strict, operatingSystem: hostOperatingSystem, delegate: delegate) + try await registerToolchainsInDirectory(path, strict: strict, aliases: searchPath.aliases, operatingSystem: hostOperatingSystem, delegate: delegate) } catch let err { delegate.issue(strict: strict, path, "failed to load toolchains in \(path.str): \(err)") @@ -462,7 +490,7 @@ public final class ToolchainRegistry: @unchecked Sendable { } /// Register all the toolchains in the given directory. - private func registerToolchainsInDirectory(_ path: Path, strict: Bool, operatingSystem: OperatingSystem, delegate: any ToolchainRegistryDelegate) async throws { + private func registerToolchainsInDirectory(_ path: Path, strict: Bool, aliases: Set, operatingSystem: OperatingSystem, delegate: any ToolchainRegistryDelegate) async throws { let toolchainPaths: [Path] = try fs.listdir(path) .sorted() .map { path.join($0) } @@ -475,7 +503,7 @@ public final class ToolchainRegistry: @unchecked Sendable { guard toolchainPath.basenameWithoutSuffix != "swift-latest" else { continue } do { - let toolchain = try await Toolchain(path: toolchainPath, operatingSystem: operatingSystem, fs: fs, pluginManager: delegate.pluginManager, platformRegistry: delegate.platformRegistry) + let toolchain = try await Toolchain(path: toolchainPath, operatingSystem: operatingSystem, aliases: aliases, fs: fs, pluginManager: delegate.pluginManager, platformRegistry: delegate.platformRegistry) try register(toolchain) } catch let err { delegate.issue(strict: strict, toolchainPath, "failed to load toolchain: \(err)") @@ -505,24 +533,15 @@ public final class ToolchainRegistry: @unchecked Sendable { /// Look up the toolchain with the given identifier. public func lookup(_ identifier: String) -> Toolchain? { let lowercasedIdentifier = identifier.lowercased() - if hostOperatingSystem == .macOS { - if ["default", "xcode"].contains(lowercasedIdentifier) { - return toolchainsByIdentifier[ToolchainRegistry.defaultToolchainIdentifier] ?? toolchainsByAlias[lowercasedIdentifier] - } else { - return toolchainsByIdentifier[identifier] ?? toolchainsByAlias[lowercasedIdentifier] - } + if ["default", "xcode"].contains(lowercasedIdentifier) { + return toolchainsByIdentifier[ToolchainRegistry.defaultToolchainIdentifier] ?? toolchainsByAlias[lowercasedIdentifier] } else { - // On non-Darwin, assume if there is only one registered toolchain, it is the default. - if ["default", "xcode"].contains(lowercasedIdentifier) || identifier == ToolchainRegistry.defaultToolchainIdentifier { - return toolchainsByIdentifier[ToolchainRegistry.defaultToolchainIdentifier] ?? toolchainsByAlias[lowercasedIdentifier] ?? toolchainsByIdentifier.values.only - } else { - return toolchainsByIdentifier[identifier] ?? toolchainsByAlias[lowercasedIdentifier] - } + return toolchainsByIdentifier[identifier] ?? toolchainsByAlias[lowercasedIdentifier] } } public var defaultToolchain: Toolchain? { - return self.lookup(ToolchainRegistry.defaultToolchainIdentifier) + return self.lookup("default") } public var toolchains: Set { diff --git a/Tests/SWBAndroidPlatformTests/AndroidSDKTests.swift b/Tests/SWBAndroidPlatformTests/AndroidSDKTests.swift index b567c294..9439339e 100644 --- a/Tests/SWBAndroidPlatformTests/AndroidSDKTests.swift +++ b/Tests/SWBAndroidPlatformTests/AndroidSDKTests.swift @@ -13,7 +13,7 @@ import Foundation @_spi(Testing) import SWBAndroidPlatform import SWBTestSupport -import SWBUtil +@_spi(Testing) import SWBUtil import Testing @Suite @@ -415,7 +415,11 @@ fileprivate struct AndroidSDKTests { return ndkVersionPath } let host = try ProcessInfo.processInfo.hostOperatingSystem() - try await block(host, fs, sdkPath, ndkVersionPaths.map { try AbsolutePath(validating: $0) }) + + // Clear the environment to avoid influence from Android SDK/NDK environment overrides + try await withEnvironment([:], clean: true) { + try await block(host, fs, sdkPath, ndkVersionPaths.map { try AbsolutePath(validating: $0) }) + } } private func withNDKVersion(fs: PseudoFS = PseudoFS(), sdkPath: AbsolutePath = .root, version: Version, _ block: (OperatingSystem, any FSProxy, AbsolutePath, AbsolutePath) async throws -> ()) async throws { diff --git a/Tests/SWBCoreTests/CoreTests.swift b/Tests/SWBCoreTests/CoreTests.swift index 7cd8b525..ad6305e4 100644 --- a/Tests/SWBCoreTests/CoreTests.swift +++ b/Tests/SWBCoreTests/CoreTests.swift @@ -460,8 +460,8 @@ import SWBServiceCore } let toolchainPaths = try #require(core?.toolchainPaths) for expectedPathString in expectedPathStrings { - #expect(toolchainPaths.contains(where: { paths in - paths.0 == Path(expectedPathString) && paths.strict == false + #expect(toolchainPaths.contains(where: { searchPath in + searchPath.path == Path(expectedPathString) && searchPath.strict == false }), "Unable to find \(expectedPathString)") } diff --git a/Tests/SWBCoreTests/ToolchainRegistryTests.swift b/Tests/SWBCoreTests/ToolchainRegistryTests.swift index 4a5a495d..678b42dc 100644 --- a/Tests/SWBCoreTests/ToolchainRegistryTests.swift +++ b/Tests/SWBCoreTests/ToolchainRegistryTests.swift @@ -107,7 +107,7 @@ import SWBServiceCore return } let delegate = TestDataDelegate(pluginManager: core.pluginManager) - let registry = await ToolchainRegistry(delegate: delegate, searchPaths: [(tmpDirPath, strict: strict)], fs: fs, hostOperatingSystem: core.hostOperatingSystem) + let registry = await ToolchainRegistry(delegate: delegate, searchPaths: [.init(path: tmpDirPath, strict: strict)], fs: fs, hostOperatingSystem: core.hostOperatingSystem) try perform(registry, delegate.warnings, delegate.errors) } }